Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Dispose of file handles in BSA creation, don't load unused patches.

Browse Source
This commit is contained in:
Timothy Baldridge
2021-01-15 07:00:58 -07:00
parent c8abca742a
commit a754d9ae68
5 changed files with 41 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Compression.BSA/BA2Builder.cs
View File

@ -238,6 +238,7 @@ namespace Compression.BSA
await builder._dataSrc.CopyToAsync(ds); await builder._dataSrc.CopyToAsync(ds);
} }
await builder._dataSrc.DisposeAsync();
builder._dataSrc = slab.Allocate(ms.Length); builder._dataSrc = slab.Allocate(ms.Length);
ms.Position = 0; ms.Position = 0;
await ms.CopyToAsync(builder._dataSrc); await ms.CopyToAsync(builder._dataSrc);

4
Compression.BSA/BSA/Builder/BSABuilder.cs
View File

@ -276,6 +276,8 @@ namespace Compression.BSA
{ {
await _srcData.CopyToWithStatusAsync(_srcData.Length, w, $"Compressing {_path}"); await _srcData.CopyToWithStatusAsync(_srcData.Length, w, $"Compressing {_path}");
} }
await _srcData.DisposeAsync();
_srcData = _bsa._slab.Allocate(r.Length); _srcData = _bsa._slab.Allocate(r.Length);
r.Position = 0; r.Position = 0;
await r.CopyToWithStatusAsync(r.Length, _srcData, $"Writing {_path}"); await r.CopyToWithStatusAsync(r.Length, _srcData, $"Writing {_path}");
@ -291,6 +293,8 @@ namespace Compression.BSA
w.IsStreamOwner = false; w.IsStreamOwner = false;
await _srcData.CopyToWithStatusAsync(_srcData.Length, w, $"Compressing {_path}"); await _srcData.CopyToWithStatusAsync(_srcData.Length, w, $"Compressing {_path}");
} }
await _srcData.DisposeAsync();
_srcData = _bsa._slab.Allocate(r.Length); _srcData = _bsa._slab.Allocate(r.Length);
r.Position = 0; r.Position = 0;
await r.CopyToWithStatusAsync(r.Length, _srcData, $"Writing {_path}"); await r.CopyToWithStatusAsync(r.Length, _srcData, $"Writing {_path}");

42
Wabbajack.Common/Patches.cs
View File

@ -26,7 +26,9 @@ namespace Wabbajack.Common
PatchSize BLOB, PatchSize BLOB,
Patch BLOB, Patch BLOB,
PRIMARY KEY (FromHash, ToHash)) PRIMARY KEY (FromHash, ToHash))
WITHOUT ROWID"; WITHOUT ROWID;";
cmd.ExecuteNonQuery(); cmd.ExecuteNonQuery();
} }
@ -82,10 +84,10 @@ namespace Wabbajack.Common
else else
{ {
if (TryGetPatch(srcHash, destHash, out var array)) if (TryGetPatch(srcHash, destHash, out var entry))
{ {
await patchOutStream!.WriteAsync(array); await patchOutStream!.WriteAsync(await entry.GetData());
return array.Length; return entry.PatchSize;
} }
} }
@ -122,27 +124,43 @@ namespace Wabbajack.Common
return patchStream.Position; return patchStream.Position;
} }
public static bool TryGetPatch(Hash fromHash, Hash toHash, [MaybeNullWhen(false)] out byte[] array) public static bool TryGetPatch(Hash fromHash, Hash toHash, [MaybeNullWhen(false)] out CacheEntry found)
{ {
using var cmd = new SQLiteCommand(_conn); using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"SELECT PatchSize, Patch FROM PatchCache WHERE FromHash = @fromHash AND ToHash = @toHash"; cmd.CommandText = @"SELECT PatchSize FROM PatchCache WHERE FromHash = @fromHash AND ToHash = @toHash";
cmd.Parameters.AddWithValue("@fromHash", (long)fromHash); cmd.Parameters.AddWithValue("@fromHash", (long)fromHash);
cmd.Parameters.AddWithValue("@toHash", (long)toHash); cmd.Parameters.AddWithValue("@toHash", (long)toHash);
using var rdr = cmd.ExecuteReader(); using var rdr = cmd.ExecuteReader();
while (rdr.Read()) while (rdr.Read())
{ {
array = new byte[rdr.GetInt64(0)]; found = new CacheEntry(fromHash, toHash, rdr.GetInt64(0));
rdr.GetBytes(1, 0, array, 0, array.Length);
return true; return true;
} }
array = Array.Empty<byte>(); found = default;
return false; return false;
}
public record CacheEntry(Hash From, Hash To, long PatchSize)
{
public async Task<byte[]> GetData()
{
await using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"SELECT PatchSize, Patch FROM PatchCache WHERE FromHash = @fromHash AND ToHash = @toHash";
cmd.Parameters.AddWithValue("@fromHash", (long)From);
cmd.Parameters.AddWithValue("@toHash", (long)To);
await using var rdr = await cmd.ExecuteReaderAsync();
while (await rdr.ReadAsync())
{
var array = new byte[rdr.GetInt64(0)];
rdr.GetBytes(1, 0, array, 0, array.Length);
return array;
}
return Array.Empty<byte>();
}
} }
public static void VacuumDatabase() public static void VacuumDatabase()
@ -186,7 +204,7 @@ namespace Wabbajack.Common
public static Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, Stream? patchOutStream = null) => public static Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, Stream? patchOutStream = null) =>
PatchCache.CreatePatchCached(srcStream, srcHash, destStream, destHash, patchOutStream); PatchCache.CreatePatchCached(srcStream, srcHash, destStream, destHash, patchOutStream);
public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch) => public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out PatchCache.CacheEntry ePatch) =>
PatchCache.TryGetPatch(foundHash, fileHash, out ePatch); PatchCache.TryGetPatch(foundHash, fileHash, out ePatch);
} }
} }

2
Wabbajack.Lib/ACompiler.cs
View File

@ -430,7 +430,7 @@ namespace Wabbajack.Lib
pfa.FromFile = file; pfa.FromFile = file;
pfa.FromHash = file.Hash; pfa.FromHash = file.Hash;
pfa.ArchiveHashPath = file.MakeRelativePaths(); pfa.ArchiveHashPath = file.MakeRelativePaths();
pfa.PatchID = await IncludeFile(bytes!); pfa.PatchID = await IncludeFile(await bytes!.GetData());
} }
}); });

9
Wabbajack.Lib/CompilationSteps/IncludePatches.cs
View File

@ -110,7 +110,7 @@ namespace Wabbajack.Lib.CompilationSteps
var (_, bytes, file) = PickPatch(_compiler, patches); var (_, bytes, file) = PickPatch(_compiler, patches);
e.FromHash = file.Hash; e.FromHash = file.Hash;
e.ArchiveHashPath = file.MakeRelativePaths(); e.ArchiveHashPath = file.MakeRelativePaths();
e.PatchID = await _compiler.IncludeFile(bytes!); e.PatchID = await _compiler.IncludeFile(await bytes!.GetData());
} }
else else
{ {
@ -128,11 +128,11 @@ namespace Wabbajack.Lib.CompilationSteps
return e; return e;
} }
public static (bool, byte[], VirtualFile) PickPatch(ACompiler compiler, IEnumerable<(bool foundHash, byte[]? data, VirtualFile file)> patches) public static (bool, PatchCache.CacheEntry, VirtualFile) PickPatch(ACompiler compiler, IEnumerable<(bool foundHash, PatchCache.CacheEntry? data, VirtualFile file)> patches)
{ {
var ordered = patches var ordered = patches
.Select(f => (f.foundHash, f.data!, f.file)) .Select(f => (f.foundHash, f.data!, f.file))
.OrderBy(f => f.Item2.Length) .OrderBy(f => f.Item2.PatchSize)
.ToArray(); .ToArray();
var primaryChoice = ordered.FirstOrDefault(itm => var primaryChoice = ordered.FirstOrDefault(itm =>
@ -148,7 +148,8 @@ namespace Wabbajack.Lib.CompilationSteps
}); });
// If we didn't find a file from an archive or the primary game, use a secondary game file. // If we didn't find a file from an archive or the primary game, use a secondary game file.
return primaryChoice != default ? primaryChoice : ordered.FirstOrDefault(); var result = primaryChoice != default ? primaryChoice : ordered.FirstOrDefault();
return result;
} }
private AbsolutePath ModForFile(AbsolutePath file) private AbsolutePath ModForFile(AbsolutePath file)