Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Consider all possible binary patches and select the smallest

Browse Source
This commit is contained in:
Timothy Baldridge 2020-06-30 17:09:59 -06:00
parent bd5aef446b
commit d48489d4fe
7 changed files with 89 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Compression.BSA/BA2Builder.cs
View File

@ -128,7 +128,7 @@ namespace Compression.BSA
public void WriteHeader(BinaryWriter bw)
{
bw.Write(_state.NameHash);
bw.Write(Encoding.ASCII.GetBytes(_state.Extension));
bw.Write(Encoding.UTF8.GetBytes(_state.Extension));
bw.Write(_state.DirHash);
bw.Write(_state.Unk8);
bw.Write((byte)_chunks.Count);
@ -254,7 +254,7 @@ namespace Compression.BSA
public void WriteHeader(BinaryWriter wtr)
{
wtr.Write(_state.NameHash);
wtr.Write(Encoding.ASCII.GetBytes(_state.Extension));
wtr.Write(Encoding.UTF8.GetBytes(_state.Extension));
wtr.Write(_state.DirHash);
wtr.Write(_state.Flags);
_offsetOffset = wtr.BaseStream.Position;

6
Compression.BSA/BA2Reader.cs
View File

@ -50,7 +50,7 @@ namespace Compression.BSA
private BA2Reader(Stream stream)
{
_stream = stream;
_rdr = new BinaryReader(_stream, Encoding.UTF7);
_rdr = new BinaryReader(_stream, Encoding.UTF8);
}
private async Task LoadHeaders()
@ -173,7 +173,7 @@ namespace Compression.BSA
var _rdr = ba2Reader._rdr;
_nameHash = _rdr.ReadUInt32();
FullPath = _nameHash.ToString("X");
_extension = Encoding.UTF7.GetString(_rdr.ReadBytes(4));
_extension = Encoding.UTF8.GetString(_rdr.ReadBytes(4));
_dirHash = _rdr.ReadUInt32();
_unk8 = _rdr.ReadByte();
_numChunks = _rdr.ReadByte();
@ -463,7 +463,7 @@ namespace Compression.BSA
var _rdr = ba2Reader._rdr;
_nameHash = _rdr.ReadUInt32();
FullPath = _nameHash.ToString("X");
_extension = Encoding.UTF7.GetString(_rdr.ReadBytes(4));
_extension = Encoding.UTF8.GetString(_rdr.ReadBytes(4));
_dirHash = _rdr.ReadUInt32();
_flags = _rdr.ReadUInt32();
_offset = _rdr.ReadUInt64();

18
Wabbajack.Common/Patches.cs
View File

@ -49,15 +49,17 @@ namespace Wabbajack.Common
await patch.CopyToAsync(output);
}
public static async Task CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash,
Stream patchOutStream)
public static async Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash,
Stream? patchOutStream = null)
{
var key = PatchKey(srcHash, destHash);
var patch = _patchCache!.Get(key);
if (patch != null)
{
if (patchOutStream == null) return patch.Length;
await patchOutStream.WriteAsync(patch);
return;
return patch.Length;
}
Status("Creating Patch");
@ -66,8 +68,12 @@ namespace Wabbajack.Common
OctoDiff.Create(srcStream, destStream, sigStream, patchStream);
_patchCache.Put(key, patchStream.ToArray());
if (patchOutStream == null) return patchStream.Position;
patchStream.Position = 0;
await patchStream.CopyToAsync(patchOutStream);
return patchStream.Position;
}
public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch)
@ -86,6 +92,12 @@ namespace Wabbajack.Common
}
public static bool HavePatch(Hash foundHash, Hash fileHash)
{
var key = PatchKey(foundHash, fileHash);
return _patchCache!.Get(key) != null;
}
public static void ApplyPatch(Stream input, Func<Stream> openPatchStream, Stream output)
{
using var ps = openPatchStream();

2
Wabbajack.Lib/CompilationSteps/DirectMatch.cs
View File

@ -19,7 +19,7 @@ namespace Wabbajack.Lib.CompilationSteps
var adata = compiler.ArchivesByFullPath[archive.AbsoluteName];
if (adata.State is GameFileSourceDownloader.State gs)
{
return gs.Game == compiler.CompilingGame.Game ? 1 : 3;
return gs.Game == compiler.CompilingGame.Game ? 2 : 3;
}
return 1;
}

41
Wabbajack.Lib/CompilationSteps/IncludePatches.cs
View File

@ -53,53 +53,54 @@ namespace Wabbajack.Lib.CompilationSteps
var installationFile = (string?)modIni?.General?.installationFile;
VirtualFile? found = null;
VirtualFile[] found = {};
// Find based on exact file name + ext
if (choices != null && installationFile != null)
{
var relName = (RelativePath)Path.GetFileName(installationFile);
found = choices.FirstOrDefault(
f => f.FilesInFullPath.First().Name.FileName == relName);
found = choices.Where(f => f.FilesInFullPath.First().Name.FileName == relName).ToArray();
}
// Find based on file name only (not ext)
if (found == null && choices != null)
if (found.Length == 0 && choices != null)
{
found = choices.OrderBy(f => f.NestingFactor)
.ThenByDescending(f => (f.FilesInFullPath.First() ?? f).LastModified)
.First();
found = choices.ToArray();
}
// Find based on matchAll=<archivename> in [General] in meta.ini
var matchAllName = (string?)modIni?.General?.matchAll;
if (matchAllName != null)
if (matchAllName != null && found.Length == 0)
{
var relName = (RelativePath)Path.GetFileName(matchAllName);
if (_indexedByName.TryGetValue(relName, out var arch))
{
// Just match some file in the archive based on the smallest delta difference
found = arch.SelectMany(a => a.ThisAndAllChildren)
.OrderBy(o => DirectMatch.GetFilePriority(_mo2Compiler, o))
.ThenBy(o => Math.Abs(o.Size - source.File.Size))
.First();
found = arch.SelectMany(a => a.ThisAndAllChildren).ToArray();
}
}
if (found == null)
if (found.Length == 0)
return null;
var e = source.EvolveTo<PatchedFromArchive>();
e.FromHash = found.Hash;
e.ArchiveHashPath = found.MakeRelativePaths();
e.To = source.Path;
e.Hash = source.File.Hash;
if (Utils.TryGetPatch(found.Hash, source.File.Hash, out var data))
var patches = found.Select(c => (Utils.TryGetPatch(c.Hash, source.File.Hash, out var data), data, c))
.ToArray();
if (patches.All(p => p.Item1))
{
e.PatchID = await _compiler.IncludeFile(data);
var (_, bytes, file) = patches.OrderBy(f => f.data!.Length).First();
e.FromHash = file.Hash;
e.ArchiveHashPath = file.MakeRelativePaths();
e.PatchID = await _compiler.IncludeFile(bytes!);
}
else
{
e.Choices = found;
}
return e;
}

8
Wabbajack.Lib/Data.cs
View File

@ -240,6 +240,14 @@ namespace Wabbajack.Lib
/// The file to apply to the source file to patch it
/// </summary>
public RelativePath PatchID { get; set; }
/// <summary>
/// During compilation this holds several possible files that could be used as a patch source. At the end
/// of compilation we'll go through all of these and find the smallest patch file.
/// </summary>
[JsonIgnore]
public VirtualFile[] Choices { get; set; } = { };
}
[JsonName("SourcePatch")]

54
Wabbajack.Lib/MO2Compiler.cs
View File

@ -318,13 +318,14 @@ namespace Wabbajack.Lib
UpdateTracker.NextStep("Verifying Files");
zEditIntegration.VerifyMerges(this);
UpdateTracker.NextStep("Building Patches");
await BuildPatches();
UpdateTracker.NextStep("Gathering Archives");
await GatherArchives();
UpdateTracker.NextStep("Including Archive Metadata");
await IncludeArchiveMetadata();
UpdateTracker.NextStep("Building Patches");
await BuildPatches();
UpdateTracker.NextStep("Gathering Metadata");
await GatherMetaData();
@ -468,15 +469,21 @@ namespace Wabbajack.Lib
{
Info("Gathering patch files");
await InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => p.PatchID == null)
.PMap(Queue, async p =>
{
if (Utils.TryGetPatch(p.FromHash, p.Hash, out var bytes))
p.PatchID = await IncludeFile(bytes);
});
var toBuild = InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => p.Choices.Length > 0)
.SelectMany(p => p.Choices.Select(c => new PatchedFromArchive
{
To = p.To,
Hash = p.Hash,
ArchiveHashPath = c.MakeRelativePaths(),
FromFile = c,
Size = p.Size,
}))
.ToArray();
var groups = InstallDirectives.OfType<PatchedFromArchive>()
if (toBuild.Length == 0) return;
var groups = toBuild
.Where(p => p.PatchID == default)
.GroupBy(p => p.ArchiveHashPath.BaseHash)
.ToList();
@ -485,7 +492,26 @@ namespace Wabbajack.Lib
var absolutePaths = AllFiles.ToDictionary(e => e.Path, e => e.AbsolutePath);
await groups.PMap(Queue, group => BuildArchivePatches(group.Key, group, absolutePaths));
var firstFailedPatch = InstallDirectives.OfType<PatchedFromArchive>().FirstOrDefault(f => f.PatchID == null);
await InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => p.PatchID == default)
.PMap(Queue, async pfa =>
{
var patches = pfa.Choices
.Select(c => (Utils.TryGetPatch(c.Hash, pfa.Hash, out var data), data, c))
.ToArray();
if (patches.All(p => p.Item1))
{
var (_, bytes, file) = patches.OrderBy(f => f.data!.Length).First();
pfa.FromFile = file;
pfa.FromHash = file.Hash;
pfa.ArchiveHashPath = file.MakeRelativePaths();
pfa.PatchID = await IncludeFile(bytes!);
}
});
var firstFailedPatch = InstallDirectives.OfType<PatchedFromArchive>().FirstOrDefault(f => f.PatchID == default);
if (firstFailedPatch != null)
Error($"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}");
}
@ -503,11 +529,9 @@ namespace Wabbajack.Lib
Status($"Patching {entry.To}");
var srcFile = byPath[string.Join("|", entry.ArchiveHashPath.Paths)];
await using var srcStream = await srcFile.OpenRead();
await using var outputStream = await IncludeFile(out var id).Create();
entry.PatchID = id;
await using var destStream = await LoadDataForTo(entry.To, absolutePaths);
await Utils.CreatePatchCached(srcStream, srcFile.Hash, destStream, entry.Hash, outputStream);
Info($"Patch size {outputStream.Length} for {entry.To}");
var patchSize = await Utils.CreatePatchCached(srcStream, srcFile.Hash, destStream, entry.Hash);
Info($"Patch size {patchSize} for {entry.To}");
});
}