From d48489d4fe185da61e23be2dc172f64a9f8c201f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 30 Jun 2020 17:09:59 -0600 Subject: [PATCH] Consider all possible binary patches and select the smallest --- Compression.BSA/BA2Builder.cs | 4 +- Compression.BSA/BA2Reader.cs | 6 +-- Wabbajack.Common/Patches.cs | 18 +++++-- Wabbajack.Lib/CompilationSteps/DirectMatch.cs | 2 +- .../CompilationSteps/IncludePatches.cs | 41 +++++++------- Wabbajack.Lib/Data.cs | 8 +++ Wabbajack.Lib/MO2Compiler.cs | 54 +++++++++++++------ 7 files changed, 89 insertions(+), 44 deletions(-) diff --git a/Compression.BSA/BA2Builder.cs b/Compression.BSA/BA2Builder.cs index c182bd7c..217e3ce0 100644 --- a/Compression.BSA/BA2Builder.cs +++ b/Compression.BSA/BA2Builder.cs @@ -128,7 +128,7 @@ namespace Compression.BSA public void WriteHeader(BinaryWriter bw) { bw.Write(_state.NameHash); - bw.Write(Encoding.ASCII.GetBytes(_state.Extension)); + bw.Write(Encoding.UTF8.GetBytes(_state.Extension)); bw.Write(_state.DirHash); bw.Write(_state.Unk8); bw.Write((byte)_chunks.Count); @@ -254,7 +254,7 @@ namespace Compression.BSA public void WriteHeader(BinaryWriter wtr) { wtr.Write(_state.NameHash); - wtr.Write(Encoding.ASCII.GetBytes(_state.Extension)); + wtr.Write(Encoding.UTF8.GetBytes(_state.Extension)); wtr.Write(_state.DirHash); wtr.Write(_state.Flags); _offsetOffset = wtr.BaseStream.Position; diff --git a/Compression.BSA/BA2Reader.cs b/Compression.BSA/BA2Reader.cs index b5d20ed9..1fcf390b 100644 --- a/Compression.BSA/BA2Reader.cs +++ b/Compression.BSA/BA2Reader.cs @@ -50,7 +50,7 @@ namespace Compression.BSA private BA2Reader(Stream stream) { _stream = stream; - _rdr = new BinaryReader(_stream, Encoding.UTF7); + _rdr = new BinaryReader(_stream, Encoding.UTF8); } private async Task LoadHeaders() @@ -173,7 +173,7 @@ namespace Compression.BSA var _rdr = ba2Reader._rdr; _nameHash = _rdr.ReadUInt32(); FullPath = _nameHash.ToString("X"); - _extension = Encoding.UTF7.GetString(_rdr.ReadBytes(4)); + _extension = Encoding.UTF8.GetString(_rdr.ReadBytes(4)); _dirHash = _rdr.ReadUInt32(); _unk8 = _rdr.ReadByte(); _numChunks = _rdr.ReadByte(); @@ -463,7 +463,7 @@ namespace Compression.BSA var _rdr = ba2Reader._rdr; _nameHash = _rdr.ReadUInt32(); FullPath = _nameHash.ToString("X"); - _extension = Encoding.UTF7.GetString(_rdr.ReadBytes(4)); + _extension = Encoding.UTF8.GetString(_rdr.ReadBytes(4)); _dirHash = _rdr.ReadUInt32(); _flags = _rdr.ReadUInt32(); _offset = _rdr.ReadUInt64(); diff --git a/Wabbajack.Common/Patches.cs b/Wabbajack.Common/Patches.cs index 8f6b66fa..1b21da54 100644 --- a/Wabbajack.Common/Patches.cs +++ b/Wabbajack.Common/Patches.cs @@ -49,15 +49,17 @@ namespace Wabbajack.Common await patch.CopyToAsync(output); } - public static async Task CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash, - Stream patchOutStream) + public static async Task CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash, + Stream? patchOutStream = null) { var key = PatchKey(srcHash, destHash); var patch = _patchCache!.Get(key); if (patch != null) { + if (patchOutStream == null) return patch.Length; + await patchOutStream.WriteAsync(patch); - return; + return patch.Length; } Status("Creating Patch"); @@ -66,8 +68,12 @@ namespace Wabbajack.Common OctoDiff.Create(srcStream, destStream, sigStream, patchStream); _patchCache.Put(key, patchStream.ToArray()); + if (patchOutStream == null) return patchStream.Position; + patchStream.Position = 0; await patchStream.CopyToAsync(patchOutStream); + + return patchStream.Position; } public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch) @@ -86,6 +92,12 @@ namespace Wabbajack.Common } + public static bool HavePatch(Hash foundHash, Hash fileHash) + { + var key = PatchKey(foundHash, fileHash); + return _patchCache!.Get(key) != null; + } + public static void ApplyPatch(Stream input, Func openPatchStream, Stream output) { using var ps = openPatchStream(); diff --git a/Wabbajack.Lib/CompilationSteps/DirectMatch.cs b/Wabbajack.Lib/CompilationSteps/DirectMatch.cs index 327b6c1d..75973753 100644 --- a/Wabbajack.Lib/CompilationSteps/DirectMatch.cs +++ b/Wabbajack.Lib/CompilationSteps/DirectMatch.cs @@ -19,7 +19,7 @@ namespace Wabbajack.Lib.CompilationSteps var adata = compiler.ArchivesByFullPath[archive.AbsoluteName]; if (adata.State is GameFileSourceDownloader.State gs) { - return gs.Game == compiler.CompilingGame.Game ? 1 : 3; + return gs.Game == compiler.CompilingGame.Game ? 2 : 3; } return 1; } diff --git a/Wabbajack.Lib/CompilationSteps/IncludePatches.cs b/Wabbajack.Lib/CompilationSteps/IncludePatches.cs index 4fb886bc..74e8476d 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludePatches.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludePatches.cs @@ -53,53 +53,54 @@ namespace Wabbajack.Lib.CompilationSteps var installationFile = (string?)modIni?.General?.installationFile; - VirtualFile? found = null; + VirtualFile[] found = {}; // Find based on exact file name + ext if (choices != null && installationFile != null) { var relName = (RelativePath)Path.GetFileName(installationFile); - found = choices.FirstOrDefault( - f => f.FilesInFullPath.First().Name.FileName == relName); + found = choices.Where(f => f.FilesInFullPath.First().Name.FileName == relName).ToArray(); } // Find based on file name only (not ext) - if (found == null && choices != null) + if (found.Length == 0 && choices != null) { - found = choices.OrderBy(f => f.NestingFactor) - .ThenByDescending(f => (f.FilesInFullPath.First() ?? f).LastModified) - .First(); + found = choices.ToArray(); } // Find based on matchAll= in [General] in meta.ini var matchAllName = (string?)modIni?.General?.matchAll; - if (matchAllName != null) + if (matchAllName != null && found.Length == 0) { var relName = (RelativePath)Path.GetFileName(matchAllName); if (_indexedByName.TryGetValue(relName, out var arch)) { // Just match some file in the archive based on the smallest delta difference - found = arch.SelectMany(a => a.ThisAndAllChildren) - .OrderBy(o => DirectMatch.GetFilePriority(_mo2Compiler, o)) - .ThenBy(o => Math.Abs(o.Size - source.File.Size)) - .First(); + found = arch.SelectMany(a => a.ThisAndAllChildren).ToArray(); } } - if (found == null) + if (found.Length == 0) return null; + var e = source.EvolveTo(); - e.FromHash = found.Hash; - e.ArchiveHashPath = found.MakeRelativePaths(); - e.To = source.Path; - e.Hash = source.File.Hash; - if (Utils.TryGetPatch(found.Hash, source.File.Hash, out var data)) + var patches = found.Select(c => (Utils.TryGetPatch(c.Hash, source.File.Hash, out var data), data, c)) + .ToArray(); + + if (patches.All(p => p.Item1)) { - e.PatchID = await _compiler.IncludeFile(data); + var (_, bytes, file) = patches.OrderBy(f => f.data!.Length).First(); + e.FromHash = file.Hash; + e.ArchiveHashPath = file.MakeRelativePaths(); + e.PatchID = await _compiler.IncludeFile(bytes!); } - + else + { + e.Choices = found; + } + return e; } diff --git a/Wabbajack.Lib/Data.cs b/Wabbajack.Lib/Data.cs index cc0cab5e..b3a26413 100644 --- a/Wabbajack.Lib/Data.cs +++ b/Wabbajack.Lib/Data.cs @@ -240,6 +240,14 @@ namespace Wabbajack.Lib /// The file to apply to the source file to patch it /// public RelativePath PatchID { get; set; } + + + /// + /// During compilation this holds several possible files that could be used as a patch source. At the end + /// of compilation we'll go through all of these and find the smallest patch file. + /// + [JsonIgnore] + public VirtualFile[] Choices { get; set; } = { }; } [JsonName("SourcePatch")] diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 2cbfb1ea..05acfdaf 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -318,13 +318,14 @@ namespace Wabbajack.Lib UpdateTracker.NextStep("Verifying Files"); zEditIntegration.VerifyMerges(this); + UpdateTracker.NextStep("Building Patches"); + await BuildPatches(); + UpdateTracker.NextStep("Gathering Archives"); await GatherArchives(); UpdateTracker.NextStep("Including Archive Metadata"); await IncludeArchiveMetadata(); - UpdateTracker.NextStep("Building Patches"); - await BuildPatches(); UpdateTracker.NextStep("Gathering Metadata"); await GatherMetaData(); @@ -468,15 +469,21 @@ namespace Wabbajack.Lib { Info("Gathering patch files"); - await InstallDirectives.OfType() - .Where(p => p.PatchID == null) - .PMap(Queue, async p => - { - if (Utils.TryGetPatch(p.FromHash, p.Hash, out var bytes)) - p.PatchID = await IncludeFile(bytes); - }); + var toBuild = InstallDirectives.OfType() + .Where(p => p.Choices.Length > 0) + .SelectMany(p => p.Choices.Select(c => new PatchedFromArchive + { + To = p.To, + Hash = p.Hash, + ArchiveHashPath = c.MakeRelativePaths(), + FromFile = c, + Size = p.Size, + })) + .ToArray(); - var groups = InstallDirectives.OfType() + if (toBuild.Length == 0) return; + + var groups = toBuild .Where(p => p.PatchID == default) .GroupBy(p => p.ArchiveHashPath.BaseHash) .ToList(); @@ -485,7 +492,26 @@ namespace Wabbajack.Lib var absolutePaths = AllFiles.ToDictionary(e => e.Path, e => e.AbsolutePath); await groups.PMap(Queue, group => BuildArchivePatches(group.Key, group, absolutePaths)); - var firstFailedPatch = InstallDirectives.OfType().FirstOrDefault(f => f.PatchID == null); + + await InstallDirectives.OfType() + .Where(p => p.PatchID == default) + .PMap(Queue, async pfa => + { + var patches = pfa.Choices + .Select(c => (Utils.TryGetPatch(c.Hash, pfa.Hash, out var data), data, c)) + .ToArray(); + + if (patches.All(p => p.Item1)) + { + var (_, bytes, file) = patches.OrderBy(f => f.data!.Length).First(); + pfa.FromFile = file; + pfa.FromHash = file.Hash; + pfa.ArchiveHashPath = file.MakeRelativePaths(); + pfa.PatchID = await IncludeFile(bytes!); + } + }); + + var firstFailedPatch = InstallDirectives.OfType().FirstOrDefault(f => f.PatchID == default); if (firstFailedPatch != null) Error($"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}"); } @@ -503,11 +529,9 @@ namespace Wabbajack.Lib Status($"Patching {entry.To}"); var srcFile = byPath[string.Join("|", entry.ArchiveHashPath.Paths)]; await using var srcStream = await srcFile.OpenRead(); - await using var outputStream = await IncludeFile(out var id).Create(); - entry.PatchID = id; await using var destStream = await LoadDataForTo(entry.To, absolutePaths); - await Utils.CreatePatchCached(srcStream, srcFile.Hash, destStream, entry.Hash, outputStream); - Info($"Patch size {outputStream.Length} for {entry.To}"); + var patchSize = await Utils.CreatePatchCached(srcStream, srcFile.Hash, destStream, entry.Hash); + Info($"Patch size {patchSize} for {entry.To}"); }); }