From 27025db484cb21dd13b15b51d3fa8840f5c911d0 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 24 Feb 2020 16:18:29 -0700 Subject: [PATCH] Switch from BSDiff to OctoDiff for patch generation --- CHANGELOG.md | 1 + Wabbajack.Common/OctoDiff.cs | 46 ++++++++++++++++++++++++ Wabbajack.Common/Util/Percent.cs | 5 +++ Wabbajack.Common/Utils.cs | 37 +++++++++++++++++-- Wabbajack.Common/Wabbajack.Common.csproj | 1 + Wabbajack.Lib/AInstaller.cs | 11 +++--- Wabbajack.Lib/MO2Installer.cs | 2 +- Wabbajack.Lib/zEditIntegration.cs | 2 +- Wabbajack.Test/TestUtils.cs | 18 ++++++++-- Wabbajack.Test/UtilsTests.cs | 44 +++++++++++++++++++++++ 10 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 Wabbajack.Common/OctoDiff.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index e5a26620..8e4e0c49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ #### Version - 0.9.22.0 * Server side fixes for CORS support and FTP uploads +* Use OctoDiff instead of BSDiff for better performance during diff generation #### Version - 0.9.21.0 - 2/23/2020 * Fix never ending hash issue diff --git a/Wabbajack.Common/OctoDiff.cs b/Wabbajack.Common/OctoDiff.cs new file mode 100644 index 00000000..132beb0d --- /dev/null +++ b/Wabbajack.Common/OctoDiff.cs @@ -0,0 +1,46 @@ +using System; +using System.IO; +using Octodiff.Core; +using Octodiff.Diagnostics; + +namespace Wabbajack.Common +{ + public class OctoDiff + { + private static ProgressReporter reporter = new ProgressReporter(); + public static void Create(byte[] oldData, byte[] newData, Stream output) + { + using var signature = CreateSignature(oldData); + using var oldStream = new MemoryStream(oldData); + using var newStream = new MemoryStream(newData); + var db = new DeltaBuilder {ProgressReporter = reporter}; + db.BuildDelta(newStream, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output))); + } + + private static Stream CreateSignature(byte[] oldData) + { + Utils.Status("Creating Patch Signature"); + using var oldDataStream = new MemoryStream(oldData); + var sigStream = new MemoryStream(); + var signatureBuilder = new SignatureBuilder(); + signatureBuilder.Build(oldDataStream, new SignatureWriter(sigStream)); + sigStream.Position = 0; + return sigStream; + } + + private class ProgressReporter : IProgressReporter + { + public void ReportProgress(string operation, long currentPosition, long total) + { + Utils.Status(operation, new Percent(total, currentPosition)); + } + } + + public static void Apply(Stream input, Func openPatchStream, Stream output) + { + using var deltaStream = openPatchStream(); + var deltaApplier = new DeltaApplier(); + deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, reporter), output); + } + } +} diff --git a/Wabbajack.Common/Util/Percent.cs b/Wabbajack.Common/Util/Percent.cs index e9adde5d..bd0bc749 100644 --- a/Wabbajack.Common/Util/Percent.cs +++ b/Wabbajack.Common/Util/Percent.cs @@ -24,6 +24,11 @@ namespace Wabbajack.Common } } + public Percent(long max, long current) : this((double)current / max) + { + + } + public Percent(double d) : this(d, check: true) { diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index ef84d8b8..cab10aa9 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -926,8 +926,19 @@ namespace Wabbajack.Common { if (File.Exists(cacheFile)) { - await using var f = File.OpenRead(cacheFile); - await f.CopyToAsync(output); + RETRY_OPEN: + try + { + await using var f = File.OpenRead(cacheFile); + await f.CopyToAsync(output); + } + catch (IOException) + { + // Race condition with patch caching + await Task.Delay(100); + goto RETRY_OPEN; + } + } else { @@ -936,7 +947,7 @@ namespace Wabbajack.Common await using (var f = File.Open(tmpName, System.IO.FileMode.Create)) { Status("Creating Patch"); - BSDiff.Create(a, b, f); + OctoDiff.Create(a, b, f); } RETRY: @@ -973,6 +984,26 @@ namespace Wabbajack.Common return false; } + public static void ApplyPatch(Stream input, Func openPatchStream, Stream output) + { + using var ps = openPatchStream(); + using var br = new BinaryReader(ps); + var bytes = br.ReadBytes(8); + var str = Encoding.ASCII.GetString(bytes); + switch (str) + { + case "BSDIFF40": + BSDiff.Apply(input, openPatchStream, output); + return; + case "OCTODELT": + OctoDiff.Apply(input, openPatchStream, output); + return; + default: + throw new Exception($"No diff dispatch for: {str}"); + } + + } + /* public static void Warning(string s) { diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 8d20c1b1..ae13b6a0 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -36,6 +36,7 @@ + diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 5c5f8596..b98c4224 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -199,9 +199,10 @@ namespace Wabbajack.Lib onFinish(); // Now patch all the files from this archive - foreach (var toPatch in grouping.OfType()) - using (var patchStream = new MemoryStream()) + await grouping.OfType() + .PMap(queue, async toPatch => { + await using var patchStream = new MemoryStream(); Status($"Patching {Path.GetFileName(toPatch.To)}"); // Read in the patch data @@ -214,16 +215,16 @@ namespace Wabbajack.Lib File.Delete(toFile); // Patch it - using (var outStream = File.Open(toFile, FileMode.Create)) + await using (var outStream = File.Open(toFile, FileMode.Create)) { - BSDiff.Apply(oldData, () => new MemoryStream(patchData), outStream); + Utils.ApplyPatch(oldData, () => new MemoryStream(patchData), outStream); } Status($"Verifying Patch {Path.GetFileName(toPatch.To)}"); var resultSha = toFile.FileHash(); if (resultSha != toPatch.Hash) throw new InvalidDataException($"Invalid Hash for {toPatch.To} after patching"); - } + }); } public async Task DownloadArchives() diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index 24f4a02c..55f97b8d 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -298,7 +298,7 @@ namespace Wabbajack.Lib using (var output = File.Open(toFile, FileMode.Create)) using (var input = File.OpenRead(gameFile)) { - BSDiff.Apply(input, () => new MemoryStream(patchData), output); + Utils.ApplyPatch(input, () => new MemoryStream(patchData), output); } } diff --git a/Wabbajack.Lib/zEditIntegration.cs b/Wabbajack.Lib/zEditIntegration.cs index b1617655..c855e91a 100644 --- a/Wabbajack.Lib/zEditIntegration.cs +++ b/Wabbajack.Lib/zEditIntegration.cs @@ -176,7 +176,7 @@ namespace Wabbajack.Lib var patch_data = installer.LoadBytesFromPath(m.PatchID); using (var fs = File.Open(Path.Combine(installer.OutputFolder, m.To), FileMode.Create)) - BSDiff.Apply(new MemoryStream(src_data), () => new MemoryStream(patch_data), fs); + Utils.ApplyPatch(new MemoryStream(src_data), () => new MemoryStream(patch_data), fs); }); } } diff --git a/Wabbajack.Test/TestUtils.cs b/Wabbajack.Test/TestUtils.cs index 3be92886..e94dfe3c 100644 --- a/Wabbajack.Test/TestUtils.cs +++ b/Wabbajack.Test/TestUtils.cs @@ -15,16 +15,16 @@ namespace Wabbajack.Test { public class TestUtils : IDisposable { + private static Random _rng = new Random(); public TestUtils() { - RNG = new Random(); ID = RandomName(); WorkingDirectory = Path.Combine(Directory.GetCurrentDirectory(), "tmp_data"); } public string WorkingDirectory { get;} public string ID { get; } - public Random RNG { get; } + public Random RNG => _rng; public Game Game { get; set; } @@ -113,6 +113,15 @@ namespace Wabbajack.Test File.WriteAllBytes(full_path, bytes); } + public static byte[] RandomData(int? size = null, int maxSize = 1024) + { + if (size == null) + size = _rng.Next(1, maxSize); + var arr = new byte[(int) size]; + _rng.NextBytes(arr); + return arr; + } + public void Dispose() { var exts = new [] {".md", ".exe"}; @@ -250,5 +259,10 @@ namespace Wabbajack.Test GenerateRandomFileData(full_path, i); return full_path; } + + public static object RandomeOne(params object[] opts) + { + return opts[_rng.Next(0, opts.Length)]; + } } } diff --git a/Wabbajack.Test/UtilsTests.cs b/Wabbajack.Test/UtilsTests.cs index cd913a43..8bde6f9c 100644 --- a/Wabbajack.Test/UtilsTests.cs +++ b/Wabbajack.Test/UtilsTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -11,6 +12,7 @@ namespace Wabbajack.Test [TestClass] public class UtilsTests { + [TestMethod] public void IsInPathTests() { @@ -22,5 +24,47 @@ namespace Wabbajack.Test Assert.IsTrue("c:\\foo\\bar.exe".IsInPath("c:\\foo\\")); Assert.IsTrue("c:\\foo\\bar\\".IsInPath("c:\\foo\\")); } + + + [TestMethod] + [DataTestMethod] + [DynamicData(nameof(PatchData), DynamicDataSourceType.Method)] + public async Task DiffCreateAndApply(byte[] src, byte[] dest, DiffMethod method) + { + await using var ms = new MemoryStream(); + switch (method) + { + case DiffMethod.Default: + await Utils.CreatePatch(src, dest, ms); + break; + case DiffMethod.BSDiff: + BSDiff.Create(src, dest, ms); + break; + case DiffMethod.OctoDiff: + OctoDiff.Create(src, dest, ms); + break; + default: + throw new ArgumentOutOfRangeException(nameof(method), method, null); + } + + ms.Position = 0; + var patch = ms.ToArray(); + await using var resultStream = new MemoryStream(); + Utils.ApplyPatch(new MemoryStream(src), () => new MemoryStream(patch), resultStream); + CollectionAssert.AreEqual(dest, resultStream.ToArray()); + } + + + public enum DiffMethod + { + Default, + BSDiff, + OctoDiff + } + public static IEnumerable PatchData() + { + var maxSize = 1024 * 1024 * 8; + return Enumerable.Range(0, 10).Select(x => new[] {TestUtils.RandomData(maxSize:maxSize), TestUtils.RandomData(maxSize:maxSize), TestUtils.RandomeOne(DiffMethod.Default, DiffMethod.OctoDiff, DiffMethod.BSDiff)}); + } } }