Switch from BSDiff to OctoDiff for patch generation

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 #### Version - 0.9.22.0
 * Server side fixes for CORS support and FTP uploads
+* Use OctoDiff instead of BSDiff for better performance during diff generation 
 
 #### Version - 0.9.21.0 - 2/23/2020
 * Fix never ending hash issue

Wabbajack.Common/OctoDiff.cs

@@ -0,0 +1,46 @@
+using System;
+using System.IO;
+using Octodiff.Core;
+using Octodiff.Diagnostics;
+
+namespace Wabbajack.Common
+{
+    public class OctoDiff
+    {
+        private static ProgressReporter reporter = new ProgressReporter();
+        public static void Create(byte[] oldData, byte[] newData, Stream output)
+        {
+            using var signature = CreateSignature(oldData);
+            using var oldStream = new MemoryStream(oldData);
+            using var newStream = new MemoryStream(newData);
+            var db = new DeltaBuilder {ProgressReporter = reporter};
+            db.BuildDelta(newStream, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
+        }
+
+        private static Stream CreateSignature(byte[] oldData)
+        {
+            Utils.Status("Creating Patch Signature");
+            using var oldDataStream = new MemoryStream(oldData);
+            var sigStream = new MemoryStream();
+            var signatureBuilder = new SignatureBuilder();
+            signatureBuilder.Build(oldDataStream, new SignatureWriter(sigStream));
+            sigStream.Position = 0;
+            return sigStream;
+        }
+
+        private class ProgressReporter : IProgressReporter
+        {
+            public void ReportProgress(string operation, long currentPosition, long total)
+            {
+                Utils.Status(operation, new Percent(total, currentPosition));
+            }
+        }
+
+        public static void Apply(Stream input, Func<Stream> openPatchStream, Stream output)
+        {
+            using var deltaStream = openPatchStream();
+            var deltaApplier = new DeltaApplier();
+            deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, reporter), output);
+        }
+    }
+}

Wabbajack.Common/Util/Percent.cs

@@ -24,6 +24,11 @@ namespace Wabbajack.Common
             }
         }
 
+        public Percent(long max, long current) : this((double)current / max)
+        {
+            
+        }
+
         public Percent(double d)
             : this(d, check: true)
         {

Wabbajack.Common/Utils.cs

@@ -926,8 +926,19 @@ namespace Wabbajack.Common
             {
                 if (File.Exists(cacheFile))
                 {
-                    await using var f = File.OpenRead(cacheFile);
-                    await f.CopyToAsync(output);
+                    RETRY_OPEN:
+                    try
+                    {
+                        await using var f = File.OpenRead(cacheFile);
+                        await f.CopyToAsync(output);
+                    }
+                    catch (IOException)
+                    {
+                        // Race condition with patch caching
+                        await Task.Delay(100);
+                        goto RETRY_OPEN;
+                    }
+
                 }
                 else
                 {
@@ -936,7 +947,7 @@ namespace Wabbajack.Common
                     await using (var f = File.Open(tmpName, System.IO.FileMode.Create))
                     {
                         Status("Creating Patch");
-                        BSDiff.Create(a, b, f);
+                        OctoDiff.Create(a, b, f);
                     }
 
                     RETRY:
@@ -973,6 +984,26 @@ namespace Wabbajack.Common
             return false;
         }
 
+        public static void ApplyPatch(Stream input, Func<Stream> openPatchStream, Stream output)
+        {
+            using var ps = openPatchStream();
+            using var br = new BinaryReader(ps);
+            var bytes = br.ReadBytes(8);
+            var str = Encoding.ASCII.GetString(bytes);
+            switch (str)
+            {
+                case "BSDIFF40":
+                    BSDiff.Apply(input, openPatchStream, output);
+                    return;
+                case "OCTODELT":
+                    OctoDiff.Apply(input, openPatchStream, output);
+                    return;
+                default:
+                    throw new Exception($"No diff dispatch for: {str}");
+            }
+
+        }
+
         /*
         public static void Warning(string s)
         {

Wabbajack.Common/Wabbajack.Common.csproj

@@ -36,6 +36,7 @@
         <PackageReference Include="ini-parser-netstandard" Version="2.5.2" />
         <PackageReference Include="Microsoft.Win32.Registry" Version="4.7.0" />
         <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+        <PackageReference Include="Octodiff" Version="1.2.1" />
         <PackageReference Include="OMODFramework" Version="2.0.0" />
         <PackageReference Include="ReactiveUI" Version="11.1.23" />
         <PackageReference Include="SharpZipLib" Version="1.2.0" />

Wabbajack.Lib/AInstaller.cs

@@ -199,9 +199,10 @@ namespace Wabbajack.Lib
             onFinish();
 
             // Now patch all the files from this archive
-            foreach (var toPatch in grouping.OfType<PatchedFromArchive>())
-                using (var patchStream = new MemoryStream())
+            await grouping.OfType<PatchedFromArchive>()
+                .PMap(queue, async toPatch =>
                 {
+                    await using var patchStream = new MemoryStream();
                     Status($"Patching {Path.GetFileName(toPatch.To)}");
                     // Read in the patch data
 
@@ -214,16 +215,16 @@ namespace Wabbajack.Lib
                     File.Delete(toFile);
 
                     // Patch it
-                    using (var outStream = File.Open(toFile, FileMode.Create))
+                    await using (var outStream = File.Open(toFile, FileMode.Create))
                     {
-                        BSDiff.Apply(oldData, () => new MemoryStream(patchData), outStream);
+                        Utils.ApplyPatch(oldData, () => new MemoryStream(patchData), outStream);
                     }
 
                     Status($"Verifying Patch {Path.GetFileName(toPatch.To)}");
                     var resultSha = toFile.FileHash();
                     if (resultSha != toPatch.Hash)
                         throw new InvalidDataException($"Invalid Hash for {toPatch.To} after patching");
-                }
+                });
         }
 
         public async Task DownloadArchives()

Wabbajack.Lib/MO2Installer.cs

@@ -298,7 +298,7 @@ namespace Wabbajack.Lib
             using (var output = File.Open(toFile, FileMode.Create))
             using (var input = File.OpenRead(gameFile))
             {
-                BSDiff.Apply(input, () => new MemoryStream(patchData), output);
+                Utils.ApplyPatch(input, () => new MemoryStream(patchData), output);
             }
         }
 

Wabbajack.Lib/zEditIntegration.cs

@@ -176,7 +176,7 @@ namespace Wabbajack.Lib
                     var patch_data = installer.LoadBytesFromPath(m.PatchID);
 
                     using (var fs = File.Open(Path.Combine(installer.OutputFolder, m.To), FileMode.Create)) 
-                        BSDiff.Apply(new MemoryStream(src_data), () => new MemoryStream(patch_data), fs);
+                        Utils.ApplyPatch(new MemoryStream(src_data), () => new MemoryStream(patch_data), fs);
                 });
         }
     }

Wabbajack.Test/TestUtils.cs

@@ -15,16 +15,16 @@ namespace Wabbajack.Test
 {
     public class TestUtils : IDisposable
     {
+        private static Random _rng = new Random();
         public TestUtils()
         {
-            RNG = new Random();
             ID = RandomName();
             WorkingDirectory = Path.Combine(Directory.GetCurrentDirectory(), "tmp_data");
         }
 
         public string WorkingDirectory { get;}
         public string ID { get; }
-        public Random RNG { get; }
+        public Random RNG => _rng;
 
         public Game Game { get; set; }
 
@@ -113,6 +113,15 @@ namespace Wabbajack.Test
             File.WriteAllBytes(full_path, bytes);
         }
 
+        public static byte[] RandomData(int? size = null, int maxSize = 1024)
+        {
+            if (size == null)
+                size = _rng.Next(1, maxSize);
+            var arr = new byte[(int) size];
+            _rng.NextBytes(arr);
+            return arr;
+        }
+
         public void Dispose()
         {
             var exts = new [] {".md", ".exe"};
@@ -250,5 +259,10 @@ namespace Wabbajack.Test
             GenerateRandomFileData(full_path, i);
             return full_path;
         }
+
+        public static object RandomeOne(params object[] opts)
+        {
+            return opts[_rng.Next(0, opts.Length)];
+        }
     }
 }

Wabbajack.Test/UtilsTests.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
@@ -11,6 +12,7 @@ namespace Wabbajack.Test
     [TestClass]
     public class UtilsTests
     {
+       
         [TestMethod]
         public void IsInPathTests()
         {
@@ -22,5 +24,47 @@ namespace Wabbajack.Test
             Assert.IsTrue("c:\\foo\\bar.exe".IsInPath("c:\\foo\\"));
             Assert.IsTrue("c:\\foo\\bar\\".IsInPath("c:\\foo\\"));
         }
+
+
+        [TestMethod]
+        [DataTestMethod]
+        [DynamicData(nameof(PatchData), DynamicDataSourceType.Method)]
+        public async Task DiffCreateAndApply(byte[] src, byte[] dest, DiffMethod method)
+        {
+            await using var ms = new MemoryStream();
+            switch (method)
+            {
+                case DiffMethod.Default:
+                    await Utils.CreatePatch(src, dest, ms);
+                    break;
+                case DiffMethod.BSDiff:
+                    BSDiff.Create(src, dest, ms);
+                    break;
+                case DiffMethod.OctoDiff:
+                    OctoDiff.Create(src, dest, ms);
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(method), method, null);
+            }
+
+            ms.Position = 0;
+            var patch = ms.ToArray();
+            await using var resultStream = new MemoryStream();
+            Utils.ApplyPatch(new MemoryStream(src), () => new MemoryStream(patch), resultStream);
+            CollectionAssert.AreEqual(dest, resultStream.ToArray());
+        }
+
+
+        public enum DiffMethod
+        {
+            Default,
+            BSDiff,
+            OctoDiff
+        }
+        public static IEnumerable<object[]> PatchData()
+        {
+            var maxSize = 1024 * 1024 * 8;
+            return Enumerable.Range(0, 10).Select(x => new[] {TestUtils.RandomData(maxSize:maxSize), TestUtils.RandomData(maxSize:maxSize), TestUtils.RandomeOne(DiffMethod.Default, DiffMethod.OctoDiff, DiffMethod.BSDiff)});
+        }
     }
 }