Switch from BSDiff to OctoDiff for patch generation

This commit is contained in:
Timothy Baldridge 2020-02-24 16:18:29 -07:00
parent 45fabc41db
commit 27025db484
10 changed files with 155 additions and 12 deletions

View File

@ -2,6 +2,7 @@
#### Version - 0.9.22.0
* Server side fixes for CORS support and FTP uploads
* Use OctoDiff instead of BSDiff for better performance during diff generation
#### Version - 0.9.21.0 - 2/23/2020
* Fix never ending hash issue

View File

@ -0,0 +1,46 @@
using System;
using System.IO;
using Octodiff.Core;
using Octodiff.Diagnostics;
namespace Wabbajack.Common
{
public class OctoDiff
{
private static ProgressReporter reporter = new ProgressReporter();
public static void Create(byte[] oldData, byte[] newData, Stream output)
{
using var signature = CreateSignature(oldData);
using var oldStream = new MemoryStream(oldData);
using var newStream = new MemoryStream(newData);
var db = new DeltaBuilder {ProgressReporter = reporter};
db.BuildDelta(newStream, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
}
private static Stream CreateSignature(byte[] oldData)
{
Utils.Status("Creating Patch Signature");
using var oldDataStream = new MemoryStream(oldData);
var sigStream = new MemoryStream();
var signatureBuilder = new SignatureBuilder();
signatureBuilder.Build(oldDataStream, new SignatureWriter(sigStream));
sigStream.Position = 0;
return sigStream;
}
private class ProgressReporter : IProgressReporter
{
public void ReportProgress(string operation, long currentPosition, long total)
{
Utils.Status(operation, new Percent(total, currentPosition));
}
}
public static void Apply(Stream input, Func<Stream> openPatchStream, Stream output)
{
using var deltaStream = openPatchStream();
var deltaApplier = new DeltaApplier();
deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, reporter), output);
}
}
}

View File

@ -24,6 +24,11 @@ namespace Wabbajack.Common
}
}
public Percent(long max, long current) : this((double)current / max)
{
}
public Percent(double d)
: this(d, check: true)
{

View File

@ -926,8 +926,19 @@ namespace Wabbajack.Common
{
if (File.Exists(cacheFile))
{
await using var f = File.OpenRead(cacheFile);
await f.CopyToAsync(output);
RETRY_OPEN:
try
{
await using var f = File.OpenRead(cacheFile);
await f.CopyToAsync(output);
}
catch (IOException)
{
// Race condition with patch caching
await Task.Delay(100);
goto RETRY_OPEN;
}
}
else
{
@ -936,7 +947,7 @@ namespace Wabbajack.Common
await using (var f = File.Open(tmpName, System.IO.FileMode.Create))
{
Status("Creating Patch");
BSDiff.Create(a, b, f);
OctoDiff.Create(a, b, f);
}
RETRY:
@ -973,6 +984,26 @@ namespace Wabbajack.Common
return false;
}
public static void ApplyPatch(Stream input, Func<Stream> openPatchStream, Stream output)
{
using var ps = openPatchStream();
using var br = new BinaryReader(ps);
var bytes = br.ReadBytes(8);
var str = Encoding.ASCII.GetString(bytes);
switch (str)
{
case "BSDIFF40":
BSDiff.Apply(input, openPatchStream, output);
return;
case "OCTODELT":
OctoDiff.Apply(input, openPatchStream, output);
return;
default:
throw new Exception($"No diff dispatch for: {str}");
}
}
/*
public static void Warning(string s)
{

View File

@ -36,6 +36,7 @@
<PackageReference Include="ini-parser-netstandard" Version="2.5.2" />
<PackageReference Include="Microsoft.Win32.Registry" Version="4.7.0" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="Octodiff" Version="1.2.1" />
<PackageReference Include="OMODFramework" Version="2.0.0" />
<PackageReference Include="ReactiveUI" Version="11.1.23" />
<PackageReference Include="SharpZipLib" Version="1.2.0" />

View File

@ -199,9 +199,10 @@ namespace Wabbajack.Lib
onFinish();
// Now patch all the files from this archive
foreach (var toPatch in grouping.OfType<PatchedFromArchive>())
using (var patchStream = new MemoryStream())
await grouping.OfType<PatchedFromArchive>()
.PMap(queue, async toPatch =>
{
await using var patchStream = new MemoryStream();
Status($"Patching {Path.GetFileName(toPatch.To)}");
// Read in the patch data
@ -214,16 +215,16 @@ namespace Wabbajack.Lib
File.Delete(toFile);
// Patch it
using (var outStream = File.Open(toFile, FileMode.Create))
await using (var outStream = File.Open(toFile, FileMode.Create))
{
BSDiff.Apply(oldData, () => new MemoryStream(patchData), outStream);
Utils.ApplyPatch(oldData, () => new MemoryStream(patchData), outStream);
}
Status($"Verifying Patch {Path.GetFileName(toPatch.To)}");
var resultSha = toFile.FileHash();
if (resultSha != toPatch.Hash)
throw new InvalidDataException($"Invalid Hash for {toPatch.To} after patching");
}
});
}
public async Task DownloadArchives()

View File

@ -298,7 +298,7 @@ namespace Wabbajack.Lib
using (var output = File.Open(toFile, FileMode.Create))
using (var input = File.OpenRead(gameFile))
{
BSDiff.Apply(input, () => new MemoryStream(patchData), output);
Utils.ApplyPatch(input, () => new MemoryStream(patchData), output);
}
}

View File

@ -176,7 +176,7 @@ namespace Wabbajack.Lib
var patch_data = installer.LoadBytesFromPath(m.PatchID);
using (var fs = File.Open(Path.Combine(installer.OutputFolder, m.To), FileMode.Create))
BSDiff.Apply(new MemoryStream(src_data), () => new MemoryStream(patch_data), fs);
Utils.ApplyPatch(new MemoryStream(src_data), () => new MemoryStream(patch_data), fs);
});
}
}

View File

@ -15,16 +15,16 @@ namespace Wabbajack.Test
{
public class TestUtils : IDisposable
{
private static Random _rng = new Random();
public TestUtils()
{
RNG = new Random();
ID = RandomName();
WorkingDirectory = Path.Combine(Directory.GetCurrentDirectory(), "tmp_data");
}
public string WorkingDirectory { get;}
public string ID { get; }
public Random RNG { get; }
public Random RNG => _rng;
public Game Game { get; set; }
@ -113,6 +113,15 @@ namespace Wabbajack.Test
File.WriteAllBytes(full_path, bytes);
}
public static byte[] RandomData(int? size = null, int maxSize = 1024)
{
if (size == null)
size = _rng.Next(1, maxSize);
var arr = new byte[(int) size];
_rng.NextBytes(arr);
return arr;
}
public void Dispose()
{
var exts = new [] {".md", ".exe"};
@ -250,5 +259,10 @@ namespace Wabbajack.Test
GenerateRandomFileData(full_path, i);
return full_path;
}
public static object RandomeOne(params object[] opts)
{
return opts[_rng.Next(0, opts.Length)];
}
}
}

View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
@ -11,6 +12,7 @@ namespace Wabbajack.Test
[TestClass]
public class UtilsTests
{
[TestMethod]
public void IsInPathTests()
{
@ -22,5 +24,47 @@ namespace Wabbajack.Test
Assert.IsTrue("c:\\foo\\bar.exe".IsInPath("c:\\foo\\"));
Assert.IsTrue("c:\\foo\\bar\\".IsInPath("c:\\foo\\"));
}
[TestMethod]
[DataTestMethod]
[DynamicData(nameof(PatchData), DynamicDataSourceType.Method)]
public async Task DiffCreateAndApply(byte[] src, byte[] dest, DiffMethod method)
{
await using var ms = new MemoryStream();
switch (method)
{
case DiffMethod.Default:
await Utils.CreatePatch(src, dest, ms);
break;
case DiffMethod.BSDiff:
BSDiff.Create(src, dest, ms);
break;
case DiffMethod.OctoDiff:
OctoDiff.Create(src, dest, ms);
break;
default:
throw new ArgumentOutOfRangeException(nameof(method), method, null);
}
ms.Position = 0;
var patch = ms.ToArray();
await using var resultStream = new MemoryStream();
Utils.ApplyPatch(new MemoryStream(src), () => new MemoryStream(patch), resultStream);
CollectionAssert.AreEqual(dest, resultStream.ToArray());
}
public enum DiffMethod
{
Default,
BSDiff,
OctoDiff
}
public static IEnumerable<object[]> PatchData()
{
var maxSize = 1024 * 1024 * 8;
return Enumerable.Range(0, 10).Select(x => new[] {TestUtils.RandomData(maxSize:maxSize), TestUtils.RandomData(maxSize:maxSize), TestUtils.RandomeOne(DiffMethod.Default, DiffMethod.OctoDiff, DiffMethod.BSDiff)});
}
}
}