mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
Switch from BSDiff to OctoDiff for patch generation
This commit is contained in:
parent
45fabc41db
commit
27025db484
@ -2,6 +2,7 @@
|
||||
|
||||
#### Version - 0.9.22.0
|
||||
* Server side fixes for CORS support and FTP uploads
|
||||
* Use OctoDiff instead of BSDiff for better performance during diff generation
|
||||
|
||||
#### Version - 0.9.21.0 - 2/23/2020
|
||||
* Fix never ending hash issue
|
||||
|
46
Wabbajack.Common/OctoDiff.cs
Normal file
46
Wabbajack.Common/OctoDiff.cs
Normal file
@ -0,0 +1,46 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using Octodiff.Core;
|
||||
using Octodiff.Diagnostics;
|
||||
|
||||
namespace Wabbajack.Common
|
||||
{
|
||||
public class OctoDiff
|
||||
{
|
||||
private static ProgressReporter reporter = new ProgressReporter();
|
||||
public static void Create(byte[] oldData, byte[] newData, Stream output)
|
||||
{
|
||||
using var signature = CreateSignature(oldData);
|
||||
using var oldStream = new MemoryStream(oldData);
|
||||
using var newStream = new MemoryStream(newData);
|
||||
var db = new DeltaBuilder {ProgressReporter = reporter};
|
||||
db.BuildDelta(newStream, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
|
||||
}
|
||||
|
||||
private static Stream CreateSignature(byte[] oldData)
|
||||
{
|
||||
Utils.Status("Creating Patch Signature");
|
||||
using var oldDataStream = new MemoryStream(oldData);
|
||||
var sigStream = new MemoryStream();
|
||||
var signatureBuilder = new SignatureBuilder();
|
||||
signatureBuilder.Build(oldDataStream, new SignatureWriter(sigStream));
|
||||
sigStream.Position = 0;
|
||||
return sigStream;
|
||||
}
|
||||
|
||||
private class ProgressReporter : IProgressReporter
|
||||
{
|
||||
public void ReportProgress(string operation, long currentPosition, long total)
|
||||
{
|
||||
Utils.Status(operation, new Percent(total, currentPosition));
|
||||
}
|
||||
}
|
||||
|
||||
public static void Apply(Stream input, Func<Stream> openPatchStream, Stream output)
|
||||
{
|
||||
using var deltaStream = openPatchStream();
|
||||
var deltaApplier = new DeltaApplier();
|
||||
deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, reporter), output);
|
||||
}
|
||||
}
|
||||
}
|
@ -24,6 +24,11 @@ namespace Wabbajack.Common
|
||||
}
|
||||
}
|
||||
|
||||
public Percent(long max, long current) : this((double)current / max)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public Percent(double d)
|
||||
: this(d, check: true)
|
||||
{
|
||||
|
@ -926,8 +926,19 @@ namespace Wabbajack.Common
|
||||
{
|
||||
if (File.Exists(cacheFile))
|
||||
{
|
||||
await using var f = File.OpenRead(cacheFile);
|
||||
await f.CopyToAsync(output);
|
||||
RETRY_OPEN:
|
||||
try
|
||||
{
|
||||
await using var f = File.OpenRead(cacheFile);
|
||||
await f.CopyToAsync(output);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Race condition with patch caching
|
||||
await Task.Delay(100);
|
||||
goto RETRY_OPEN;
|
||||
}
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -936,7 +947,7 @@ namespace Wabbajack.Common
|
||||
await using (var f = File.Open(tmpName, System.IO.FileMode.Create))
|
||||
{
|
||||
Status("Creating Patch");
|
||||
BSDiff.Create(a, b, f);
|
||||
OctoDiff.Create(a, b, f);
|
||||
}
|
||||
|
||||
RETRY:
|
||||
@ -973,6 +984,26 @@ namespace Wabbajack.Common
|
||||
return false;
|
||||
}
|
||||
|
||||
public static void ApplyPatch(Stream input, Func<Stream> openPatchStream, Stream output)
|
||||
{
|
||||
using var ps = openPatchStream();
|
||||
using var br = new BinaryReader(ps);
|
||||
var bytes = br.ReadBytes(8);
|
||||
var str = Encoding.ASCII.GetString(bytes);
|
||||
switch (str)
|
||||
{
|
||||
case "BSDIFF40":
|
||||
BSDiff.Apply(input, openPatchStream, output);
|
||||
return;
|
||||
case "OCTODELT":
|
||||
OctoDiff.Apply(input, openPatchStream, output);
|
||||
return;
|
||||
default:
|
||||
throw new Exception($"No diff dispatch for: {str}");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
public static void Warning(string s)
|
||||
{
|
||||
|
@ -36,6 +36,7 @@
|
||||
<PackageReference Include="ini-parser-netstandard" Version="2.5.2" />
|
||||
<PackageReference Include="Microsoft.Win32.Registry" Version="4.7.0" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
|
||||
<PackageReference Include="Octodiff" Version="1.2.1" />
|
||||
<PackageReference Include="OMODFramework" Version="2.0.0" />
|
||||
<PackageReference Include="ReactiveUI" Version="11.1.23" />
|
||||
<PackageReference Include="SharpZipLib" Version="1.2.0" />
|
||||
|
@ -199,9 +199,10 @@ namespace Wabbajack.Lib
|
||||
onFinish();
|
||||
|
||||
// Now patch all the files from this archive
|
||||
foreach (var toPatch in grouping.OfType<PatchedFromArchive>())
|
||||
using (var patchStream = new MemoryStream())
|
||||
await grouping.OfType<PatchedFromArchive>()
|
||||
.PMap(queue, async toPatch =>
|
||||
{
|
||||
await using var patchStream = new MemoryStream();
|
||||
Status($"Patching {Path.GetFileName(toPatch.To)}");
|
||||
// Read in the patch data
|
||||
|
||||
@ -214,16 +215,16 @@ namespace Wabbajack.Lib
|
||||
File.Delete(toFile);
|
||||
|
||||
// Patch it
|
||||
using (var outStream = File.Open(toFile, FileMode.Create))
|
||||
await using (var outStream = File.Open(toFile, FileMode.Create))
|
||||
{
|
||||
BSDiff.Apply(oldData, () => new MemoryStream(patchData), outStream);
|
||||
Utils.ApplyPatch(oldData, () => new MemoryStream(patchData), outStream);
|
||||
}
|
||||
|
||||
Status($"Verifying Patch {Path.GetFileName(toPatch.To)}");
|
||||
var resultSha = toFile.FileHash();
|
||||
if (resultSha != toPatch.Hash)
|
||||
throw new InvalidDataException($"Invalid Hash for {toPatch.To} after patching");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public async Task DownloadArchives()
|
||||
|
@ -298,7 +298,7 @@ namespace Wabbajack.Lib
|
||||
using (var output = File.Open(toFile, FileMode.Create))
|
||||
using (var input = File.OpenRead(gameFile))
|
||||
{
|
||||
BSDiff.Apply(input, () => new MemoryStream(patchData), output);
|
||||
Utils.ApplyPatch(input, () => new MemoryStream(patchData), output);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -176,7 +176,7 @@ namespace Wabbajack.Lib
|
||||
var patch_data = installer.LoadBytesFromPath(m.PatchID);
|
||||
|
||||
using (var fs = File.Open(Path.Combine(installer.OutputFolder, m.To), FileMode.Create))
|
||||
BSDiff.Apply(new MemoryStream(src_data), () => new MemoryStream(patch_data), fs);
|
||||
Utils.ApplyPatch(new MemoryStream(src_data), () => new MemoryStream(patch_data), fs);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
@ -15,16 +15,16 @@ namespace Wabbajack.Test
|
||||
{
|
||||
public class TestUtils : IDisposable
|
||||
{
|
||||
private static Random _rng = new Random();
|
||||
public TestUtils()
|
||||
{
|
||||
RNG = new Random();
|
||||
ID = RandomName();
|
||||
WorkingDirectory = Path.Combine(Directory.GetCurrentDirectory(), "tmp_data");
|
||||
}
|
||||
|
||||
public string WorkingDirectory { get;}
|
||||
public string ID { get; }
|
||||
public Random RNG { get; }
|
||||
public Random RNG => _rng;
|
||||
|
||||
public Game Game { get; set; }
|
||||
|
||||
@ -113,6 +113,15 @@ namespace Wabbajack.Test
|
||||
File.WriteAllBytes(full_path, bytes);
|
||||
}
|
||||
|
||||
public static byte[] RandomData(int? size = null, int maxSize = 1024)
|
||||
{
|
||||
if (size == null)
|
||||
size = _rng.Next(1, maxSize);
|
||||
var arr = new byte[(int) size];
|
||||
_rng.NextBytes(arr);
|
||||
return arr;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
var exts = new [] {".md", ".exe"};
|
||||
@ -250,5 +259,10 @@ namespace Wabbajack.Test
|
||||
GenerateRandomFileData(full_path, i);
|
||||
return full_path;
|
||||
}
|
||||
|
||||
public static object RandomeOne(params object[] opts)
|
||||
{
|
||||
return opts[_rng.Next(0, opts.Length)];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
@ -11,6 +12,7 @@ namespace Wabbajack.Test
|
||||
[TestClass]
|
||||
public class UtilsTests
|
||||
{
|
||||
|
||||
[TestMethod]
|
||||
public void IsInPathTests()
|
||||
{
|
||||
@ -22,5 +24,47 @@ namespace Wabbajack.Test
|
||||
Assert.IsTrue("c:\\foo\\bar.exe".IsInPath("c:\\foo\\"));
|
||||
Assert.IsTrue("c:\\foo\\bar\\".IsInPath("c:\\foo\\"));
|
||||
}
|
||||
|
||||
|
||||
[TestMethod]
|
||||
[DataTestMethod]
|
||||
[DynamicData(nameof(PatchData), DynamicDataSourceType.Method)]
|
||||
public async Task DiffCreateAndApply(byte[] src, byte[] dest, DiffMethod method)
|
||||
{
|
||||
await using var ms = new MemoryStream();
|
||||
switch (method)
|
||||
{
|
||||
case DiffMethod.Default:
|
||||
await Utils.CreatePatch(src, dest, ms);
|
||||
break;
|
||||
case DiffMethod.BSDiff:
|
||||
BSDiff.Create(src, dest, ms);
|
||||
break;
|
||||
case DiffMethod.OctoDiff:
|
||||
OctoDiff.Create(src, dest, ms);
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(nameof(method), method, null);
|
||||
}
|
||||
|
||||
ms.Position = 0;
|
||||
var patch = ms.ToArray();
|
||||
await using var resultStream = new MemoryStream();
|
||||
Utils.ApplyPatch(new MemoryStream(src), () => new MemoryStream(patch), resultStream);
|
||||
CollectionAssert.AreEqual(dest, resultStream.ToArray());
|
||||
}
|
||||
|
||||
|
||||
public enum DiffMethod
|
||||
{
|
||||
Default,
|
||||
BSDiff,
|
||||
OctoDiff
|
||||
}
|
||||
public static IEnumerable<object[]> PatchData()
|
||||
{
|
||||
var maxSize = 1024 * 1024 * 8;
|
||||
return Enumerable.Range(0, 10).Select(x => new[] {TestUtils.RandomData(maxSize:maxSize), TestUtils.RandomData(maxSize:maxSize), TestUtils.RandomeOne(DiffMethod.Default, DiffMethod.OctoDiff, DiffMethod.BSDiff)});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user