Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

use xxHash64 hasing

Browse Source
This commit is contained in:
Timothy Baldridge 2019-10-30 21:40:33 -06:00
parent 6c79d0a9d2
commit 610fea9d7f
8 changed files with 54 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
VirtualFileSystem/VirtualFileSystem.cs
View File

@ -5,6 +5,7 @@ using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using Ceras;
using Compression.BSA;
@ -20,6 +21,9 @@ namespace VFS
{
public class VirtualFileSystem
{
public const ulong FileVersion = 0x01;
public const string Magic = "WABBAJACK VFS FILE";
internal static string _stagedRoot;
public static VirtualFileSystem VFS;
private bool _disableDiskCache;
@ -121,6 +125,14 @@ namespace VFS
using (var fs = File.OpenRead("vfs_cache.bin"))
using (var br = new BinaryReader(fs))
{
var magic = Encoding.ASCII.GetString(br.ReadBytes(Magic.Length));
if (magic != Magic || br.ReadUInt64() != FileVersion)
{
fs.Close();
File.Delete("vfs_cache.bin");
return;
}
while (true)
{
var fr = VirtualFile.Read(br);
@ -159,6 +171,9 @@ namespace VFS
using (var fs = File.OpenWrite("vfs_cache.bin_new"))
using (var bw = new BinaryWriter(fs))
{
bw.Write(Encoding.ASCII.GetBytes(Magic));
bw.Write(FileVersion);
Utils.Log($"Syncing VFS to Disk: {_files.Count} entries");
foreach (var f in _files.Values) f.Write(bw);
}
@ -672,7 +687,7 @@ namespace VFS
var fio = new FileInfo(StagedPath);
Size = fio.Length;
Hash = StagedPath.FileSHA256();
Hash = StagedPath.FileHash();
LastModified = fio.LastWriteTime.ToMilliseconds();
}

15
Wabbajack.Common/Utils.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Data.HashFunction.xxHash;
using System.Diagnostics;
using System.IO;
using System.Linq;
@ -103,6 +104,20 @@ namespace Wabbajack.Common
return sha.Hash.ToBase64();
}
public static string FileHash(this string file)
{
var hash = new xxHashConfig();
hash.HashSizeInBits = 64;
hash.Seed = 0x42;
using (var fs = File.OpenRead(file))
{
var config = new xxHashConfig();
config.HashSizeInBits = 64;
var value = xxHashFactory.Instance.Create(config).ComputeHash(fs);
return value.AsBase64String();
}
}
public static void CopyToWithStatus(this Stream istream, long maxSize, Stream ostream, string status)
{
var buffer = new byte[1024 * 64];

9
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -86,6 +86,15 @@
</Reference>
<Reference Include="System.Configuration" />
<Reference Include="System.Core" />
<Reference Include="System.Data.HashFunction.Core, Version=2.0.0.0, Culture=neutral, PublicKeyToken=80c9288e394c1322, processorArchitecture=MSIL">
<HintPath>..\packages\System.Data.HashFunction.Core.2.0.0\lib\net45\System.Data.HashFunction.Core.dll</HintPath>
</Reference>
<Reference Include="System.Data.HashFunction.Interfaces, Version=2.0.0.0, Culture=neutral, PublicKeyToken=80c9288e394c1322, processorArchitecture=MSIL">
<HintPath>..\packages\System.Data.HashFunction.Interfaces.2.0.0\lib\net45\System.Data.HashFunction.Interfaces.dll</HintPath>
</Reference>
<Reference Include="System.Data.HashFunction.xxHash, Version=2.0.0.0, Culture=neutral, PublicKeyToken=80c9288e394c1322, processorArchitecture=MSIL">
<HintPath>..\packages\System.Data.HashFunction.xxHash.2.0.0\lib\net45\System.Data.HashFunction.xxHash.dll</HintPath>
</Reference>
<Reference Include="System.IO.Compression" />
<Reference Include="System.Numerics" />
<Reference Include="System.Runtime.CompilerServices.Unsafe, Version=4.0.4.1, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">

3
Wabbajack.Common/packages.config
View File

@ -11,6 +11,9 @@
<package id="SevenZip" version="19.0.0" targetFramework="net472" />
<package id="SharpZipLib" version="1.2.0" targetFramework="net472" />
<package id="System.Buffers" version="4.5.0" targetFramework="net472" />
<package id="System.Data.HashFunction.Core" version="2.0.0" targetFramework="net472" />
<package id="System.Data.HashFunction.Interfaces" version="2.0.0" targetFramework="net472" />
<package id="System.Data.HashFunction.xxHash" version="2.0.0" targetFramework="net472" />
<package id="System.Runtime.CompilerServices.Unsafe" version="4.5.2" targetFramework="net472" />
<package id="System.Runtime.Numerics" version="4.3.0" targetFramework="net472" />
<package id="YamlDotNet" version="8.0.0" targetFramework="net472" />

6
Wabbajack.Lib/Installer.cs
View File

@ -297,7 +297,7 @@ namespace Wabbajack.Lib
Info($"Generating cleaned ESM for {filename}");
if (!File.Exists(game_file)) throw new InvalidDataException($"Missing {filename} at {game_file}");
Status($"Hashing game version of {filename}");
var sha = game_file.FileSHA256();
var sha = game_file.FileHash();
if (sha != directive.SourceESMHash)
throw new InvalidDataException(
$"Cannot patch {filename} from the game folder hashes don't match have you already cleaned the file?");
@ -410,7 +410,7 @@ namespace Wabbajack.Lib
}
Status($"Verifying Patch {Path.GetFileName(to_patch.To)}");
var result_sha = to_file.FileSHA256();
var result_sha = to_file.FileHash();
if (result_sha != to_patch.Hash)
throw new InvalidDataException($"Invalid Hash for {to_patch.To} after patching");
}
@ -480,7 +480,7 @@ namespace Wabbajack.Lib
return File.ReadAllText(cache);
Status($"Hashing {Path.GetFileName(e)}");
File.WriteAllText(cache, e.FileSHA256());
File.WriteAllText(cache, e.FileHash());
return HashArchive(e);
}
}

12
Wabbajack.Test/DownloaderTests.cs
View File

@ -47,7 +47,7 @@ namespace Wabbajack.Test
converted.Download(new Archive {Name = "MEGA Test.txt"}, filename);
Assert.AreEqual("Lb1iTsz3iyZeHGs3e94TVmOhf22sqtHLhqkCdXbjiyc=", Utils.FileSHA256(filename));
Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
}
@ -77,7 +77,7 @@ namespace Wabbajack.Test
converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
Assert.AreEqual("Lb1iTsz3iyZeHGs3e94TVmOhf22sqtHLhqkCdXbjiyc=", Utils.FileSHA256(filename));
Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
}
@ -107,7 +107,7 @@ namespace Wabbajack.Test
converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
Assert.AreEqual("Lb1iTsz3iyZeHGs3e94TVmOhf22sqtHLhqkCdXbjiyc=", Utils.FileSHA256(filename));
Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
}
@ -136,7 +136,7 @@ namespace Wabbajack.Test
converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
Assert.AreEqual("Lb1iTsz3iyZeHGs3e94TVmOhf22sqtHLhqkCdXbjiyc=", Utils.FileSHA256(filename));
Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
}
@ -192,7 +192,7 @@ namespace Wabbajack.Test
converted.Download(new Archive { Name = "SkyUI.7z" }, filename);
Assert.AreEqual(filename.FileSHA256(), "U3Xg6RBR9XrUY9/jQSu6WKu5dfhHmpaN2dTl0ylDFmI=");
Assert.AreEqual(filename.FileHash(), "dF2yafV2Oks=");
}
[TestMethod]
@ -219,7 +219,7 @@ namespace Wabbajack.Test
converted.Download(new Archive { Name = "moddbtest.7z" }, filename);
Assert.AreEqual("lUvpEjqxfyidBONSHcDy6EnZIPpAD2K4rkJ5ejCXc2k=", filename.FileSHA256());
Assert.AreEqual("2lZt+1h6wxM=", filename.FileHash());
}
}

2
Wabbajack.Test/TestUtils.cs
View File

@ -206,7 +206,7 @@ namespace Wabbajack.Test
if (!skip_extensions.Contains(Path.GetExtension(src_file)))
{
Assert.AreEqual(fi_src.Length, fi_dest.Length, $"Differing sizes {rel_file}");
Assert.AreEqual(src_file.FileSHA256(), dest_file.FileSHA256(), $"Differing content hash {rel_file}");
Assert.AreEqual(src_file.FileHash(), dest_file.FileHash(), $"Differing content hash {rel_file}");
}
}

2
Wabbajack/UI/DownloadWindow.xaml.cs
View File

@ -65,7 +65,7 @@ namespace Wabbajack.UI
var state = DownloadDispatcher.ResolveArchive(_url);
state.Download(new Archive {Name = _downloadName}, _destination);
_destination.FileSHA256();
_destination.FileHash();
_parent.Result = DownloadWindow.WindowResult.Completed;