Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

WIP, converted Hashes to a Hash struct

Browse Source
This commit is contained in:
Timothy Baldridge 2020-03-22 09:50:53 -06:00
parent e4ecaa882c
commit 3b895f4dbb
42 changed files with 314 additions and 649 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
Wabbajack.BuildServer/Controllers/IndexedFiles.cs
View File

@ -38,7 +38,7 @@ namespace Wabbajack.BuildServer.Controllers
[Route("{xxHashAsBase64}/meta.ini")]
public async Task<IActionResult> GetFileMeta(string xxHashAsBase64)
{
var id = xxHashAsBase64.FromHex().ToBase64();
var id = Hash.FromHex(xxHashAsBase64);
var state = await Db.DownloadStates.AsQueryable()
.Where(d => d.Hash == id && d.IsValid)
.OrderByDescending(d => d.LastValidationTime)
@ -51,48 +51,6 @@ namespace Wabbajack.BuildServer.Controllers
return Ok(string.Join("\r\n", state.FirstOrDefault().State.GetMetaIni()));
}
[Authorize]
[HttpDelete]
[Route("/indexed_files/nexus/{Game}/mod/{ModId}")]
public async Task<IActionResult> PurgeBySHA256(string Game, string ModId)
{
var files = await Db.DownloadStates.AsQueryable().Where(d => d.State is NexusDownloader.State &&
((NexusDownloader.State)d.State).GameName == Game &&
((NexusDownloader.State)d.State).ModID == ModId)
.ToListAsync();
async Task DeleteParentsOf(HashSet<string> acc, string hash)
{
var parents = await Db.IndexedFiles.AsQueryable().Where(f => f.Children.Any(c => c.Hash == hash))
.ToListAsync();
foreach (var parent in parents)
await DeleteThisAndAllChildren(acc, parent.Hash);
}
async Task DeleteThisAndAllChildren(HashSet<string> acc, string hash)
{
acc.Add(hash);
var children = await Db.IndexedFiles.AsQueryable().Where(f => f.Hash == hash).FirstOrDefaultAsync();
if (children == null) return;
foreach (var child in children.Children)
{
await DeleteThisAndAllChildren(acc, child.Hash);
}
}
var acc = new HashSet<string>();
foreach (var file in files)
await DeleteThisAndAllChildren(acc, file.Hash);
var acclst = acc.ToList();
await Db.DownloadStates.DeleteManyAsync(d => acc.Contains(d.Hash));
await Db.IndexedFiles.DeleteManyAsync(d => acc.Contains(d.Hash));
return Ok(acc.ToList());
}
[HttpPost]
[Route("notify")]
public async Task<IActionResult> Notify()

12
Wabbajack.BuildServer/Controllers/ModlistUpdater.cs
View File

@ -43,7 +43,7 @@ namespace Wabbajack.BuildServer.Controllers
{
var lists = await Db.ModListStatus.AsQueryable().ToListAsync();
var archives = lists.SelectMany(list => list.DetailedStatus.Archives)
.Select(a => a.Archive.Hash.FromBase64().ToHex())
.Select(a => a.Archive.Hash.ToHex())
.ToHashSet();
var toDelete = new List<string>();
@ -89,9 +89,9 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/alternative/{xxHash}")]
public async Task<IActionResult> GetAlternative(string xxHash)
{
var startingHash = xxHash.FromHex().ToBase64();
var startingHash = Hash.FromHex(xxHash);
Utils.Log($"Alternative requested for {startingHash}");
await Metric("requested_upgrade", startingHash);
await Metric("requested_upgrade", startingHash.ToString());
var state = await Db.DownloadStates.AsQueryable()
.Where(s => s.Hash == startingHash)
@ -110,7 +110,7 @@ namespace Wabbajack.BuildServer.Controllers
if (mod_files.SelectMany(f => f.Data.files)
.Any(f => f.category_name != null && f.file_id.ToString() == nexusState.FileID))
{
await Metric("not_required_upgrade", startingHash);
await Metric("not_required_upgrade", startingHash.ToString());
return BadRequest("Upgrade Not Required");
}
@ -122,7 +122,7 @@ namespace Wabbajack.BuildServer.Controllers
}
Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}");
if (newArchive.Hash == null)
if (newArchive.Hash == Hash.Empty)
{
Db.Jobs.InsertOne(new Job
{
@ -160,7 +160,7 @@ namespace Wabbajack.BuildServer.Controllers
return Ok(newArchive.ToJSON());
}
private async Task<Archive> FindAlternatives(NexusDownloader.State state, string srcHash)
private async Task<Archive> FindAlternatives(NexusDownloader.State state, Hash srcHash)
{
var origSize = AlphaFile.GetSize(_settings.PathForArchive(srcHash));
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());

2
Wabbajack.BuildServer/Controllers/UploadedFiles.cs
View File

@ -121,7 +121,7 @@ namespace Wabbajack.BuildServer.Controllers
[Route("upload_file/{Key}/finish/{xxHashAsHex}")]
public async Task<IActionResult> UploadFileFinish(string Key, string xxHashAsHex)
{
var expectedHash = xxHashAsHex.FromHex().ToBase64();
var expectedHash = Hash.FromHex(xxHashAsHex);
var user = User.FindFirstValue(ClaimTypes.Name);
if (!Key.All(a => HexChars.Contains(a)))
return BadRequest("NOT A VALID FILENAME");

6
Wabbajack.BuildServer/Extensions.cs
View File

@ -47,13 +47,13 @@ namespace Wabbajack.BuildServer
return authenticationBuilder.AddScheme<ApiKeyAuthenticationOptions, ApiKeyAuthenticationHandler>(ApiKeyAuthenticationOptions.DefaultScheme, options);
}
private static ConcurrentDictionary<string, string> PathForArchiveHash = new ConcurrentDictionary<string, string>();
public static string PathForArchive(this AppSettings settings, string hash)
private static readonly ConcurrentDictionary<Hash, string> PathForArchiveHash = new ConcurrentDictionary<Hash, string>();
public static string PathForArchive(this AppSettings settings, Hash hash)
{
if (PathForArchiveHash.TryGetValue(hash, out string result))
return result;
var hexHash = hash.FromBase64().ToHex();
var hexHash = hash.ToHex();
var ends = "_" + hexHash + "_";
var file = Directory.EnumerateFiles(settings.ArchiveDir, DirectoryEnumerationOptions.Files,

7
Wabbajack.BuildServer/Models/DownloadState.cs
View File

@ -1,9 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.BuildServer.Models
@ -12,7 +9,7 @@ namespace Wabbajack.BuildServer.Models
{
[BsonId]
public string Key { get; set; }
public string Hash { get; set; }
public Hash Hash { get; set; }
public AbstractDownloadState State { get; set; }

5
Wabbajack.BuildServer/Models/IndexedFile.cs
View File

@ -4,6 +4,7 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.Common;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.BuildServer.Models
@ -11,7 +12,7 @@ namespace Wabbajack.BuildServer.Models
public class IndexedFile
{
[BsonId]
public string Hash { get; set; }
public Hash Hash { get; set; }
public string SHA256 { get; set; }
public string SHA1 { get; set; }
public string MD5 { get; set; }
@ -25,6 +26,6 @@ namespace Wabbajack.BuildServer.Models
{
public string Name;
public string Extension;
public string Hash;
public Hash Hash;
}
}

2
Wabbajack.BuildServer/Models/Jobs/IndexJob.cs
View File

@ -56,7 +56,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
});
var to_path = Path.Combine(settings.ArchiveDir,
$"{Path.GetFileName(fileName)}_{archive.Hash.FromBase64().ToHex()}_{Path.GetExtension(fileName)}");
$"{Path.GetFileName(fileName)}_{archive.Hash.ToHex()}_{Path.GetExtension(fileName)}");
if (File.Exists(to_path))
File.Delete(downloadDest);
else

2
Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs
View File

@ -27,7 +27,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
using (var queue = new WorkQueue())
{
var whitelists = new ValidateModlist(queue);
var whitelists = new ValidateModlist();
await whitelists.LoadListsFromGithub();
foreach (var list in modlists)

8
Wabbajack.BuildServer/Models/PatchArchive.cs
View File

@ -18,7 +18,7 @@ namespace Wabbajack.BuildServer.Models
public class PatchArchive : AJobPayload
{
public override string Description => "Create a archive update patch";
public string Src { get; set; }
public Hash Src { get; set; }
public string DestPK { get; set; }
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
{
@ -56,7 +56,7 @@ namespace Wabbajack.BuildServer.Models
await client.ConnectAsync();
try
{
await client.UploadAsync(fs, $"updates/{Src.FromBase64().ToHex()}_{destHash.FromBase64().ToHex()}", progress: new UploadToCDN.Progress(cdnPath));
await client.UploadAsync(fs, $"updates/{Src.ToHex()}_{destHash.ToHex()}", progress: new UploadToCDN.Progress(cdnPath));
}
catch (Exception ex)
{
@ -72,9 +72,9 @@ namespace Wabbajack.BuildServer.Models
}
public static string CdnPath(string srcHash, string destHash)
public static string CdnPath(Hash srcHash, Hash destHash)
{
return $"updates/{srcHash.FromBase64().ToHex()}_{destHash.FromBase64().ToHex()}";
return $"updates/{srcHash.ToHex()}_{destHash.ToHex()}";
}
}
}

14
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -49,10 +49,9 @@ namespace Wabbajack.BuildServer.Model.Models
private static void IngestFile(VirtualFile root, ICollection<IndexedFile> files, ICollection<ArchiveContent> contents)
{
var hash = BitConverter.ToInt64(root.Hash.FromBase64());
files.Add(new IndexedFile
{
Hash = hash,
Hash = (long)root.Hash,
Sha256 = root.ExtendedHashes.SHA256.FromHex(),
Sha1 = root.ExtendedHashes.SHA1.FromHex(),
Md5 = root.ExtendedHashes.MD5.FromHex(),
@ -66,22 +65,21 @@ namespace Wabbajack.BuildServer.Model.Models
{
IngestFile(child, files, contents);
var child_hash = BitConverter.ToInt64(child.Hash.FromBase64());
contents.Add(new ArchiveContent
{
Parent = hash,
Child = child_hash,
Parent = (long)root.Hash,
Child = (long)child.Hash,
Path = child.Name
});
}
}
public async Task<bool> HaveIndexdFile(string hash)
public async Task<bool> HaveIndexdFile(Hash hash)
{
await using var conn = await Open();
var row = await conn.QueryAsync(@"SELECT * FROM IndexedFile WHERE Hash = @Hash",
new {Hash = BitConverter.ToInt64(hash.FromBase64())});
new {Hash = (long)hash});
return row.Any();
}
@ -123,7 +121,7 @@ namespace Wabbajack.BuildServer.Model.Models
return children.Select(f => new IndexedVirtualFile
{
Name = f.Path,
Hash = BitConverter.GetBytes(f.Hash).ToBase64(),
Hash = Hash.FromLong(f.Hash),
Size = f.Size,
Children = Build(f.Hash)
}).ToList();

2
Wabbajack.BuildServer/Models/UploadedFile.cs
View File

@ -13,7 +13,7 @@ namespace Wabbajack.BuildServer.Models
public string Id { get; set; }
public string Name { get; set; }
public long Size { get; set; }
public string Hash { get; set; }
public Hash Hash { get; set; }
public string Uploader { get; set; }
public DateTime UploadDate { get; set; } = DateTime.UtcNow;

2
Wabbajack.CLI/Verbs/Validate.cs
View File

@ -57,7 +57,7 @@ namespace Wabbajack.CLI.Verbs
try
{
ValidateModlist.RunValidation(queue, modlist).RunSynchronously();
ValidateModlist.RunValidation(modlist).RunSynchronously();
}
catch (Exception e)
{

198
Wabbajack.Common/Hash.cs Normal file
View File

@ -0,0 +1,198 @@
using System;
using System.Collections.Generic;
using System.Data.HashFunction.xxHash;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Common
{
/// <summary>
/// Struct representing a xxHash64 value. It's a struct with a ulong in it, but wrapped so we don't confuse
/// it with other longs in the system.
/// </summary>
public struct Hash
{
private readonly ulong _code;
public Hash(ulong code = 0)
{
_code = code;
}
public override string ToString()
{
return BitConverter.GetBytes(_code).ToBase64();
}
public override bool Equals(object? obj)
{
if (obj is Hash h)
return h._code == _code;
return false;
}
public override int GetHashCode()
{
return (int)(_code >> 32) ^ (int)_code;
}
public static bool operator ==(Hash a, Hash b)
{
return a._code == b._code;
}
public static bool operator !=(Hash a, Hash b)
{
return !(a == b);
}
public static explicit operator ulong(Hash a)
{
return a._code;
}
public static explicit operator long(Hash a)
{
return BitConverter.ToInt64(BitConverter.GetBytes(a._code));
}
public string ToHex()
{
return BitConverter.GetBytes(_code).ToHex();
}
public string ToBase64()
{
return BitConverter.GetBytes(_code).ToBase64();
}
public static Hash FromBase64(string hash)
{
return new Hash(BitConverter.ToUInt64(hash.FromBase64()));
}
public static Hash Empty = new Hash();
public static Hash FromLong(in long argHash)
{
return new Hash(BitConverter.ToUInt64(BitConverter.GetBytes(argHash)));
}
public static Hash FromHex(string xxHashAsHex)
{
return new Hash(BitConverter.ToUInt64(xxHashAsHex.FromHex()));
}
}
public static partial class Utils
{
public static Hash ReadHash(this BinaryReader br)
{
return new Hash(br.ReadUInt64());
}
public static void Write(this BinaryWriter bw, Hash hash)
{
bw.Write((ulong)hash);
}
public static string StringSha256Hex(this string s)
{
var sha = new SHA256Managed();
using (var o = new CryptoStream(Stream.Null, sha, CryptoStreamMode.Write))
{
using var i = new MemoryStream(Encoding.UTF8.GetBytes(s));
i.CopyTo(o);
}
return sha.Hash.ToHex();
}
public static Hash FileHash(this string file, bool nullOnIoError = false)
{
try
{
using var fs = File.OpenRead(file);
var config = new xxHashConfig {HashSizeInBits = 64};
using var f = new StatusFileStream(fs, $"Hashing {Path.GetFileName(file)}");
return new Hash(BitConverter.ToUInt64(xxHashFactory.Instance.Create(config).ComputeHash(f).Hash));
}
catch (IOException)
{
if (nullOnIoError) return Hash.Empty;
throw;
}
}
public static Hash FileHashCached(this string file, bool nullOnIoError = false)
{
if (TryGetHashCache(file, out var foundHash)) return foundHash;
var hash = file.FileHash(nullOnIoError);
if (hash != Hash.Empty)
WriteHashCache(file, hash);
return hash;
}
public static bool TryGetHashCache(string file, out Hash hash)
{
var hashFile = file + Consts.HashFileExtension;
hash = Hash.Empty;
if (!File.Exists(hashFile)) return false;
if (File.GetSize(hashFile) != 20) return false;
using var fs = File.OpenRead(hashFile);
using var br = new BinaryReader(fs);
var version = br.ReadUInt32();
if (version != HashCacheVersion) return false;
var lastModified = br.ReadUInt64();
if (lastModified != File.GetLastWriteTimeUtc(file).AsUnixTime()) return false;
hash = new Hash(br.ReadUInt64());
return true;
}
private const uint HashCacheVersion = 0x01;
private static void WriteHashCache(string file, Hash hash)
{
using var fs = File.Create(file + Consts.HashFileExtension);
using var bw = new BinaryWriter(fs);
bw.Write(HashCacheVersion);
var lastModified = File.GetLastWriteTimeUtc(file).AsUnixTime();
bw.Write(lastModified);
bw.Write((ulong)hash);
}
public static async Task<Hash> FileHashCachedAsync(this string file, bool nullOnIOError = false)
{
if (TryGetHashCache(file, out var foundHash)) return foundHash;
var hash = await file.FileHashAsync(nullOnIOError);
if (hash != Hash.Empty)
WriteHashCache(file, hash);
return hash;
}
public static async Task<Hash> FileHashAsync(this string file, bool nullOnIOError = false)
{
try
{
await using var fs = File.OpenRead(file);
var config = new xxHashConfig {HashSizeInBits = 64};
var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(fs);
return new Hash(BitConverter.ToUInt64(value.Hash));
}
catch (IOException)
{
if (nullOnIOError) return Hash.Empty;
throw;
}
}
}
}

9
Wabbajack.Common/Serialization/Deserializer.cs
View File

@ -1,9 +0,0 @@
using System.IO;
namespace Wabbajack.Common.Serialization
{
public class Deserializer
{
public BinaryReader Reader { get; }
}
}

9
Wabbajack.Common/Serialization/IHandler.cs
View File

@ -1,9 +0,0 @@
namespace Wabbajack.Common.Serialization
{
public interface IHandler
{
public void Write<T>(Serializer serializer, T data);
public T Read<T>(Deserializer deserialiser);
}
}

34
Wabbajack.Common/Serialization/PrimitiveHandlers.cs
View File

@ -1,34 +0,0 @@
using System;
namespace Wabbajack.Common.Serialization {
public class UInt32Handler : IHandler {
public void Write<T>(Serializer serializer, UInt32 data)
{
serializer.Writer.Write(data);
}
public T Read<T>(Deserializer deserializer)
{
return deserializer.Reader.ReadUInt32();
}
}
public class Int32Handler : IHandler {
public void Write<T>(Serializer serializer, Int32 data)
{
serializer.Writer.Write(data);
}
public T Read<T>(Deserializer deserializer)
{
return deserializer.Reader.ReadInt32();
}
}
}

34
Wabbajack.Common/Serialization/PrimitiveHandlers.tt
View File

@ -1,34 +0,0 @@
<#@ template language="C#" #>
<#@ import namespace="System.Text" #>
<#@ import namespace="System.Collections.Generic" #>
using System;
namespace Wabbajack.Common.Serialization {
<#
var types = new List<(Type, string)>()
{
(typeof(UInt32), "UInt32"),
(typeof(Int32), "Int32")
};
foreach (var type in types)
{
#>
public class <#=type.Item2#>Handler : IHandler {
public void Write<T>(Serializer serializer, <#=type.Item2#> data)
{
serializer.Writer.Write(data);
}
public T Read<T>(Deserializer deserializer)
{
return deserializer.Reader.Read<#=type.Item2#>();
}
}
<# } #>
}

53
Wabbajack.Common/Serialization/Serializer.cs
View File

@ -1,53 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
namespace Wabbajack.Common.Serialization
{
public class Serializer
{
private Dictionary<string, int> _internedStrings = new Dictionary<string, int>();
private Dictionary<Type, HandlerRecord> _handlers = new Dictionary<Type, HandlerRecord>();
public BinaryWriter Writer { get; }
public Serializer(BinaryWriter bw)
{
Writer = bw;
}
public void RegisterWriteHandler<T>(string name, IHandler handler)
{
_handlers.Add(typeof(T), new HandlerRecord
{
TypeName = name,
TypeId = Intern(name),
Handler = handler
});
}
public async Task Write<T>(BinaryWriter bw, T data)
{
var handler = _handlers[typeof(T)];
handler.Handler.Write<T>(this, data);
}
private int Intern(string s)
{
if (_internedStrings.TryGetValue(s, out var idx))
return idx;
idx = _internedStrings.Count;
_internedStrings[s] = idx;
return idx;
}
}
class HandlerRecord
{
public string TypeName;
public int TypeId;
public IHandler Handler;
}
}

139
Wabbajack.Common/Utils.cs
View File

@ -32,7 +32,7 @@ using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Common
{
public static class Utils
public static partial class Utils
{
public static bool IsMO2Running(string mo2Path)
{
@ -206,135 +206,6 @@ namespace Wabbajack.Common
}
}
/// <summary>
/// MurMur3 hashes the file pointed to by this string
/// </summary>
/// <param name="file"></param>
/// <returns></returns>
public static string FileSHA256(this string file)
{
var sha = new SHA256Managed();
using (var o = new CryptoStream(Stream.Null, sha, CryptoStreamMode.Write))
{
using (var i = File.OpenRead(file))
{
i.CopyToWithStatus(new FileInfo(file).Length, o, $"Hashing {Path.GetFileName(file)}");
}
}
return sha.Hash.ToBase64();
}
public static string StringSHA256Hex(this string s)
{
var sha = new SHA256Managed();
using (var o = new CryptoStream(Stream.Null, sha, CryptoStreamMode.Write))
{
using var i = new MemoryStream(Encoding.UTF8.GetBytes(s));
i.CopyTo(o);
}
return sha.Hash.ToHex();
}
public static string FileHash(this string file, bool nullOnIOError = false)
{
try
{
var hash = new xxHashConfig();
hash.HashSizeInBits = 64;
hash.Seed = 0x42;
using (var fs = File.OpenRead(file))
{
var config = new xxHashConfig();
config.HashSizeInBits = 64;
using (var f = new StatusFileStream(fs, $"Hashing {Path.GetFileName(file)}"))
{
var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
return value.AsBase64String();
}
}
}
catch (IOException ex)
{
if (nullOnIOError) return null;
throw ex;
}
}
public static string FileHashCached(this string file, bool nullOnIOError = false)
{
if (TryGetHashCache(file, out var foundHash)) return foundHash;
var hash = file.FileHash(nullOnIOError);
if (hash != null)
WriteHashCache(file, hash);
return hash;
}
public static bool TryGetHashCache(string file, out string hash)
{
var hashFile = file + Consts.HashFileExtension;
hash = null;
if (!File.Exists(hashFile)) return false;
if (File.GetSize(hashFile) != 20) return false;
using var fs = File.OpenRead(hashFile);
using var br = new BinaryReader(fs);
var version = br.ReadUInt32();
if (version != HashCacheVersion) return false;
var lastModified = br.ReadUInt64();
if (lastModified != File.GetLastWriteTimeUtc(file).AsUnixTime()) return false;
hash = BitConverter.GetBytes(br.ReadUInt64()).ToBase64();
return true;
}
private const uint HashCacheVersion = 0x01;
private static void WriteHashCache(string file, string hash)
{
using var fs = File.Create(file + Consts.HashFileExtension);
using var bw = new BinaryWriter(fs);
bw.Write(HashCacheVersion);
var lastModified = File.GetLastWriteTimeUtc(file).AsUnixTime();
bw.Write(lastModified);
bw.Write(BitConverter.ToUInt64(hash.FromBase64()));
}
public static async Task<string> FileHashCachedAsync(this string file, bool nullOnIOError = false)
{
if (TryGetHashCache(file, out var foundHash)) return foundHash;
var hash = await file.FileHashAsync(nullOnIOError);
if (hash != null)
WriteHashCache(file, hash);
return hash;
}
public static async Task<string> FileHashAsync(this string file, bool nullOnIOError = false)
{
try
{
var hash = new xxHashConfig();
hash.HashSizeInBits = 64;
hash.Seed = 0x42;
using (var fs = File.OpenRead(file))
{
var config = new xxHashConfig();
config.HashSizeInBits = 64;
var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(fs);
return value.AsBase64String();
}
}
catch (IOException ex)
{
if (nullOnIOError) return null;
throw ex;
}
}
public static void CopyToWithStatus(this Stream istream, long maxSize, Stream ostream, string status)
{
var buffer = new byte[1024 * 64];
@ -980,7 +851,7 @@ namespace Wabbajack.Common
}
}
public static async Task CreatePatch(FileStream srcStream, string srcHash, FileStream destStream, string destHash,
public static async Task CreatePatch(FileStream srcStream, Hash srcHash, FileStream destStream, Hash destHash,
FileStream patchStream)
{
await using var sigFile = new TempStream();
@ -996,7 +867,7 @@ namespace Wabbajack.Common
try
{
var cacheFile = Path.Combine(Consts.PatchCacheFolder, $"{srcHash.FromBase64().ToHex()}_{srcHash.FromBase64().ToHex()}.patch");
var cacheFile = Path.Combine(Consts.PatchCacheFolder, $"{srcHash.ToHex()}_{destHash.ToHex()}.patch");
if (!Directory.Exists(Consts.PatchCacheFolder))
Directory.CreateDirectory(Consts.PatchCacheFolder);
@ -1009,10 +880,10 @@ namespace Wabbajack.Common
}
}
public static bool TryGetPatch(string foundHash, string fileHash, out byte[] ePatch)
public static bool TryGetPatch(Hash foundHash, Hash fileHash, out byte[] ePatch)
{
var patchName = Path.Combine(Consts.PatchCacheFolder,
$"{foundHash.FromBase64().ToHex()}_{fileHash.FromBase64().ToHex()}.patch");
$"{foundHash.ToHex()}_{fileHash.ToHex()}.patch");
if (File.Exists(patchName))
{
ePatch = File.ReadAllBytes(patchName);

1
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -29,6 +29,7 @@
<PackageReference Include="Ceras" Version="4.1.7" />
<PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
<PackageReference Include="ini-parser-netstandard" Version="2.5.2" />
<PackageReference Include="MessagePack" Version="2.1.90" />
<PackageReference Include="Microsoft.Win32.Registry" Version="4.7.0" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="Octodiff" Version="1.2.1" />

14
Wabbajack.Lib/ACompiler.cs
View File

@ -46,7 +46,7 @@ namespace Wabbajack.Lib
public ModList ModList = new ModList();
public List<IndexedArchive> IndexedArchives = new List<IndexedArchive>();
public Dictionary<string, IEnumerable<VirtualFile>> IndexedFiles = new Dictionary<string, IEnumerable<VirtualFile>>();
public Dictionary<Hash, IEnumerable<VirtualFile>> IndexedFiles = new Dictionary<Hash, IEnumerable<VirtualFile>>();
public static void Info(string msg)
{
@ -194,25 +194,25 @@ namespace Wabbajack.Lib
{
Info("Building a list of archives based on the files required");
var shas = InstallDirectives.OfType<FromArchive>()
.Select(a => a.ArchiveHashPath[0])
var hashes = InstallDirectives.OfType<FromArchive>()
.Select(a => Hash.FromBase64(a.ArchiveHashPath[0]))
.Distinct();
var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified)
.GroupBy(f => f.File.Hash)
.ToDictionary(f => f.Key, f => f.First());
SelectedArchives = await shas.PMap(Queue, sha => ResolveArchive(sha, archives));
SelectedArchives = await hashes.PMap(Queue, hash => ResolveArchive(hash, archives));
}
public async Task<Archive> ResolveArchive(string sha, IDictionary<string, IndexedArchive> archives)
public async Task<Archive> ResolveArchive(Hash hash, IDictionary<Hash, IndexedArchive> archives)
{
if (archives.TryGetValue(sha, out var found))
if (archives.TryGetValue(hash, out var found))
{
return await ResolveArchive(found);
}
Error($"No match found for Archive sha: {sha} this shouldn't happen");
Error($"No match found for Archive sha: {hash.ToBase64()} this shouldn't happen");
return null;
}

8
Wabbajack.Lib/AInstaller.cs
View File

@ -27,7 +27,7 @@ namespace Wabbajack.Lib
public string ModListArchive { get; private set; }
public ModList ModList { get; private set; }
public Dictionary<string, string> HashedArchives { get; set; }
public Dictionary<Hash, string> HashedArchives { get; set; }
public SystemParameters SystemParameters { get; set; }
@ -127,7 +127,7 @@ namespace Wabbajack.Lib
var grouped = ModList.Directives
.OfType<FromArchive>()
.GroupBy(e => e.ArchiveHashPath[0])
.ToDictionary(k => k.Key);
.ToDictionary(k => Hash.FromBase64(k.Key));
var archives = ModList.Archives
.Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) })
.Where(a => a.AbsolutePath != null)
@ -264,7 +264,7 @@ namespace Wabbajack.Lib
{
var orig_name = Path.GetFileNameWithoutExtension(archive.Name);
var ext = Path.GetExtension(archive.Name);
var unique_key = archive.State.PrimaryKeyString.StringSHA256Hex();
var unique_key = archive.State.PrimaryKeyString.StringSha256Hex();
outputPath = Path.Combine(DownloadFolder, orig_name + "_" + unique_key + "_" + ext);
if (outputPath.FileExists())
File.Delete(outputPath);
@ -413,7 +413,7 @@ namespace Wabbajack.Lib
Utils.Log($"Optimized {ModList.Directives.Count} directives to {indexed.Count} required");
var requiredArchives = indexed.Values.OfType<FromArchive>()
.GroupBy(d => d.ArchiveHashPath[0])
.Select(d => d.Key)
.Select(d => Hash.FromBase64(d.Key))
.ToHashSet();
ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToList();

4
Wabbajack.Lib/ClientAPI.cs
View File

@ -13,10 +13,10 @@ namespace Wabbajack.Lib
return client;
}
public static async Task<Archive> GetModUpgrade(string hash)
public static async Task<Archive> GetModUpgrade(Hash hash)
{
using var response = await GetClient()
.GetAsync($"https://{Consts.WabbajackCacheHostname}/alternative/{hash.FromBase64().ToHex()}");
.GetAsync($"https://{Consts.WabbajackCacheHostname}/alternative/{hash.ToHex()}");
return !response.IsSuccessStatusCode ? null : (await response.Content.ReadAsStringAsync()).FromJSONString<Archive>();
}
}

5
Wabbajack.Lib/CompilationSteps/CompilationErrors/InvalidGameESMError.cs
View File

@ -4,13 +4,14 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed;
namespace Wabbajack.Lib.CompilationSteps.CompilationErrors
{
public class InvalidGameESMError : AErrorMessage
{
public string Hash { get; }
public Hash Hash { get; }
public string PathToFile { get; }
private readonly CleanedESM _esm;
public string GameFileName => Path.GetFileName(_esm.To);
@ -29,7 +30,7 @@ the modlist expecting a different of the game than you currently have installed,
the game, and then attempting to re-install this modlist. Also verify that the version of the game you have installed matches the version expected by this modlist.";
}
public InvalidGameESMError(CleanedESM esm, string hash, string path)
public InvalidGameESMError(CleanedESM esm, Hash hash, string path)
{
Hash = hash;
PathToFile = path;

24
Wabbajack.Lib/Data.cs
View File

@ -26,7 +26,7 @@ namespace Wabbajack.Lib
public VirtualFile File { get; }
public string Hash => File.Hash;
public Hash Hash => File.Hash;
public T EvolveTo<T>() where T : Directive, new()
{
@ -130,7 +130,7 @@ namespace Wabbajack.Lib
/// </summary>
public string To;
public long Size;
public string Hash;
public Hash Hash;
}
public class IgnoredDirectly : Directive
@ -167,7 +167,7 @@ namespace Wabbajack.Lib
public class CleanedESM : InlineFile
{
public string SourceESMHash;
public Hash SourceESMHash;
}
/// <summary>
@ -191,23 +191,13 @@ namespace Wabbajack.Lib
{
private string _fullPath;
/// <summary>
/// MurMur3 hash of the archive this file comes from
/// </summary>
public string[] ArchiveHashPath;
[Exclude]
public VirtualFile FromFile;
[Exclude]
public string FullPath
{
get
{
if (_fullPath == null) _fullPath = string.Join("|", ArchiveHashPath);
return _fullPath;
}
}
public string FullPath => _fullPath ??= string.Join("|", ArchiveHashPath);
}
public class CreateBSA : Directive
@ -226,13 +216,13 @@ namespace Wabbajack.Lib
public string PatchID;
[Exclude]
public string FromHash;
public Hash FromHash;
}
public class SourcePatch
{
public string RelativePath;
public string Hash;
public Hash Hash;
}
public class MergedPatch : Directive
@ -246,7 +236,7 @@ namespace Wabbajack.Lib
/// <summary>
/// MurMur3 Hash of the archive
/// </summary>
public string Hash { get; set; }
public Hash Hash { get; set; }
/// <summary>
/// Meta INI for the downloaded archive

2
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -108,7 +108,7 @@ namespace Wabbajack.Lib.Downloaders
var upgradeResult = await Download(upgrade, upgradePath);
if (!upgradeResult) return false;
var patchName = $"{archive.Hash.FromBase64().ToHex()}_{upgrade.Hash.FromBase64().ToHex()}";
var patchName = $"{archive.Hash.ToHex()}_{upgrade.Hash.ToHex()}";
var patchPath = Path.Combine(Path.GetDirectoryName(destination), "_Patch_" + patchName);
var patchState = new Archive

2
Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
View File

@ -49,7 +49,7 @@ namespace Wabbajack.Lib.Downloaders
{
public Game Game { get; set; }
public string GameFile { get; set; }
public string Hash { get; set; }
public Hash Hash { get; set; }
public string GameVersion { get; set; }

2
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -105,7 +105,7 @@ namespace Wabbajack.Lib.FileUploader
if (!tcs.Task.IsFaulted)
{
progressFn(1.0);
var hash = (await hash_task).FromBase64().ToHex();
var hash = (await hash_task).ToHex();
response = await client.PutAsync(UploadURL + $"/{key}/finish/{hash}", new StringContent(""));
if (response.IsSuccessStatusCode)
tcs.SetResult(await response.Content.ReadAsStringAsync());

6
Wabbajack.Lib/MO2Compiler.cs
View File

@ -45,7 +45,7 @@ namespace Wabbajack.Lib
public override string VFSCacheName => Path.Combine(
Consts.LocalAppDataPath,
$"vfs_compile_cache-{Path.Combine(MO2Folder ?? "Unknown", "ModOrganizer.exe").StringSHA256Hex()}.bin");
$"vfs_compile_cache-{Path.Combine(MO2Folder ?? "Unknown", "ModOrganizer.exe").StringSha256Hex()}.bin");
public MO2Compiler(string mo2Folder, string mo2Profile, string outputFile)
{
@ -318,7 +318,7 @@ namespace Wabbajack.Lib
UpdateTracker.NextStep("Running Validation");
await ValidateModlist.RunValidation(Queue, ModList);
await ValidateModlist.RunValidation(ModList);
UpdateTracker.NextStep("Generating Report");
GenerateManifest();
@ -382,7 +382,7 @@ namespace Wabbajack.Lib
var client = new Common.Http.Client();
using var response =
await client.GetAsync(
$"http://build.wabbajack.org/indexed_files/{vf.Hash.FromBase64().ToHex()}/meta.ini");
$"http://build.wabbajack.org/indexed_files/{vf.Hash.ToHex()}/meta.ini");
if (!response.IsSuccessStatusCode)
{

2
Wabbajack.Lib/MO2Installer.cs
View File

@ -82,7 +82,7 @@ namespace Wabbajack.Lib
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Validating Modlist");
await ValidateModlist.RunValidation(Queue, ModList);
await ValidateModlist.RunValidation(ModList);
Directory.CreateDirectory(OutputFolder);
Directory.CreateDirectory(DownloadFolder);

2
Wabbajack.Lib/ModListRegistry/ModListMetadata.cs
View File

@ -96,7 +96,7 @@ namespace Wabbajack.Lib.ModListRegistry
public class DownloadMetadata
{
public string Hash { get; set; }
public Hash Hash { get; set; }
public long Size { get; set; }
public long NumberOfArchives { get; set; }

103
Wabbajack.Lib/Validation/ValidateModlist.cs
View File

@ -16,21 +16,7 @@ namespace Wabbajack.Lib.Validation
/// </summary>
public class ValidateModlist
{
public Dictionary<string, Author> AuthorPermissions { get; set; } = new Dictionary<string, Author>();
private readonly WorkQueue _queue;
public ServerWhitelist ServerWhitelist { get; set; } = new ServerWhitelist();
public ValidateModlist(WorkQueue workQueue)
{
_queue = workQueue;
}
public void LoadAuthorPermissionsFromString(string s)
{
AuthorPermissions = s.FromYaml<Dictionary<string, Author>>();
}
public ServerWhitelist ServerWhitelist { get; private set; } = new ServerWhitelist();
public void LoadServerWhitelist(string s)
{
ServerWhitelist = s.FromYaml<ServerWhitelist>();
@ -50,9 +36,9 @@ namespace Wabbajack.Lib.Validation
}
public static async Task RunValidation(WorkQueue queue, ModList modlist)
public static async Task RunValidation(ModList modlist)
{
var validator = new ValidateModlist(queue);
var validator = new ValidateModlist();
await validator.LoadListsFromGithub();
@ -69,92 +55,9 @@ namespace Wabbajack.Lib.Validation
}
}
/// <summary>
/// Takes all the permissions for a given Nexus mods and merges them down to a single permissions record
/// the more specific record having precedence in each field.
/// </summary>
/// <param name="mod"></param>
/// <returns></returns>
public Permissions FilePermissions(NexusDownloader.State mod)
{
var author_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Permissions;
var game_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Permissions;
var mod_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Mods.GetOrDefault(mod.ModID)
?.Permissions;
var file_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Mods
.GetOrDefault(mod.ModID)?.Files.GetOrDefault(mod.FileID)?.Permissions;
return new Permissions
{
CanExtractBSAs = file_permissions?.CanExtractBSAs ?? mod_permissions?.CanExtractBSAs ??
game_permissions?.CanExtractBSAs ?? author_permissions?.CanExtractBSAs ?? true,
CanModifyAssets = file_permissions?.CanModifyAssets ?? mod_permissions?.CanModifyAssets ??
game_permissions?.CanModifyAssets ?? author_permissions?.CanModifyAssets ?? true,
CanModifyESPs = file_permissions?.CanModifyESPs ?? mod_permissions?.CanModifyESPs ??
game_permissions?.CanModifyESPs ?? author_permissions?.CanModifyESPs ?? true,
CanUseInOtherGames = file_permissions?.CanUseInOtherGames ?? mod_permissions?.CanUseInOtherGames ??
game_permissions?.CanUseInOtherGames ?? author_permissions?.CanUseInOtherGames ?? true,
};
}
public async Task<IEnumerable<string>> Validate(ModList modlist)
{
ConcurrentStack<string> ValidationErrors = new ConcurrentStack<string>();
var nexus_mod_permissions = (await modlist.Archives
.Where(a => a.State is NexusDownloader.State)
.PMap(_queue, a => (a.Hash, FilePermissions((NexusDownloader.State)a.State), a)))
.ToDictionary(a => a.Hash, a => new { permissions = a.Item2, archive = a.a });
await modlist.Directives
.OfType<PatchedFromArchive>()
.PMap(_queue, p =>
{
if (nexus_mod_permissions.TryGetValue(p.ArchiveHashPath[0], out var archive))
{
var ext = Path.GetExtension(p.ArchiveHashPath.Last());
var url = (archive.archive.State as NexusDownloader.State).URL;
if (Consts.AssetFileExtensions.Contains(ext) && !(archive.permissions.CanModifyAssets ?? true))
{
ValidationErrors.Push($"{p.To} from {url} is set to disallow asset modification");
}
else if (Consts.ESPFileExtensions.Contains(ext) && !(archive.permissions.CanModifyESPs ?? true))
{
ValidationErrors.Push($"{p.To} from {url} is set to disallow asset ESP modification");
}
}
});
await modlist.Directives
.OfType<FromArchive>()
.PMap(_queue, p =>
{
if (nexus_mod_permissions.TryGetValue(p.ArchiveHashPath[0], out var archive))
{
var url = (archive.archive.State as NexusDownloader.State).URL;
if (!(archive.permissions.CanExtractBSAs ?? true) &&
p.ArchiveHashPath.Skip(1).ButLast().Any(a => Consts.SupportedBSAs.Contains(Path.GetExtension(a).ToLower())))
{
ValidationErrors.Push($"{p.To} from {url} is set to disallow BSA extraction");
}
}
});
var nexus = NexusApi.NexusApiUtils.ConvertGameName(modlist.GameType.MetaData().NexusName);
modlist.Archives
.Where(a => a.State is NexusDownloader.State)
.Where(m => NexusApi.NexusApiUtils.ConvertGameName(((NexusDownloader.State)m.State).GameName) != nexus)
.Do(m =>
{
var permissions = FilePermissions((NexusDownloader.State)m.State);
if (!(permissions.CanUseInOtherGames ?? true))
{
ValidationErrors.Push(
$"The ModList is for {nexus} but {m.Name} is for game type {((NexusDownloader.State)m.State).GameName} and is not allowed to be converted to other game types");
}
});
modlist.Archives
.Where(m => !m.State.IsWhitelisted(ServerWhitelist))
.Do(m =>

4
Wabbajack.Lib/VortexCompiler.cs
View File

@ -49,7 +49,7 @@ namespace Wabbajack.Lib
public override string VFSCacheName => Path.Combine(
Consts.LocalAppDataPath,
$"vfs_compile_cache-{StagingFolder?.StringSHA256Hex() ?? "Unknown"}.bin");
$"vfs_compile_cache-{StagingFolder?.StringSha256Hex() ?? "Unknown"}.bin");
public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder, string stagingFolder, string outputFile)
{
@ -245,7 +245,7 @@ namespace Wabbajack.Lib
};
UpdateTracker.NextStep("Running Validation");
await ValidateModlist.RunValidation(Queue, ModList);
await ValidateModlist.RunValidation(ModList);
UpdateTracker.NextStep("Generating Report");
GenerateManifest();

133
Wabbajack.Test/ContentRightsManagementTests.cs
View File

@ -52,8 +52,7 @@ namespace Wabbajack.Test
public void TestSetup()
{
queue = new WorkQueue();
validate = new ValidateModlist(queue);
validate.LoadAuthorPermissionsFromString(permissions);
validate = new ValidateModlist();
validate.LoadServerWhitelist(server_whitelist);
}
@ -63,76 +62,6 @@ namespace Wabbajack.Test
queue?.Dispose();
}
[TestMethod]
public void TestRightsFallthrough()
{
var permissions = validate.FilePermissions(new NexusDownloader.State
{
Author = "bill",
GameName = "Skyrim",
ModID = "42",
FileID = "33"
});
permissions.CanExtractBSAs.AssertIsFalse();
permissions.CanModifyESPs.AssertIsFalse();
permissions.CanModifyAssets.AssertIsFalse();
permissions.CanUseInOtherGames.AssertIsFalse();
permissions = validate.FilePermissions(new NexusDownloader.State
{
Author = "bob",
GameName = "Skyrim",
ModID = "42",
FileID = "33"
});
permissions.CanExtractBSAs.AssertIsTrue();
permissions.CanModifyESPs.AssertIsTrue();
permissions.CanModifyAssets.AssertIsTrue();
permissions.CanUseInOtherGames.AssertIsTrue();
permissions = validate.FilePermissions(new NexusDownloader.State
{
Author = "bill",
GameName = "Fallout4",
ModID = "42",
FileID = "33"
});
permissions.CanExtractBSAs.AssertIsFalse();
permissions.CanModifyESPs.AssertIsTrue();
permissions.CanModifyAssets.AssertIsTrue();
permissions.CanUseInOtherGames.AssertIsTrue();
permissions = validate.FilePermissions(new NexusDownloader.State
{
Author = "bill",
GameName = "Skyrim",
ModID = "43",
FileID = "33"
});
permissions.CanExtractBSAs.AssertIsFalse();
permissions.CanModifyESPs.AssertIsFalse();
permissions.CanModifyAssets.AssertIsTrue();
permissions.CanUseInOtherGames.AssertIsTrue();
permissions = validate.FilePermissions(new NexusDownloader.State
{
Author = "bill",
GameName = "Skyrim",
ModID = "42",
FileID = "31"
});
permissions.CanExtractBSAs.AssertIsFalse();
permissions.CanModifyESPs.AssertIsFalse();
permissions.CanModifyAssets.AssertIsFalse();
permissions.CanUseInOtherGames.AssertIsTrue();
}
[TestMethod]
public async Task TestModValidation()
{
@ -150,9 +79,8 @@ namespace Wabbajack.Test
ModID = "42",
FileID = "33",
},
Hash = "DEADBEEF"
Hash = Hash.FromLong(42)
}
},
Directives = new List<Directive>
{
@ -163,59 +91,14 @@ namespace Wabbajack.Test
}
}
};
IEnumerable<string> errors;
// No errors, simple archive extraction
errors = await validate.Validate(modlist);
Assert.AreEqual(errors.Count(), 0);
// Error due to patched file
modlist.Directives[0] = new PatchedFromArchive
{
PatchID = Guid.NewGuid().ToString(),
ArchiveHashPath = new[] {"DEADBEEF", "foo\\bar\\baz.pex"},
};
errors = await validate.Validate(modlist);
Assert.AreEqual(errors.Count(), 1);
// Error due to extracted BSA file
modlist.Directives[0] = new FromArchive
{
ArchiveHashPath = new[] { "DEADBEEF", "foo.bsa", "foo\\bar\\baz.dds" },
};
errors = await validate.Validate(modlist);
Assert.AreEqual(errors.Count(), 1);
// No error since we're just installing the .bsa, not extracting it
modlist.Directives[0] = new FromArchive
{
ArchiveHashPath = new[] { "DEADBEEF", "foo.bsa"},
};
errors = await validate.Validate(modlist);
Assert.AreEqual(0, errors.Count());
// Error due to game conversion
modlist.GameType = Game.SkyrimSpecialEdition;
modlist.Directives[0] = new FromArchive
{
ArchiveHashPath = new[] { "DEADBEEF", "foo\\bar\\baz.dds" },
};
errors = await validate.Validate(modlist);
Assert.AreEqual(errors.Count(), 1);
// Error due to file downloaded from 3rd party
modlist.GameType = Game.Skyrim;
modlist.Archives[0] = new Archive()
{
State = new HTTPDownloader.State() { Url = "https://somebadplace.com" },
Hash = "DEADBEEF"
Hash = Hash.FromLong(42)
};
errors = await validate.Validate(modlist);
var errors = await validate.Validate(modlist);
Assert.AreEqual(1, errors.Count());
// Ok due to file downloaded from whitelisted 3rd party
@ -223,7 +106,7 @@ namespace Wabbajack.Test
modlist.Archives[0] = new Archive
{
State = new HTTPDownloader.State { Url = "https://somegoodplace.com/baz.7z" },
Hash = "DEADBEEF"
Hash = Hash.FromLong(42)
};
errors = await validate.Validate(modlist);
Assert.AreEqual(0, errors.Count());
@ -234,7 +117,7 @@ namespace Wabbajack.Test
modlist.Archives[0] = new Archive
{
State = new GoogleDriveDownloader.State { Id = "bleg"},
Hash = "DEADBEEF"
Hash = Hash.FromLong(42)
};
errors = await validate.Validate(modlist);
Assert.AreEqual(errors.Count(), 1);
@ -244,7 +127,7 @@ namespace Wabbajack.Test
modlist.Archives[0] = new Archive
{
State = new GoogleDriveDownloader.State { Id = "googleDEADBEEF" },
Hash = "DEADBEEF"
Hash = Hash.FromLong(42)
};
errors = await validate.Validate(modlist);
Assert.AreEqual(0, errors.Count());
@ -256,7 +139,7 @@ namespace Wabbajack.Test
{
using (var workQueue = new WorkQueue())
{
await new ValidateModlist(workQueue).LoadListsFromGithub();
await new ValidateModlist().LoadListsFromGithub();
}
}
}

2
Wabbajack.Test/DownloaderTests.cs
View File

@ -536,7 +536,7 @@ namespace Wabbajack.Test
var archive = new Archive
{
Name = "Cori.7z",
Hash = "gCRVrvzDNH0=",
Hash = Hash.FromBase64("gCRVrvzDNH0="),
State = new NexusDownloader.State
{
GameName = Game.SkyrimSpecialEdition.MetaData().NexusName,

10
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -82,7 +82,7 @@ namespace Wabbajack.VirtualFileSystem.Test
await AddTestRoot();
var files = context.Index.ByHash["qX0GZvIaTKM="];
var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")];
Assert.AreEqual(files.Count(), 2);
}
@ -150,7 +150,7 @@ namespace Wabbajack.VirtualFileSystem.Test
await AddTestRoot();
var files = context.Index.ByHash["qX0GZvIaTKM="];
var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")];
var cleanup = await context.Stage(files);
@ -173,7 +173,7 @@ namespace Wabbajack.VirtualFileSystem.Test
await AddTestRoot();
var files = context.Index.ByHash["qX0GZvIaTKM="];
var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")];
var archive = context.Index.ByRootPath[Path.Combine(VFS_TEST_DIR_FULL, "test.zip")];
var state = context.GetPortableState(files);
@ -181,9 +181,9 @@ namespace Wabbajack.VirtualFileSystem.Test
var new_context = new Context(Queue);
await new_context.IntegrateFromPortable(state,
new Dictionary<string, string> {{archive.Hash, archive.FullPath}});
new Dictionary<Hash, string> {{archive.Hash, archive.FullPath}});
var new_files = new_context.Index.ByHash["qX0GZvIaTKM="];
var new_files = new_context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")];
var close = await new_context.Stage(new_files);

22
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -249,16 +249,16 @@ namespace Wabbajack.VirtualFileSystem
{
Name = f.Parent != null ? f.Name : null,
Hash = f.Hash,
ParentHash = f.Parent?.Hash,
ParentHash = f.Parent?.Hash ?? Hash.Empty,
Size = f.Size
}).ToList();
}
public async Task IntegrateFromPortable(List<PortableFile> state, Dictionary<string, string> links)
public async Task IntegrateFromPortable(List<PortableFile> state, Dictionary<Hash, string> links)
{
var indexedState = state.GroupBy(f => f.ParentHash)
.ToDictionary(f => f.Key ?? "", f => (IEnumerable<PortableFile>) f);
var parents = await indexedState[""]
.ToDictionary(f => f.Key, f => (IEnumerable<PortableFile>) f);
var parents = await indexedState[Hash.Empty]
.PMap(Queue,f => VirtualFile.CreateFromPortable(this, indexedState, links, f));
var newIndex = await Index.Integrate(parents);
@ -297,7 +297,7 @@ namespace Wabbajack.VirtualFileSystem
void BackFillOne(KnownFile file)
{
var parent = newFiles[file.Paths[0]];
var parent = newFiles[Hash.FromBase64(file.Paths[0])];
foreach (var path in file.Paths.Skip(1))
{
if (parentchild.TryGetValue((parent, path), out var foundParent))
@ -331,7 +331,7 @@ namespace Wabbajack.VirtualFileSystem
public class KnownFile
{
public string[] Paths { get; set; }
public string Hash { get; set; }
public Hash Hash { get; set; }
}
public class DisposableList<T> : List<T>, IDisposable
@ -355,7 +355,7 @@ namespace Wabbajack.VirtualFileSystem
public IndexRoot(ImmutableList<VirtualFile> aFiles,
ImmutableDictionary<string, VirtualFile> byFullPath,
ImmutableDictionary<string, ImmutableStack<VirtualFile>> byHash,
ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> byHash,
ImmutableDictionary<string, VirtualFile> byRoot,
ImmutableDictionary<string, ImmutableStack<VirtualFile>> byName)
{
@ -370,7 +370,7 @@ namespace Wabbajack.VirtualFileSystem
{
AllFiles = ImmutableList<VirtualFile>.Empty;
ByFullPath = ImmutableDictionary<string, VirtualFile>.Empty;
ByHash = ImmutableDictionary<string, ImmutableStack<VirtualFile>>.Empty;
ByHash = ImmutableDictionary<Hash, ImmutableStack<VirtualFile>>.Empty;
ByRootPath = ImmutableDictionary<string, VirtualFile>.Empty;
ByName = ImmutableDictionary<string, ImmutableStack<VirtualFile>>.Empty;
}
@ -378,7 +378,7 @@ namespace Wabbajack.VirtualFileSystem
public ImmutableList<VirtualFile> AllFiles { get; }
public ImmutableDictionary<string, VirtualFile> ByFullPath { get; }
public ImmutableDictionary<string, ImmutableStack<VirtualFile>> ByHash { get; }
public ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> ByHash { get; }
public ImmutableDictionary<string, ImmutableStack<VirtualFile>> ByName { get; set; }
public ImmutableDictionary<string, VirtualFile> ByRootPath { get; }
@ -391,7 +391,7 @@ namespace Wabbajack.VirtualFileSystem
.ToImmutableDictionary(f => f.FullPath));
var byHash = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.Where(f => f.Hash != null)
.Where(f => f.Hash != Hash.Empty)
.ToGroupedImmutableDictionary(f => f.Hash));
var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
@ -410,7 +410,7 @@ namespace Wabbajack.VirtualFileSystem
public VirtualFile FileForArchiveHashPath(string[] argArchiveHashPath)
{
var cur = ByHash[argArchiveHashPath[0]].First(f => f.Parent == null);
var cur = ByHash[Hash.FromBase64(argArchiveHashPath[0])].First(f => f.Parent == null);
return argArchiveHashPath.Skip(1).Aggregate(cur, (current, itm) => ByName[itm].First(f => f.Parent == current));
}
}

3
Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
@ -8,7 +9,7 @@ namespace Wabbajack.VirtualFileSystem
public class IndexedVirtualFile
{
public string Name { get; set; }
public string Hash { get; set; }
public Hash Hash { get; set; }
public long Size { get; set; }
public List<IndexedVirtualFile> Children { get; set; } = new List<IndexedVirtualFile>();
}

10
Wabbajack.VirtualFileSystem/PortableFile.cs
View File

@ -1,10 +1,12 @@
namespace Wabbajack.VirtualFileSystem
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public class PortableFile
{
public string Name { get; set; }
public string Hash { get; set; }
public string ParentHash { get; set; }
public Hash Hash { get; set; }
public Hash ParentHash { get; set; }
public long Size { get; set; }
}
}
}

14
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -42,7 +42,7 @@ namespace Wabbajack.VirtualFileSystem
}
}
public string Hash { get; internal set; }
public Hash Hash { get; internal set; }
public ExtendedHashes ExtendedHashes { get; set; }
public long Size { get; internal set; }
@ -214,12 +214,12 @@ namespace Wabbajack.VirtualFileSystem
return self;
}
private static async Task<IndexedVirtualFile> TryGetContentsFromServer(string hash)
private static async Task<IndexedVirtualFile> TryGetContentsFromServer(Hash hash)
{
try
{
var client = new HttpClient();
var response = await client.GetAsync($"http://{Consts.WabbajackCacheHostname}/indexed_files/{hash.FromBase64().ToHex()}");
var response = await client.GetAsync($"http://{Consts.WabbajackCacheHostname}/indexed_files/{hash.ToHex()}");
if (!response.IsSuccessStatusCode)
return null;
@ -274,7 +274,7 @@ namespace Wabbajack.VirtualFileSystem
Parent = parent,
Name = br.ReadString(),
_fullPath = br.ReadString(),
Hash = br.ReadString(),
Hash = br.ReadHash(),
Size = br.ReadInt64(),
LastModified = br.ReadInt64(),
LastAnalyzed = br.ReadInt64(),
@ -288,7 +288,7 @@ namespace Wabbajack.VirtualFileSystem
}
public static VirtualFile CreateFromPortable(Context context,
Dictionary<string, IEnumerable<PortableFile>> state, Dictionary<string, string> links,
Dictionary<Hash, IEnumerable<PortableFile>> state, Dictionary<Hash, string> links,
PortableFile portableFile)
{
var vf = new VirtualFile
@ -305,7 +305,7 @@ namespace Wabbajack.VirtualFileSystem
}
public static VirtualFile CreateFromPortable(Context context, VirtualFile parent,
Dictionary<string, IEnumerable<PortableFile>> state, PortableFile portableFile)
Dictionary<Hash, IEnumerable<PortableFile>> state, PortableFile portableFile)
{
var vf = new VirtualFile
{
@ -323,7 +323,7 @@ namespace Wabbajack.VirtualFileSystem
public string[] MakeRelativePaths()
{
var path = new string[NestingFactor];
path[0] = FilesInFullPath.First().Hash;
path[0] = FilesInFullPath.First().Hash.ToBase64();
var idx = 1;

6
Wabbajack/View Models/Gallery/ModListGalleryVM.cs
View File

@ -65,19 +65,19 @@ namespace Wabbajack
return list
// Sort randomly initially, just to give each list a fair shake
.Shuffle(random)
.AsObservableChangeSet(x => x.DownloadMetadata?.Hash ?? $"Fallback{missingHashFallbackCounter++}");
.AsObservableChangeSet(x => x.DownloadMetadata?.Hash ?? Hash.Empty);
}
catch (Exception ex)
{
Utils.Error(ex);
Error = ErrorResponse.Fail(ex);
return Observable.Empty<IChangeSet<ModlistMetadata, string>>();
return Observable.Empty<IChangeSet<ModlistMetadata, Hash>>();
}
})
// Unsubscribe and release when not active
.FlowSwitch(
this.WhenAny(x => x.IsActive),
valueWhenOff: Observable.Return(ChangeSet<ModlistMetadata, string>.Empty))
valueWhenOff: Observable.Return(ChangeSet<ModlistMetadata, Hash>.Empty))
.Switch()
.RefCount();

2
Wabbajack/View Models/ManifestVM.cs
View File

@ -92,7 +92,7 @@ namespace Wabbajack
return Order(Archives.Where(x =>
{
if (term.StartsWith("hash:"))
return x.Hash.StartsWith(term.Replace("hash:", ""));
return x.Hash.ToString().StartsWith(term.Replace("hash:", ""));
return x.Name.StartsWith(term);
}));
})