From 8a680a8f146e618c60c5b607fb43dab7595e773b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 21 Nov 2019 22:19:42 -0700 Subject: [PATCH] Several fixes for compilation and caching. --- Compression.BSA.Test/BSATests.cs | 4 +- Wabbajack.CacheServer/NexusCacheModule.cs | 4 +- Wabbajack.Common/StatusFileStream.cs | 4 +- Wabbajack.Common/Utils.cs | 42 ++++++++--- .../CompilationSteps/IncludePatches.cs | 1 + Wabbajack.Lib/Data.cs | 3 + Wabbajack.Lib/Downloaders/NexusDownloader.cs | 4 +- Wabbajack.Lib/MO2Compiler.cs | 69 ++++++++++--------- Wabbajack.Lib/NexusApi/NexusApi.cs | 4 +- Wabbajack.Lib/Validation/ValidateModlist.cs | 5 ++ Wabbajack.Test/EndToEndTests.cs | 2 +- Wabbajack.VirtualFileSystem/Context.cs | 63 +++++++++++++---- 12 files changed, 140 insertions(+), 65 deletions(-) diff --git a/Compression.BSA.Test/BSATests.cs b/Compression.BSA.Test/BSATests.cs index 67acb98d..96116f30 100644 --- a/Compression.BSA.Test/BSATests.cs +++ b/Compression.BSA.Test/BSATests.cs @@ -64,8 +64,8 @@ namespace Compression.BSA.Test using (var client = new NexusApiClient()) { var results = client.GetModFiles(info.Item1, info.Item2); - var file = results.FirstOrDefault(f => f.is_primary) ?? - results.OrderByDescending(f => f.uploaded_timestamp).First(); + var file = results.files.FirstOrDefault(f => f.is_primary) ?? + results.files.OrderByDescending(f => f.uploaded_timestamp).First(); var src = Path.Combine(_stagingFolder, file.file_name); if (File.Exists(src)) return src; diff --git a/Wabbajack.CacheServer/NexusCacheModule.cs b/Wabbajack.CacheServer/NexusCacheModule.cs index 9e36bf3d..88213c19 100644 --- a/Wabbajack.CacheServer/NexusCacheModule.cs +++ b/Wabbajack.CacheServer/NexusCacheModule.cs @@ -85,7 +85,9 @@ namespace Wabbajack.CacheServer var client = new HttpClient(); var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 80}; client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]); - return client.GetStringSync(builder.Uri.ToString()); + client.GetStringSync(builder.Uri.ToString()); + if (!File.Exists(path)) + throw new InvalidDataException("Invalid Data"); } Utils.Log($"{DateTime.Now} - From Cached - {url}"); diff --git a/Wabbajack.Common/StatusFileStream.cs b/Wabbajack.Common/StatusFileStream.cs index e321d457..69a61909 100644 --- a/Wabbajack.Common/StatusFileStream.cs +++ b/Wabbajack.Common/StatusFileStream.cs @@ -5,9 +5,9 @@ namespace Wabbajack.Common public class StatusFileStream : Stream { private string _message; - private FileStream _inner; + private Stream _inner; - public StatusFileStream(FileStream fs, string message) + public StatusFileStream(Stream fs, string message) { _inner = fs; _message = message; diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 7bb3814f..05b630dc 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -177,10 +177,29 @@ namespace Wabbajack.Common Status(status, (int) (totalRead * 100 / maxSize)); } } - - public static string SHA256(this byte[] data) + public static string xxHash(this byte[] data, bool nullOnIOError = false) { - return new SHA256Managed().ComputeHash(data).ToBase64(); + try + { + var hash = new xxHashConfig(); + hash.HashSizeInBits = 64; + hash.Seed = 0x42; + using (var fs = new MemoryStream(data)) + { + var config = new xxHashConfig(); + config.HashSizeInBits = 64; + using (var f = new StatusFileStream(fs, $"Hashing memory stream")) + { + var value = xxHashFactory.Instance.Create(config).ComputeHash(f); + return value.AsBase64String(); + } + } + } + catch (IOException ex) + { + if (nullOnIOError) return null; + throw ex; + } } /// @@ -594,8 +613,8 @@ namespace Wabbajack.Common public static void CreatePatch(byte[] a, byte[] b, Stream output) { - var dataA = a.SHA256().FromBase64().ToHex(); - var dataB = b.SHA256().FromBase64().ToHex(); + var dataA = a.xxHash().FromBase64().ToHex(); + var dataB = b.xxHash().FromBase64().ToHex(); var cacheFile = Path.Combine("patch_cache", $"{dataA}_{dataB}.patch"); if (!Directory.Exists("patch_cache")) Directory.CreateDirectory("patch_cache"); @@ -618,7 +637,7 @@ namespace Wabbajack.Common BSDiff.Create(a, b, f); } - File.Move(tmpName, cacheFile); + File.Move(tmpName, cacheFile, MoveOptions.ReplaceExisting); continue; } @@ -626,11 +645,18 @@ namespace Wabbajack.Common } } - public static void TryGetPatch(string foundHash, string fileHash, out byte[] ePatch) + public static bool TryGetPatch(string foundHash, string fileHash, out byte[] ePatch) { var patchName = Path.Combine("patch_cache", $"{foundHash.FromBase64().ToHex()}_{fileHash.FromBase64().ToHex()}.patch"); - ePatch = File.Exists(patchName) ? File.ReadAllBytes(patchName) : null; + if (File.Exists(patchName)) + { + ePatch = File.ReadAllBytes(patchName); + return true; + } + + ePatch = null; + return false; } public static void Warning(string s) diff --git a/Wabbajack.Lib/CompilationSteps/IncludePatches.cs b/Wabbajack.Lib/CompilationSteps/IncludePatches.cs index 379be67c..58e0568d 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludePatches.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludePatches.cs @@ -38,6 +38,7 @@ namespace Wabbajack.Lib.CompilationSteps } var e = source.EvolveTo(); + e.FromHash = found.Hash; e.ArchiveHashPath = found.MakeRelativePaths(); e.To = source.Path; e.Hash = source.File.Hash; diff --git a/Wabbajack.Lib/Data.cs b/Wabbajack.Lib/Data.cs index 1a312bde..82cd6593 100644 --- a/Wabbajack.Lib/Data.cs +++ b/Wabbajack.Lib/Data.cs @@ -207,6 +207,9 @@ namespace Wabbajack.Lib /// The file to apply to the source file to patch it /// public string PatchID; + + [Exclude] + public string FromHash; } public class SourcePatch diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 7a99ddac..bacb18e4 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -102,13 +102,13 @@ namespace Wabbajack.Lib.Downloaders { var modfiles = new NexusApiClient().GetModFiles(GameRegistry.GetByMO2ArchiveName(GameName).Game, int.Parse(ModID)); var fileid = ulong.Parse(FileID); - var found = modfiles + var found = modfiles.files .FirstOrDefault(file => file.file_id == fileid && file.category_name != null); return found != null; } catch (Exception ex) { - Utils.Log($"{ModName} - {GameName} - {ModID} - {FileID} - Error Getting Nexus Download URL - {ex.Message}"); + Utils.Log($"{ModName} - {GameName} - {ModID} - {FileID} - Error Getting Nexus Download URL - {ex}"); return false; } diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 0ef9951f..d0ea02ee 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -78,25 +78,32 @@ namespace Wabbajack.Lib VFS.IntegrateFromFile(_vfsCacheName); - UpdateTracker.NextStep($"Indexing {MO2Folder}"); - VFS.AddRoot(MO2Folder); + var roots = new List() + { + MO2Folder, GamePath, MO2DownloadsFolder + }; + + // TODO: make this generic so we can add more paths - UpdateTracker.NextStep("Writing VFS Cache"); + var lootPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + "LOOT"); + IEnumerable lootFiles = new List(); + if (Directory.Exists(lootPath)) + { + roots.Add(lootPath); + } + UpdateTracker.NextStep("Indexing folders"); + + VFS.AddRoots(roots); VFS.WriteToFile(_vfsCacheName); - - UpdateTracker.NextStep($"Indexing {GamePath}"); - VFS.AddRoot(GamePath); - - UpdateTracker.NextStep("Writing VFS Cache"); - VFS.WriteToFile(_vfsCacheName); - - - UpdateTracker.NextStep($"Indexing {MO2DownloadsFolder}"); - VFS.AddRoot(MO2DownloadsFolder); - - UpdateTracker.NextStep("Writing VFS Cache"); - VFS.WriteToFile(_vfsCacheName); - + + if (Directory.Exists(lootPath)) + { + lootFiles = Directory.EnumerateFiles(lootPath, "userlist.yaml", SearchOption.AllDirectories) + .Where(p => p.FileExists()) + .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p]) + { Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath)) }); + } UpdateTracker.NextStep("Cleaning output folder"); if (Directory.Exists(ModListOutputFolder)) @@ -114,23 +121,8 @@ namespace Wabbajack.Lib .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p]) { Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) }); - var lootPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), - "LOOT"); - - // TODO: make this generic so we can add more paths - IEnumerable lootFiles = new List(); - if (Directory.Exists(lootPath)) - { - Info($"Indexing {lootPath}"); - VFS.AddRoot(lootPath); - VFS.WriteToFile(_vfsCacheName); - lootFiles = Directory.EnumerateFiles(lootPath, "userlist.yaml", SearchOption.AllDirectories) - .Where(p => p.FileExists()) - .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p]) - { Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath)) }); - } IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder) .Where(f => File.Exists(f + ".meta")) @@ -297,6 +289,15 @@ namespace Wabbajack.Lib private void BuildPatches() { Info("Gathering patch files"); + + InstallDirectives.OfType() + .Where(p => p.PatchID == null) + .Do(p => + { + if (Utils.TryGetPatch(p.FromHash, p.Hash, out var bytes)) + p.PatchID = IncludeFile(bytes); + }); + var groups = InstallDirectives.OfType() .Where(p => p.PatchID == null) .GroupBy(p => p.ArchiveHashPath[0]) @@ -326,7 +327,7 @@ namespace Wabbajack.Lib using (var output = new MemoryStream()) { var a = origin.ReadAll(); - var b = LoadDataForTo(entry.To, absolutePaths).Result; + var b = LoadDataForTo(entry.To, absolutePaths); Utils.CreatePatch(a, b, output); entry.PatchID = IncludeFile(output.ToArray()); var fileSize = File.GetSize(Path.Combine(ModListOutputFolder, entry.PatchID)); @@ -336,7 +337,7 @@ namespace Wabbajack.Lib } } - private async Task LoadDataForTo(string to, Dictionary absolutePaths) + private byte[] LoadDataForTo(string to, Dictionary absolutePaths) { if (absolutePaths.TryGetValue(to, out var absolute)) return File.ReadAllBytes(absolute); diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index 8faf3296..5beef59a 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -289,10 +289,10 @@ namespace Wabbajack.Lib.NexusApi public List files; } - public IList GetModFiles(Game game, int modid) + public GetModFilesResponse GetModFiles(Game game, int modid) { var url = $"https://api.nexusmods.com/v1/games/{GameRegistry.Games[game].NexusName}/mods/{modid}/files.json"; - return GetCached(url).files; + return GetCached(url); } public List GetModInfoFromMD5(Game game, string md5Hash) diff --git a/Wabbajack.Lib/Validation/ValidateModlist.cs b/Wabbajack.Lib/Validation/ValidateModlist.cs index 51ff03de..72f6dc9f 100644 --- a/Wabbajack.Lib/Validation/ValidateModlist.cs +++ b/Wabbajack.Lib/Validation/ValidateModlist.cs @@ -77,6 +77,11 @@ namespace Wabbajack.Lib.Validation /// public Permissions FilePermissions(NexusDownloader.State mod) { + if (mod.Author == null || mod.GameName == null || mod.ModID == null || mod.FileID == null) + { + Utils.Error($"Error: Null data for {mod.Author} {mod.GameName} {mod.ModID} {mod.FileID}"); + } + var author_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Permissions; var game_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Permissions; var mod_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Mods.GetOrDefault(mod.ModID) diff --git a/Wabbajack.Test/EndToEndTests.cs b/Wabbajack.Test/EndToEndTests.cs index 1a82be7a..d08f3e22 100644 --- a/Wabbajack.Test/EndToEndTests.cs +++ b/Wabbajack.Test/EndToEndTests.cs @@ -103,7 +103,7 @@ namespace Wabbajack.Test { utils.AddMod(mod_name); var client = new NexusApiClient(); - var file = client.GetModFiles(game, modid).First(f => f.is_primary); + var file = client.GetModFiles(game, modid).files.First(f => f.is_primary); var src = Path.Combine(DOWNLOAD_FOLDER, file.file_name); var ini = string.Join("\n", diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index abbf58d9..443e1b11 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Reactive.Linq; using System.Reactive.Subjects; using System.Text; +using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; using Wabbajack.Common.CSP; @@ -81,6 +82,42 @@ namespace Wabbajack.VirtualFileSystem return newIndex; } + public IndexRoot AddRoots(List roots) + { + if (!roots.All(p => Path.IsPathRooted(p))) + throw new InvalidDataException($"Paths are not absolute"); + + var filtered = Index.AllFiles.Where(file => File.Exists(file.Name)).ToList(); + + var byPath = filtered.ToImmutableDictionary(f => f.Name); + + var filesToIndex = roots.SelectMany(root => Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive)).ToList(); + + var results = Channel.Create(1024, ProgressUpdater($"Indexing roots", filesToIndex.Count)); + + var allFiles = filesToIndex + .PMap(Queue, f => + { + if (byPath.TryGetValue(f, out var found)) + { + var fi = new FileInfo(f); + if (found.LastModified == fi.LastWriteTimeUtc.Ticks && found.Size == fi.Length) + return found; + } + + return VirtualFile.Analyze(this, null, f, f); + }); + + var newIndex = IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList()); + + lock (this) + { + Index = newIndex; + } + + return newIndex; + } + class Box { public T Value { get; set; } @@ -338,26 +375,26 @@ namespace Wabbajack.VirtualFileSystem public IndexRoot Integrate(List files) { - Utils.Log($"Integrating"); + Utils.Log($"Integrating {files.Count} files"); var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList(); - var byFullPath = allFiles.SelectMany(f => f.ThisAndAllChildren) - .ToImmutableDictionary(f => f.FullPath); + var byFullPath = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) + .ToImmutableDictionary(f => f.FullPath)); - var byHash = allFiles.SelectMany(f => f.ThisAndAllChildren) + var byHash = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) .Where(f => f.Hash != null) - .ToGroupedImmutableDictionary(f => f.Hash); + .ToGroupedImmutableDictionary(f => f.Hash)); - var byName = allFiles.SelectMany(f => f.ThisAndAllChildren) - .ToGroupedImmutableDictionary(f => f.Name); + var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) + .ToGroupedImmutableDictionary(f => f.Name)); - var byRootPath = allFiles.ToImmutableDictionary(f => f.Name); + var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.Name)); var result = new IndexRoot(allFiles, - byFullPath, - byHash, - byRootPath, - byName); + byFullPath.Result, + byHash.Result, + byRootPath.Result, + byName.Result); Utils.Log($"Done integrating"); return result; } @@ -383,4 +420,4 @@ namespace Wabbajack.VirtualFileSystem Directory.Delete(FullName, true, true); } } -} \ No newline at end of file +}