Merge pull request #200 from wabbajack-tools/compilation-fixes

Compilation fixes

Compression.BSA.Test/BSATests.cs

@@ -64,8 +64,8 @@ namespace Compression.BSA.Test
             using (var client = new NexusApiClient())
             {
                 var results = client.GetModFiles(info.Item1, info.Item2);
-                var file = results.FirstOrDefault(f => f.is_primary) ??
+                var file = results.files.FirstOrDefault(f => f.is_primary) ??
-                           results.OrderByDescending(f => f.uploaded_timestamp).First();
+                           results.files.OrderByDescending(f => f.uploaded_timestamp).First();
                 var src = Path.Combine(_stagingFolder, file.file_name);
 
                 if (File.Exists(src)) return src;

Wabbajack.CacheServer/NexusCacheModule.cs

@@ -85,7 +85,9 @@ namespace Wabbajack.CacheServer
                 var client = new HttpClient();
                 var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 80};
                 client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]);
-                return client.GetStringSync(builder.Uri.ToString());
+                client.GetStringSync(builder.Uri.ToString());
+                if (!File.Exists(path))
+                    throw new InvalidDataException("Invalid Data");
             }
 
             Utils.Log($"{DateTime.Now} - From Cached - {url}");

Wabbajack.Common/StatusFileStream.cs

@@ -5,9 +5,9 @@ namespace Wabbajack.Common
     public class StatusFileStream : Stream
     {
         private string _message;
-        private FileStream _inner;
+        private Stream _inner;
 
-        public StatusFileStream(FileStream fs, string message)
+        public StatusFileStream(Stream fs, string message)
         {
             _inner = fs;
             _message = message;

Wabbajack.Common/Utils.cs

@@ -177,10 +177,29 @@ namespace Wabbajack.Common
                 Status(status, (int) (totalRead * 100 / maxSize));
             }
         }
-
+        public static string xxHash(this byte[] data, bool nullOnIOError = false)
-        public static string SHA256(this byte[] data)
         {
-            return new SHA256Managed().ComputeHash(data).ToBase64();
+            try
+            {
+                var hash = new xxHashConfig();
+                hash.HashSizeInBits = 64;
+                hash.Seed = 0x42;
+                using (var fs = new MemoryStream(data))
+                {
+                    var config = new xxHashConfig();
+                    config.HashSizeInBits = 64;
+                    using (var f = new StatusFileStream(fs, $"Hashing memory stream"))
+                    {
+                        var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
+                        return value.AsBase64String();
+                    }
+                }
+            }
+            catch (IOException ex)
+            {
+                if (nullOnIOError) return null;
+                throw ex;
+            }
         }
 
         /// <summary>
@@ -594,8 +613,8 @@ namespace Wabbajack.Common
 
         public static void CreatePatch(byte[] a, byte[] b, Stream output)
         {
-            var dataA = a.SHA256().FromBase64().ToHex();
+            var dataA = a.xxHash().FromBase64().ToHex();
-            var dataB = b.SHA256().FromBase64().ToHex();
+            var dataB = b.xxHash().FromBase64().ToHex();
             var cacheFile = Path.Combine("patch_cache", $"{dataA}_{dataB}.patch");
             if (!Directory.Exists("patch_cache"))
                 Directory.CreateDirectory("patch_cache");
@@ -618,7 +637,7 @@ namespace Wabbajack.Common
                         BSDiff.Create(a, b, f);
                     }
 
-                    File.Move(tmpName, cacheFile);
+                    File.Move(tmpName, cacheFile, MoveOptions.ReplaceExisting);
                     continue;
                 }
 
@@ -626,11 +645,18 @@ namespace Wabbajack.Common
             }
         }
 
-        public static void TryGetPatch(string foundHash, string fileHash, out byte[] ePatch)
+        public static bool TryGetPatch(string foundHash, string fileHash, out byte[] ePatch)
         {
             var patchName = Path.Combine("patch_cache",
                 $"{foundHash.FromBase64().ToHex()}_{fileHash.FromBase64().ToHex()}.patch");
-            ePatch = File.Exists(patchName) ? File.ReadAllBytes(patchName) : null;
+            if (File.Exists(patchName))
+            {
+                ePatch = File.ReadAllBytes(patchName);
+                return true;
+            }
+
+            ePatch = null;
+            return false;
         }
 
         public static void Warning(string s)

Wabbajack.Lib/CompilationSteps/IncludePatches.cs

@@ -38,6 +38,7 @@ namespace Wabbajack.Lib.CompilationSteps
             }
 
             var e = source.EvolveTo<PatchedFromArchive>();
+            e.FromHash = found.Hash;
             e.ArchiveHashPath = found.MakeRelativePaths();
             e.To = source.Path;
             e.Hash = source.File.Hash;

Wabbajack.Lib/Data.cs

@@ -207,6 +207,9 @@ namespace Wabbajack.Lib
         ///     The file to apply to the source file to patch it
         /// </summary>
         public string PatchID;
+
+        [Exclude]
+        public string FromHash;
     }
 
     public class SourcePatch

Wabbajack.Lib/Downloaders/NexusDownloader.cs

@@ -102,13 +102,13 @@ namespace Wabbajack.Lib.Downloaders
                 {
                     var modfiles = new NexusApiClient().GetModFiles(GameRegistry.GetByMO2ArchiveName(GameName).Game, int.Parse(ModID));
                     var fileid = ulong.Parse(FileID);
-                    var found = modfiles
+                    var found = modfiles.files
                         .FirstOrDefault(file => file.file_id == fileid && file.category_name != null);
                     return found != null;
                 }
                 catch (Exception ex)
                 {
-                    Utils.Log($"{ModName} - {GameName} - {ModID} - {FileID} - Error Getting Nexus Download URL - {ex.Message}");
+                    Utils.Log($"{ModName} - {GameName} - {ModID} - {FileID} - Error Getting Nexus Download URL - {ex}");
                     return false;
                 }
 

Wabbajack.Lib/MO2Compiler.cs

@@ -78,25 +78,32 @@ namespace Wabbajack.Lib
 
             VFS.IntegrateFromFile(_vfsCacheName);
 
-            UpdateTracker.NextStep($"Indexing {MO2Folder}");
+            var roots = new List<string>()
-            VFS.AddRoot(MO2Folder);
+            {
+                MO2Folder, GamePath, MO2DownloadsFolder
+            };
             
-            UpdateTracker.NextStep("Writing VFS Cache");
+            // TODO: make this generic so we can add more paths
-            VFS.WriteToFile(_vfsCacheName);
+
-
+            var lootPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
-            UpdateTracker.NextStep($"Indexing {GamePath}");
+                "LOOT");
-            VFS.AddRoot(GamePath);
+            IEnumerable<RawSourceFile> lootFiles = new List<RawSourceFile>();
-
+            if (Directory.Exists(lootPath))
-            UpdateTracker.NextStep("Writing VFS Cache");
+            {
-            VFS.WriteToFile(_vfsCacheName);
+                roots.Add(lootPath);
-
+            }
-
+            UpdateTracker.NextStep("Indexing folders");
-            UpdateTracker.NextStep($"Indexing {MO2DownloadsFolder}");
+
-            VFS.AddRoot(MO2DownloadsFolder);
+            VFS.AddRoots(roots);
-
-            UpdateTracker.NextStep("Writing VFS Cache");
             VFS.WriteToFile(_vfsCacheName);
             
+            if (Directory.Exists(lootPath))
+            {
+                    lootFiles = Directory.EnumerateFiles(lootPath, "userlist.yaml", SearchOption.AllDirectories)
+                    .Where(p => p.FileExists())
+                    .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
+                        { Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath)) });
+            }
 
             UpdateTracker.NextStep("Cleaning output folder");
             if (Directory.Exists(ModListOutputFolder))
@@ -114,23 +121,8 @@ namespace Wabbajack.Lib
                 .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
                 { Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) });
 
-            var lootPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
-                "LOOT");
-
-            // TODO: make this generic so we can add more paths
-            IEnumerable<RawSourceFile> lootFiles = new List<RawSourceFile>();
-            if (Directory.Exists(lootPath))
-            {
-                Info($"Indexing {lootPath}");
-                VFS.AddRoot(lootPath);
-                VFS.WriteToFile(_vfsCacheName);
 
 
-                lootFiles = Directory.EnumerateFiles(lootPath, "userlist.yaml", SearchOption.AllDirectories)
-                    .Where(p => p.FileExists())
-                    .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
-                    { Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath)) });
-            }
 
             IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder)
                 .Where(f => File.Exists(f + ".meta"))
@@ -297,6 +289,15 @@ namespace Wabbajack.Lib
         private void BuildPatches()
         {
             Info("Gathering patch files");
+
+            InstallDirectives.OfType<PatchedFromArchive>()
+                .Where(p => p.PatchID == null)
+                .Do(p =>
+                {
+                    if (Utils.TryGetPatch(p.FromHash, p.Hash, out var bytes))
+                        p.PatchID = IncludeFile(bytes);
+                });
+
             var groups = InstallDirectives.OfType<PatchedFromArchive>()
                 .Where(p => p.PatchID == null)
                 .GroupBy(p => p.ArchiveHashPath[0])
@@ -326,7 +327,7 @@ namespace Wabbajack.Lib
                     using (var output = new MemoryStream())
                     {
                         var a = origin.ReadAll();
-                        var b = LoadDataForTo(entry.To, absolutePaths).Result;
+                        var b = LoadDataForTo(entry.To, absolutePaths);
                         Utils.CreatePatch(a, b, output);
                         entry.PatchID = IncludeFile(output.ToArray());
                         var fileSize = File.GetSize(Path.Combine(ModListOutputFolder, entry.PatchID));
@@ -336,7 +337,7 @@ namespace Wabbajack.Lib
             }
         }
 
-        private async Task<byte[]> LoadDataForTo(string to, Dictionary<string, string> absolutePaths)
+        private byte[] LoadDataForTo(string to, Dictionary<string, string> absolutePaths)
         {
             if (absolutePaths.TryGetValue(to, out var absolute))
                 return File.ReadAllBytes(absolute);

Wabbajack.Lib/NexusApi/NexusApi.cs

@@ -289,10 +289,10 @@ namespace Wabbajack.Lib.NexusApi
             public List<NexusFileInfo> files;
         }
 
-        public IList<NexusFileInfo> GetModFiles(Game game, int modid)
+        public GetModFilesResponse GetModFiles(Game game, int modid)
         {
             var url = $"https://api.nexusmods.com/v1/games/{GameRegistry.Games[game].NexusName}/mods/{modid}/files.json";
-            return GetCached<GetModFilesResponse>(url).files;
+            return GetCached<GetModFilesResponse>(url);
         }
 
         public List<MD5Response> GetModInfoFromMD5(Game game, string md5Hash)

Wabbajack.Lib/Validation/ValidateModlist.cs

@@ -77,6 +77,11 @@ namespace Wabbajack.Lib.Validation
         /// <returns></returns>
         public Permissions FilePermissions(NexusDownloader.State mod)
         {
+            if (mod.Author == null || mod.GameName == null || mod.ModID == null || mod.FileID == null)
+            {
+                Utils.Error($"Error: Null data for {mod.Author} {mod.GameName} {mod.ModID} {mod.FileID}");
+            }
+
             var author_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Permissions;
             var game_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Permissions;
             var mod_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Mods.GetOrDefault(mod.ModID)

Wabbajack.Test/EndToEndTests.cs

@@ -103,7 +103,7 @@ namespace Wabbajack.Test
         {
             utils.AddMod(mod_name);
             var client = new NexusApiClient();
-            var file = client.GetModFiles(game, modid).First(f => f.is_primary);
+            var file = client.GetModFiles(game, modid).files.First(f => f.is_primary);
             var src = Path.Combine(DOWNLOAD_FOLDER, file.file_name);
 
             var ini = string.Join("\n",

Wabbajack.VirtualFileSystem/Context.cs

@@ -6,6 +6,7 @@ using System.Linq;
 using System.Reactive.Linq;
 using System.Reactive.Subjects;
 using System.Text;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
 using Wabbajack.Common.CSP;
@@ -54,7 +55,7 @@ namespace Wabbajack.VirtualFileSystem
 
             var byPath = filtered.ToImmutableDictionary(f => f.Name);
 
-            var filesToIndex = Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive).ToList();
+            var filesToIndex = Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive).Distinct().ToList();
             
             var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing {root}", filesToIndex.Count));
 
@@ -81,6 +82,42 @@ namespace Wabbajack.VirtualFileSystem
             return newIndex;
         }
 
+        public IndexRoot AddRoots(List<string> roots)
+        {
+            if (!roots.All(p => Path.IsPathRooted(p)))
+                throw new InvalidDataException($"Paths are not absolute");
+
+            var filtered = Index.AllFiles.Where(file => File.Exists(file.Name)).ToList();
+
+            var byPath = filtered.ToImmutableDictionary(f => f.Name);
+
+            var filesToIndex = roots.SelectMany(root => Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive)).ToList();
+
+            var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing roots", filesToIndex.Count));
+
+            var allFiles = filesToIndex
+                .PMap(Queue, f =>
+                {
+                    if (byPath.TryGetValue(f, out var found))
+                    {
+                        var fi = new FileInfo(f);
+                        if (found.LastModified == fi.LastWriteTimeUtc.Ticks && found.Size == fi.Length)
+                            return found;
+                    }
+
+                    return VirtualFile.Analyze(this, null, f, f);
+                });
+
+            var newIndex = IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
+
+            lock (this)
+            {
+                Index = newIndex;
+            }
+
+            return newIndex;
+        }
+
         class Box<T>
         {
             public T Value { get; set; }
@@ -338,26 +375,26 @@ namespace Wabbajack.VirtualFileSystem
 
         public IndexRoot Integrate(List<VirtualFile> files)
         {
-            Utils.Log($"Integrating");
+            Utils.Log($"Integrating {files.Count} files");
             var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList();
 
-            var byFullPath = allFiles.SelectMany(f => f.ThisAndAllChildren)
+            var byFullPath = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
-                                     .ToImmutableDictionary(f => f.FullPath);
+                                     .ToImmutableDictionary(f => f.FullPath));
 
-            var byHash = allFiles.SelectMany(f => f.ThisAndAllChildren)
+            var byHash = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
                                  .Where(f => f.Hash != null)
-                                 .ToGroupedImmutableDictionary(f => f.Hash);
+                                 .ToGroupedImmutableDictionary(f => f.Hash));
 
-            var byName = allFiles.SelectMany(f => f.ThisAndAllChildren)
+            var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
-                                 .ToGroupedImmutableDictionary(f => f.Name);
+                                 .ToGroupedImmutableDictionary(f => f.Name));
 
-            var byRootPath = allFiles.ToImmutableDictionary(f => f.Name);
+            var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.Name));
 
             var result = new IndexRoot(allFiles,
-                byFullPath,
+                byFullPath.Result,
-                byHash,
+                byHash.Result,
-                byRootPath,
+                byRootPath.Result,
-                byName);
+                byName.Result);
             Utils.Log($"Done integrating");
             return result;
         }