more perf improvements to compilation and storage

CHANGELOG.md

@@ -11,6 +11,9 @@
 * Detect when VFS root folders don't exist
 * Only reauth against the Nexus every 3 days (instead of 12 hours)
 * Optimize executable patching by switching to .NET serialization and LZ4 compression
+* Ignore some files Wabbajack creates
+* Improve compilation times by reworking file indexing algorithm
+* Store patch files in byte format instead of base64 strings
 
 #### Version 0.9.1 - 9/5/2019
 * Fixed a bug where having only one profile selected would result in no profiles being selected

VirtualFileSystem/VirtualFileSystem.cs

@@ -472,6 +472,11 @@ namespace VFS
             string fullPath = archive.FullPath + "|" + String.Join("|", archiveHashPath.Skip(1));
             return Lookup(fullPath);
         }
+
+        public IDictionary<VirtualFile, IEnumerable<VirtualFile>> GroupedByArchive()
+        {
+            return _files.Values.GroupBy(f => f.TopLevelArchive).ToDictionary(f => f.Key, f => (IEnumerable<VirtualFile>)f);
+        }
     }
 
     public class StagingGroup : List<VirtualFile>, IDisposable

Wabbajack.Common/Utils.cs

@@ -450,10 +450,10 @@ namespace Wabbajack.Common
             }
         }
 
-        public static void TryGetPatch(string foundHash, string fileHash, out string ePatch)
+        public static void TryGetPatch(string foundHash, string fileHash, out byte[] ePatch)
         {
             var patch_name = Path.Combine("patch_cache", $"{foundHash.FromBase64().ToHEX()}_{fileHash.FromBase64().ToHEX()}.patch");
-            ePatch = File.Exists(patch_name) ? File.ReadAllBytes(patch_name).ToBase64() : null;
+            ePatch = File.Exists(patch_name) ? File.ReadAllBytes(patch_name) : null;
         }
     }
 }

Wabbajack/Compiler.cs

@@ -156,10 +156,12 @@ namespace Wabbajack
                 .ToList();
 
             Info("Indexing Files");
-            IndexedFiles = IndexedArchives.PMap(f =>
+            var grouped = VFS.GroupedByArchive();
+            IndexedFiles = IndexedArchives.Select(f =>
                 {
-                    Status($"Finding files in {Path.GetFileName(f.File.FullPath)}");
-                    return VFS.FilesInArchive(f.File);
+                    if (grouped.TryGetValue(f.File, out var result))
+                        return result;
+                    return new List<VirtualFile>();
                 })
                 .SelectMany(fs => fs)
                 .Concat(IndexedArchives.Select(f => f.File))
@@ -316,7 +318,7 @@ namespace Wabbajack
                         var a = origin.ReadAll();
                         var b = LoadDataForTo(entry.To, absolute_paths);
                         Utils.CreatePatch(a, b, output);
-                        entry.Patch = output.ToArray().ToBase64();
+                        entry.Patch = output.ToArray();
                         Info($"Patch size {entry.Patch.Length} for {entry.To}");
                     }
                 });
@@ -553,6 +555,8 @@ namespace Wabbajack
                 // Theme file MO2 downloads somehow
                 IgnoreEndsWith("splash.png"),
 
+                IgnoreWabbajackInstallCruft(),
+
                 PatchStockESMs(),
 
                 IncludeAllConfigs(),
@@ -561,6 +565,18 @@ namespace Wabbajack
             };
         }
 
+        private Func<RawSourceFile, Directive> IgnoreWabbajackInstallCruft()
+        {
+            var cruft_files = new HashSet<string> {"7z.dll", "7z.exe", "vfs_staged_files\\", "nexus.key_cache", "patch_cache\\", Consts.NexusCacheDirectory + "\\"};
+            return source =>
+            {
+                if (!cruft_files.Any(f => source.Path.StartsWith(f))) return null;
+                var result = source.EvolveTo<IgnoredDirectly>();
+                result.Reason = "Wabbajack Cruft file";
+                return result;
+            };
+        }
+
         private Func<RawSourceFile, Directive> IncludeAllConfigs()
         {
             return source =>

Wabbajack/Data.cs

@@ -156,7 +156,7 @@ namespace Wabbajack
         /// <summary>
         /// The file to apply to the source file to patch it
         /// </summary>
-        public string Patch;
+        public byte[] Patch;
     }
 
     [Serializable]

Wabbajack/Installer.cs

@@ -398,7 +398,7 @@ namespace Wabbajack
                     Status("Patching {0}", Path.GetFileName(to_patch.To));
                     // Read in the patch data
 
-                    var patch_data = to_patch.Patch.FromBase64();
+                    var patch_data = to_patch.Patch;
 
                     var to_file = Path.Combine(Outputfolder, to_patch.To);
                     MemoryStream old_data = new MemoryStream(File.ReadAllBytes(to_file));