Merge pull request #1132 from wabbajack-tools/more-abstract-compiler

More abstract compiler

Wabbajack.Common/Consts.cs

@@ -130,6 +130,8 @@ namespace Wabbajack.Common
         public static AbsolutePath SettingsFile => LocalAppDataPath.Combine("settings.json");
         public static RelativePath SettingsIni = (RelativePath)"settings.ini";
         public static byte SettingsVersion => 2;
+        public static RelativePath NativeSettingsJson = (RelativePath)"native_compiler_settings.json";
+
         public static bool IsServer = false;
 
         public static string CompressedBodyHeader = "x-compressed-body";

Wabbajack.Lib/ACompiler.cs

@@ -19,46 +19,75 @@ namespace Wabbajack.Lib
 {
     public abstract class ACompiler : ABatchProcessor
     {
-        public string? ModListName, ModListAuthor, ModListDescription, ModListWebsite, ModlistReadme;
+        protected readonly Subject<(string, float)> _progressUpdates = new Subject<(string, float)>();
-        public Version? ModlistVersion;
+
+        public List<IndexedArchive> IndexedArchives = new List<IndexedArchive>();
+
+        public Dictionary<Hash, IEnumerable<VirtualFile>> IndexedFiles =
+            new Dictionary<Hash, IEnumerable<VirtualFile>>();
+
+        public ModList ModList = new ModList();
         public AbsolutePath ModListImage;
         public bool ModlistIsNSFW;
+
+        public string? ModListName, ModListAuthor, ModListDescription, ModListWebsite, ModlistReadme;
+        public Version? ModlistVersion;
         protected Version? WabbajackVersion;
 
-        public abstract AbsolutePath VFSCacheName { get; }
+        public ACompiler(int steps, string modlistName, AbsolutePath sourcePath, AbsolutePath downloadsPath,
+            AbsolutePath outputModListName)
+            : base(steps)
+        {
+            SourcePath = sourcePath;
+            DownloadsPath = downloadsPath;
+            ModListName = modlistName;
+            ModListOutputFile = outputModListName;
+            //set in MainWindowVM
+            WabbajackVersion = Consts.CurrentMinimumWabbajackVersion;
+            Settings = new CompilerSettings();
+            ModListOutputFolder = AbsolutePath.EntryPoint.Combine("output_folder", Guid.NewGuid().ToString());
+            CompilingGame = new GameMetaData();
+        }
+
+        /// <summary>
+        /// Set to true to include game files during compilation, only ever disabled
+        /// in testing (to speed up tests)
+        /// </summary>
+        public bool UseGamePaths { get; set; } = true;
+
+        public CompilerSettings Settings { get; set; }
+
+        public AbsolutePath VFSCacheName =>
+            Consts.LocalAppDataPath.Combine(
+                $"vfs_compile_cache-2-{Path.Combine((string)SourcePath ?? "Unknown", "ModOrganizer.exe").StringSha256Hex()}.bin");
+
         //protected string VFSCacheName => Path.Combine(Consts.LocalAppDataPath, $"vfs_compile_cache.bin");
         /// <summary>
         /// A stream of tuples of ("Update Title", 0.25) which represent the name of the current task
         /// and the current progress.
         /// </summary>
         public IObservable<(string, float)> ProgressUpdates => _progressUpdates;
-        protected readonly Subject<(string, float)> _progressUpdates = new Subject<(string, float)>();
-
-        public abstract ModManager ModManager { get; }
 
         public abstract AbsolutePath GamePath { get; }
+        public Dictionary<Game, HashSet<Hash>> GameHashes { get; set; } = new Dictionary<Game, HashSet<Hash>>();
+        public Dictionary<Hash, Game[]> GamesWithHashes { get; set; } = new Dictionary<Hash, Game[]>();
 
-        public abstract AbsolutePath ModListOutputFolder { get; }
+        public AbsolutePath SourcePath { get; }
-        public abstract AbsolutePath ModListOutputFile { get; }
+        public AbsolutePath DownloadsPath { get; }
+
+        public GameMetaData CompilingGame { get; set; }
+
+        public AbsolutePath ModListOutputFolder { get; }
+        public AbsolutePath ModListOutputFile { get; }
 
         public bool IgnoreMissingFiles { get; set; }
 
         public List<Archive> SelectedArchives { get; protected set; } = new List<Archive>();
         public List<Directive> InstallDirectives { get; protected set; } = new List<Directive>();
         public List<RawSourceFile> AllFiles { get; protected set; } = new List<RawSourceFile>();
-        public ModList ModList = new ModList();
 
-        public List<IndexedArchive> IndexedArchives = new List<IndexedArchive>();
+        public Dictionary<AbsolutePath, IndexedArchive> ArchivesByFullPath { get; set; } =
-        public Dictionary<AbsolutePath, IndexedArchive> ArchivesByFullPath { get; set; } = new Dictionary<AbsolutePath, IndexedArchive>();
+            new Dictionary<AbsolutePath, IndexedArchive>();
-        
-        public Dictionary<Hash, IEnumerable<VirtualFile>> IndexedFiles = new Dictionary<Hash, IEnumerable<VirtualFile>>();
-
-        public ACompiler(int steps)
-            : base(steps)
-        {
-            //set in MainWindowVM
-            WabbajackVersion = Consts.CurrentMinimumWabbajackVersion;
-        }
 
         public static void Info(string msg)
         {
@@ -100,21 +129,21 @@ namespace Wabbajack.Lib
             await ModListOutputFolder.Combine(id).WriteAllTextAsync(data);
             return id;
         }
-        
+
         internal async Task<RelativePath> IncludeFile(Stream data)
         {
             var id = IncludeId();
             await ModListOutputFolder.Combine(id).WriteAllAsync(data);
             return id;
         }
-        
+
         internal async Task<RelativePath> IncludeFile(AbsolutePath data)
         {
             await using var stream = await data.OpenRead();
             return await IncludeFile(stream);
         }
 
-        
+
         internal async Task<(RelativePath, AbsolutePath)> IncludeString(string str)
         {
             var id = IncludeId();
@@ -147,7 +176,123 @@ namespace Wabbajack.Lib
             return true;
         }
 
-        public async Task ExportModList()
+        protected async Task IndexGameFileHashes()
+        {
+            if (UseGamePaths)
+            {
+                foreach (var ag in Settings.IncludedGames.Cons(CompilingGame.Game))
+                {
+                    try
+                    {
+                        var files = await ClientAPI.GetExistingGameFiles(Queue, ag);
+                        Utils.Log($"Including {files.Length} stock game files from {ag} as download sources");
+                        GameHashes[ag] = files.Select(f => f.Hash).ToHashSet();
+
+                        IndexedArchives.AddRange(files.Select(f =>
+                        {
+                            var meta = f.State.GetMetaIniString();
+                            var ini = meta.LoadIniString();
+                            var state = (GameFileSourceDownloader.State)f.State;
+                            return new IndexedArchive(
+                                VFS.Index.ByRootPath[ag.MetaData().GameLocation().Combine(state.GameFile)])
+                            {
+                                IniData = ini, Meta = meta
+                            };
+                        }));
+                    }
+                    catch (Exception e)
+                    {
+                        Utils.Error(e, "Unable to find existing game files, skipping.");
+                    }
+                }
+
+                GamesWithHashes = GameHashes.SelectMany(g => g.Value.Select(h => (g, h)))
+                    .GroupBy(gh => gh.h)
+                    .ToDictionary(gh => gh.Key, gh => gh.Select(p => p.g.Key).ToArray());
+            }
+        }
+
+        protected async Task CleanInvalidArchivesAndFillState()
+        {
+            var remove = (await IndexedArchives.PMap(Queue, async a =>
+            {
+                try
+                {
+                    a.State = (await ResolveArchive(a)).State;
+                    return null;
+                }
+                catch
+                {
+                    return a;
+                }
+            })).NotNull().ToHashSet();
+
+            if (remove.Count == 0)
+            {
+                return;
+            }
+
+            Utils.Log(
+                $"Removing {remove.Count} archives from the compilation state, this is probably not an issue but reference this if you have compilation failures");
+            remove.Do(r => Utils.Log($"Resolution failed for: ({r.File.Size} {r.File.Hash}) {r.File.FullPath}"));
+            IndexedArchives.RemoveAll(a => remove.Contains(a));
+        }
+
+        protected async Task InferMetas()
+        {
+            async Task<bool> HasInvalidMeta(AbsolutePath filename)
+            {
+                var metaname = filename.WithExtension(Consts.MetaFileExtension);
+                if (!metaname.Exists)
+                {
+                    return true;
+                }
+
+                try
+                {
+                    return await DownloadDispatcher.ResolveArchive(metaname.LoadIniFile()) == null;
+                }
+                catch (Exception e)
+                {
+                    Utils.ErrorThrow(e, $"Exception while checking meta {filename}");
+                    return false;
+                }
+            }
+
+            var to_find = (await DownloadsPath.EnumerateFiles()
+                    .Where(f => f.Extension != Consts.MetaFileExtension && f.Extension != Consts.HashFileExtension)
+                    .PMap(Queue, async f => await HasInvalidMeta(f) ? f : default))
+                .Where(f => f.Exists)
+                .ToList();
+
+            if (to_find.Count == 0)
+            {
+                return;
+            }
+
+            Utils.Log($"Attempting to infer {to_find.Count} metas from the server.");
+
+            await to_find.PMap(Queue, async f =>
+            {
+                var vf = VFS.Index.ByRootPath[f];
+
+                var meta = await ClientAPI.InferDownloadState(vf.Hash);
+
+                if (meta == null)
+                {
+                    await vf.AbsoluteName.WithExtension(Consts.MetaFileExtension).WriteAllLinesAsync(
+                        "[General]",
+                        "unknownArchive=true");
+                    return;
+                }
+
+                Utils.Log($"Inferred .meta for {vf.FullPath.FileName}, writing to disk");
+                await vf.AbsoluteName.WithExtension(Consts.MetaFileExtension)
+                    .WriteAllTextAsync(meta.GetMetaIniString());
+            });
+        }
+
+        protected async Task ExportModList()
         {
             Utils.Log($"Exporting ModList to {ModListOutputFile}");
 
@@ -157,7 +302,7 @@ namespace Wabbajack.Lib
                 ModList.Image = (RelativePath)"modlist-image.png";
             }
 
-            await using (var of = await ModListOutputFolder.Combine("modlist").Create()) 
+            await using (var of = await ModListOutputFolder.Combine("modlist").Create())
                 ModList.ToJson(of);
 
             await ModListOutputFolder.Combine("sig")
@@ -170,16 +315,16 @@ namespace Wabbajack.Lib
             await using (var fs = await ModListOutputFile.Create())
             {
                 using var za = new ZipArchive(fs, ZipArchiveMode.Create);
-                
+
                 await ModListOutputFolder.EnumerateFiles()
                     .DoProgress("Compressing ModList",
-                async f =>
+                        async f =>
-                {
+                        {
-                    var ze = za.CreateEntry((string)f.FileName);
+                            var ze = za.CreateEntry((string)f.FileName);
-                    await using var os = ze.Open();
+                            await using var os = ze.Open();
-                    await using var ins = await f.OpenRead();
+                            await using var ins = await f.OpenRead();
-                    await ins.CopyToAsync(os);
+                            await ins.CopyToAsync(os);
-                });
+                        });
 
                 // Copy in modimage
                 if (ModListImage.Exists)
@@ -207,6 +352,114 @@ namespace Wabbajack.Lib
             await Utils.DeleteDirectory(ModListOutputFolder);
         }
 
+        /// <summary>
+        ///     Fills in the Patch fields in files that require them
+        /// </summary>
+        protected async Task BuildPatches()
+        {
+            Info("Gathering patch files");
+
+            var toBuild = InstallDirectives.OfType<PatchedFromArchive>()
+                .Where(p => p.Choices.Length > 0)
+                .SelectMany(p => p.Choices.Select(c => new PatchedFromArchive
+                {
+                    To = p.To,
+                    Hash = p.Hash,
+                    ArchiveHashPath = c.MakeRelativePaths(),
+                    FromFile = c,
+                    Size = p.Size
+                }))
+                .ToArray();
+
+            if (toBuild.Length == 0)
+            {
+                return;
+            }
+
+            // Extract all the source files
+            var indexed = toBuild.GroupBy(f => VFS.Index.FileForArchiveHashPath(f.ArchiveHashPath))
+                .ToDictionary(f => f.Key);
+            await VFS.Extract(Queue, indexed.Keys.ToHashSet(),
+                async (vf, sf) =>
+                {
+                    // For each, extract the destination
+                    var matches = indexed[vf];
+                    using var iqueue = new WorkQueue(1);
+                    foreach (var match in matches)
+                    {
+                        var destFile = FindDestFile(match.To);
+                        // Build the patch
+                        await VFS.Extract(iqueue, new[] {destFile}.ToHashSet(),
+                            async (destvf, destsfn) =>
+                            {
+                                Info($"Patching {match.To}");
+                                Status($"Patching {match.To}");
+                                await using var srcStream = await sf.GetStream();
+                                await using var destStream = await destsfn.GetStream();
+                                var patchSize =
+                                    await Utils.CreatePatchCached(srcStream, vf.Hash, destStream, destvf.Hash);
+                                Info($"Patch size {patchSize} for {match.To}");
+                            });
+                    }
+                });
+
+            // Load in the patches
+            await InstallDirectives.OfType<PatchedFromArchive>()
+                .Where(p => p.PatchID == default)
+                .PMap(Queue, async pfa =>
+                {
+                    var patches = pfa.Choices
+                        .Select(c => (Utils.TryGetPatch(c.Hash, pfa.Hash, out var data), data, c))
+                        .ToArray();
+
+                    // Pick the best patch
+                    if (patches.All(p => p.Item1))
+                    {
+                        var (_, bytes, file) = IncludePatches.PickPatch(this, patches);
+                        pfa.FromFile = file;
+                        pfa.FromHash = file.Hash;
+                        pfa.ArchiveHashPath = file.MakeRelativePaths();
+                        pfa.PatchID = await IncludeFile(bytes!);
+                    }
+                });
+
+            var firstFailedPatch =
+                InstallDirectives.OfType<PatchedFromArchive>().FirstOrDefault(f => f.PatchID == default);
+            if (firstFailedPatch != null)
+            {
+                Utils.Log("Missing data from failed patch, starting data dump");
+                Utils.Log($"Dest File: {firstFailedPatch.To}");
+                Utils.Log($"Options ({firstFailedPatch.Choices.Length}:");
+                foreach (var choice in firstFailedPatch.Choices)
+                {
+                    Utils.Log($"  {choice.FullPath}");
+                }
+
+                Error(
+                    $"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}");
+            }
+        }
+
+        private VirtualFile FindDestFile(RelativePath to)
+        {
+            var abs = to.RelativeTo(SourcePath);
+            if (abs.Exists)
+            {
+                return VFS.Index.ByRootPath[abs];
+            }
+
+            if (to.StartsWith(Consts.BSACreationDir))
+            {
+                var bsaId = (RelativePath)((string)to).Split('\\')[1];
+                var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsaId);
+                var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray());
+
+                return VFS.Index.ByRootPath[SourcePath.Combine(bsa.To)].Children.First(c => c.RelativeName == find);
+            }
+
+            throw new ArgumentException($"Couldn't load data for {to}");
+        }
+
         public void GenerateManifest()
         {
             var manifest = new Manifest(ModList);
@@ -240,7 +493,7 @@ namespace Wabbajack.Lib
 
         public async Task<Archive> ResolveArchive([NotNull] IndexedArchive archive)
         {
-            if (!string.IsNullOrWhiteSpace(archive.Name)) 
+            if (!string.IsNullOrWhiteSpace(archive.Name))
                 Utils.Status($"Checking link for {archive.Name}", alsoLog: true);
 
             if (archive.IniData == null)
@@ -264,7 +517,7 @@ namespace Wabbajack.Lib
 
             result.Meta = string.Join("\n", result.State!.GetMetaIni());
 
-            
+
             return result;
         }
 
@@ -300,6 +553,7 @@ namespace Wabbajack.Lib
                     {
                         Utils.LogStraightToFile($"     {file.To} - {file.Reason}");
                     }
+
                     if (count == max && noMatches.Count > max)
                     {
                         Utils.Log($"     ...");
@@ -325,7 +579,6 @@ namespace Wabbajack.Lib
                     file.SourceDataID = id;
                     file.SourceDataFile = null;
                 }
-
             });
         }
 
@@ -344,6 +597,7 @@ namespace Wabbajack.Lib
                     return true;
                 }
             }
+
             return false;
         }
     }

Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs

@@ -28,7 +28,7 @@ namespace Wabbajack.Lib.CompilationSteps
                     if (general.comments != null && (general.notes.Contains(Consts.WABBAJACK_INCLUDE) || general.notes.Contains(Consts.WABBAJACK_NOMATCH_INCLUDE))) return true;
                     return false;
                 })
-                .Select(kv => kv.Key.RelativeTo(_mo2Compiler.MO2Folder))
+                .Select(kv => kv.Key.RelativeTo(_mo2Compiler.SourcePath))
                 .ToList();
 
             _microstack = bsa => new List<ICompilationStep>
@@ -51,7 +51,7 @@ namespace Wabbajack.Lib.CompilationSteps
             if (!Consts.SupportedBSAs.Contains(source.Path.Extension)) return null;
 
             var defaultInclude = false;
-            if (source.Path.RelativeTo(_mo2Compiler.MO2Folder).InFolder(_mo2Compiler.MO2Folder.Combine(Consts.MO2ModFolderName)))
+            if (source.Path.RelativeTo(_mo2Compiler.SourcePath).InFolder(_mo2Compiler.SourcePath.Combine(Consts.MO2ModFolderName)))
                 if (_includeDirectly.Any(path => source.Path.StartsWith(path)))
                     defaultInclude = true;
 

Wabbajack.Lib/CompilationSteps/DirectMatch.cs

@@ -13,7 +13,7 @@ namespace Wabbajack.Lib.CompilationSteps
         {
         }
 
-        public static int GetFilePriority(MO2Compiler compiler, VirtualFile file)
+        public static int GetFilePriority(ACompiler compiler, VirtualFile file)
         {
             var archive = file.TopParent;
             var adata = compiler.ArchivesByFullPath[archive.AbsoluteName];
@@ -26,12 +26,11 @@ namespace Wabbajack.Lib.CompilationSteps
 
         public override async ValueTask<Directive?> Run(RawSourceFile source)
         {
-            var mo2Compiler = (MO2Compiler)_compiler;
             if (!_compiler.IndexedFiles.TryGetValue(source.Hash, out var found)) return null;
             var result = source.EvolveTo<FromArchive>();
 
             var match = found.Where(f => f.Name.FileName == source.Path.FileName)
-                            .OrderBy(f => GetFilePriority(mo2Compiler, f))
+                            .OrderBy(f => GetFilePriority(_compiler, f))
                             .ThenBy(f => f.NestingFactor)
                             .FirstOrDefault()
                         ?? found.OrderBy(f => f.NestingFactor).FirstOrDefault();

Wabbajack.Lib/CompilationSteps/IgnoreDisabledMods.cs

@@ -20,7 +20,7 @@ namespace Wabbajack.Lib.CompilationSteps
                 .Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_DISABLE)).Select(f => f.Key).Distinct();
 
             _allEnabledMods = _mo2Compiler.SelectedProfiles
-                .SelectMany(p => _mo2Compiler.MO2Folder.Combine("profiles", p, "modlist.txt").ReadAllLines())
+                .SelectMany(p => _mo2Compiler.SourcePath.Combine("profiles", p, "modlist.txt").ReadAllLines())
                 .Where(line => line.StartsWith("+") || line.EndsWith("_separator"))
                 .Select(line => line.Substring(1).RelativeTo(_mo2Compiler.MO2ModsFolder))
                 .Concat(alwaysEnabled)

Wabbajack.Lib/CompilationSteps/IgnoreGameFilesIfGameFolderFilesExist.cs

@@ -11,7 +11,7 @@ namespace Wabbajack.Lib.CompilationSteps
 
         public IgnoreGameFilesIfGameFolderFilesExist(ACompiler compiler) : base(compiler)
         {
-            _gameFolderFilesExists = ((MO2Compiler)compiler).MO2Folder.Combine(Consts.GameFolderFilesDir).IsDirectory;
+            _gameFolderFilesExists = ((MO2Compiler)compiler).SourcePath.Combine(Consts.GameFolderFilesDir).IsDirectory;
             _gameFolder = compiler.GamePath;
         }
 

Wabbajack.Lib/CompilationSteps/IgnoreSaveFiles.cs

@@ -11,7 +11,7 @@ namespace Wabbajack.Lib.CompilationSteps
         public IgnoreSaveFiles(ACompiler compiler) : base(compiler)
         {
             _profilePaths =
-                MO2Compiler.SelectedProfiles.Select(p => MO2Compiler.MO2Folder.Combine("profiles", p, "saves")).ToArray();
+                MO2Compiler.SelectedProfiles.Select(p => MO2Compiler.SourcePath.Combine("profiles", p, "saves")).ToArray();
         }
 
         public override async ValueTask<Directive?> Run(RawSourceFile source)

Wabbajack.Lib/CompilationSteps/IncludeOtherProfiles.cs

@@ -16,7 +16,7 @@ namespace Wabbajack.Lib.CompilationSteps
         public IgnoreOtherProfiles(ACompiler compiler) : base(compiler)
         {
             _mo2Compiler = (MO2Compiler) compiler;
-            _modProfilesFolder = _mo2Compiler.MO2Folder.Combine("profiles");
+            _modProfilesFolder = _mo2Compiler.SourcePath.Combine("profiles");
 
             _profiles = _mo2Compiler.SelectedProfiles
                 .Select(p => _modProfilesFolder.Combine(p))

Wabbajack.Lib/CompilationSteps/IncludePatches.cs

@@ -16,13 +16,13 @@ namespace Wabbajack.Lib.CompilationSteps
         private readonly Dictionary<RelativePath, IGrouping<RelativePath, VirtualFile>> _indexed;
         private VirtualFile? _bsa;
         private Dictionary<RelativePath, IEnumerable<VirtualFile>> _indexedByName;
-        private MO2Compiler _mo2Compiler;
+        private ACompiler _compiler;
         private bool _isGenericGame;
 
         public IncludePatches(ACompiler compiler, VirtualFile? constructingFromBSA = null) : base(compiler)
         {
             _bsa = constructingFromBSA;
-            _mo2Compiler = (MO2Compiler)compiler;
+            _compiler = compiler;
             _indexed = _compiler.IndexedFiles.Values
                 .SelectMany(f => f)
                 .GroupBy(f => f.Name.FileName)
@@ -33,7 +33,7 @@ namespace Wabbajack.Lib.CompilationSteps
                                      .GroupBy(f => f.Name.FileName)
                                      .ToDictionary(f => f.Key, f => (IEnumerable<VirtualFile>)f);
 
-            _isGenericGame = _mo2Compiler.CompilingGame.IsGenericMO2Plugin;
+            _isGenericGame = _compiler.CompilingGame.IsGenericMO2Plugin;
         }
 
         public override async ValueTask<Directive?> Run(RawSourceFile source)
@@ -53,13 +53,16 @@ namespace Wabbajack.Lib.CompilationSteps
                 _indexed.TryGetValue(nameWithoutExt, out choices);
 
             dynamic? modIni = null;
-            
+
-            if (_bsa == null && source.File.IsNative && source.AbsolutePath.InFolder(_mo2Compiler.MO2ModsFolder))
+            if (_compiler is MO2Compiler)
-                ((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(source.AbsolutePath), out modIni);
-            else if (_bsa != null)
             {
-                var bsaPath = _bsa.FullPath.Base;
+                if (_bsa == null && source.File.IsNative && source.AbsolutePath.InFolder(((MO2Compiler)_compiler).MO2ModsFolder))
-                ((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(bsaPath), out modIni);
+                    ((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(source.AbsolutePath), out modIni);
+                else if (_bsa != null)
+                {
+                    var bsaPath = _bsa.FullPath.Base;
+                    ((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(bsaPath), out modIni);
+                }
             }
 
             var installationFile = (string?)modIni?.General?.installationFile;
@@ -105,7 +108,7 @@ namespace Wabbajack.Lib.CompilationSteps
 
             if (patches.All(p => p.Item1))
             {
-                var (_, bytes, file) = PickPatch(_mo2Compiler, patches);
+                var (_, bytes, file) = PickPatch(_compiler, patches);
                 e.FromHash = file.Hash;
                 e.ArchiveHashPath = file.MakeRelativePaths();
                 e.PatchID = await _compiler.IncludeFile(bytes!);
@@ -126,7 +129,7 @@ namespace Wabbajack.Lib.CompilationSteps
             return e;
         }
 
-        public static (bool, byte[], VirtualFile) PickPatch(MO2Compiler mo2Compiler, IEnumerable<(bool foundHash, byte[]? data, VirtualFile file)> patches)
+        public static (bool, byte[], VirtualFile) PickPatch(ACompiler compiler, IEnumerable<(bool foundHash, byte[]? data, VirtualFile file)> patches)
         {
             var ordered = patches
                 .Select(f => (f.foundHash, f.data!, f.file))
@@ -138,11 +141,11 @@ namespace Wabbajack.Lib.CompilationSteps
                 var baseHash = itm.file.TopParent.Hash;
                 
                 // If this file doesn't come from a game use it
-                if (!mo2Compiler.GamesWithHashes.TryGetValue(baseHash, out var games))
+                if (!compiler.GamesWithHashes.TryGetValue(baseHash, out var games))
                     return true;
 
                 // Otherwise skip files that are not from the primary game
-                return games.Contains(mo2Compiler.CompilingGame.Game);
+                return games.Contains(compiler.CompilingGame.Game);
             });
             
             // If we didn't find a file from an archive or the primary game, use a secondary game file.

Wabbajack.Lib/CompilationSteps/IncludeStubbedConfigfiles.cs

@@ -28,14 +28,14 @@ namespace Wabbajack.Lib.CompilationSteps
             data = data.Replace(((string)_mo2Compiler.GamePath).Replace("\\", "\\\\"), Consts.GAME_PATH_MAGIC_DOUBLE_BACK);
             data = data.Replace(((string)_mo2Compiler.GamePath).Replace("\\", "/"), Consts.GAME_PATH_MAGIC_FORWARD);
 
-            data = data.Replace((string)_mo2Compiler.MO2Folder, Consts.MO2_PATH_MAGIC_BACK);
+            data = data.Replace((string)_mo2Compiler.SourcePath, Consts.MO2_PATH_MAGIC_BACK);
-            data = data.Replace(((string)_mo2Compiler.MO2Folder).Replace("\\", "\\\\"), Consts.MO2_PATH_MAGIC_DOUBLE_BACK);
+            data = data.Replace(((string)_mo2Compiler.SourcePath).Replace("\\", "\\\\"), Consts.MO2_PATH_MAGIC_DOUBLE_BACK);
-            data = data.Replace(((string)_mo2Compiler.MO2Folder).Replace("\\", "/"), Consts.MO2_PATH_MAGIC_FORWARD);
+            data = data.Replace(((string)_mo2Compiler.SourcePath).Replace("\\", "/"), Consts.MO2_PATH_MAGIC_FORWARD);
 
-            data = data.Replace((string)_mo2Compiler.MO2DownloadsFolder, Consts.DOWNLOAD_PATH_MAGIC_BACK);
+            data = data.Replace((string)_mo2Compiler.DownloadsPath, Consts.DOWNLOAD_PATH_MAGIC_BACK);
-            data = data.Replace(((string)_mo2Compiler.MO2DownloadsFolder).Replace("\\", "\\\\"),
+            data = data.Replace(((string)_mo2Compiler.DownloadsPath).Replace("\\", "\\\\"),
                 Consts.DOWNLOAD_PATH_MAGIC_DOUBLE_BACK);
-            data = data.Replace(((string)_mo2Compiler.MO2DownloadsFolder).Replace("\\", "/"), Consts.DOWNLOAD_PATH_MAGIC_FORWARD);
+            data = data.Replace(((string)_mo2Compiler.DownloadsPath).Replace("\\", "/"), Consts.DOWNLOAD_PATH_MAGIC_FORWARD);
 
             if (data == originalData)
                 return null;

Wabbajack.Lib/MO2Compiler.cs

@@ -4,12 +4,12 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
 using Wabbajack.Lib.CompilationSteps;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.Validation;
 using Wabbajack.VirtualFileSystem;
-using Path = Alphaleonis.Win32.Filesystem.Path;
 
 namespace Wabbajack.Lib
 {
@@ -17,70 +17,43 @@ namespace Wabbajack.Lib
     {
         private AbsolutePath _mo2DownloadsFolder;
         
-        public AbsolutePath MO2Folder;
+        public MO2Compiler(AbsolutePath sourcePath, AbsolutePath downloadsPath, string mo2Profile, AbsolutePath outputFile)
+            : base(21, mo2Profile, sourcePath, downloadsPath, outputFile)
+        {
+            MO2Profile = mo2Profile;
+            MO2Ini = SourcePath.Combine("ModOrganizer.ini").LoadIniFile();
+            var mo2game = (string)MO2Ini.General.gameName;
+            CompilingGame = GameRegistry.Games.First(g => g.Value.MO2Name == mo2game).Value;
+            GamePath = CompilingGame.GameLocation();
+        }
 
-        public AbsolutePath MO2ModsFolder => MO2Folder.Combine(Consts.MO2ModFolderName);
+        public AbsolutePath MO2ModsFolder => SourcePath.Combine(Consts.MO2ModFolderName);
 
         public string MO2Profile { get; }
 
-        public override ModManager ModManager => ModManager.MO2;
-
         public override AbsolutePath GamePath { get; }
 
-        public GameMetaData CompilingGame { get; }
-     
-        public CompilerSettings Settings { get; set; }
-
-        public override AbsolutePath ModListOutputFolder => ((RelativePath)"output_folder").RelativeToEntryPoint();
-
-        public override AbsolutePath ModListOutputFile { get; }
-
-        public override AbsolutePath VFSCacheName => 
-            Consts.LocalAppDataPath.Combine( 
-            $"vfs_compile_cache-2-{Path.Combine((string)MO2Folder ?? "Unknown", "ModOrganizer.exe").StringSha256Hex()}.bin");
-
         public dynamic MO2Ini { get; }
 
-        public static AbsolutePath GetTypicalDownloadsFolder(AbsolutePath mo2Folder) => mo2Folder.Combine("downloads");
+        public AbsolutePath MO2ProfileDir => SourcePath.Combine("profiles", MO2Profile);
-
-        public AbsolutePath MO2ProfileDir => MO2Folder.Combine("profiles", MO2Profile);
 
         public ConcurrentBag<Directive> ExtraFiles { get; private set; } = new ConcurrentBag<Directive>();
         public Dictionary<AbsolutePath, dynamic> ModInis { get; } = new Dictionary<AbsolutePath, dynamic>();
 
         public HashSet<string> SelectedProfiles { get; set; } = new HashSet<string>();
 
-        public MO2Compiler(AbsolutePath mo2Folder, string mo2Profile, AbsolutePath outputFile)
+        public static AbsolutePath GetTypicalDownloadsFolder(AbsolutePath mo2Folder)
-            : base(steps: 21)
         {
-            MO2Folder = mo2Folder;
+            return mo2Folder.Combine("downloads");
-            MO2Profile = mo2Profile;
-            MO2Ini = MO2Folder.Combine("ModOrganizer.ini").LoadIniFile();
-            var mo2game = (string)MO2Ini.General.gameName;
-            CompilingGame = GameRegistry.Games.First(g => g.Value.MO2Name == mo2game).Value;
-            GamePath = CompilingGame.GameLocation();
-            ModListOutputFile = outputFile;
-            Settings = new CompilerSettings();
-        }
-
-        public AbsolutePath MO2DownloadsFolder
-        {
-            get
-            {
-                if (_mo2DownloadsFolder != default) return _mo2DownloadsFolder;
-                if (MO2Ini != null)
-                    if (MO2Ini.Settings != null)
-                        if (MO2Ini.Settings.download_directory != null)
-                            return MO2Ini.Settings.download_directory.Replace("/", "\\");
-                return GetTypicalDownloadsFolder(MO2Folder);
-            }
-            set => _mo2DownloadsFolder = value;
         }
 
         protected override async Task<bool> _Begin(CancellationToken cancel)
         {
             await Metrics.Send("begin_compiling", MO2Profile ?? "unknown");
-            if (cancel.IsCancellationRequested) return false;
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
 
             DesiredThreads.OnNext(DiskThreads);
             FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;
@@ -88,31 +61,36 @@ namespace Wabbajack.Lib
             UpdateTracker.Reset();
             UpdateTracker.NextStep("Gathering information");
 
-            Utils.Log($"Loading compiler Settings");
+            Utils.Log("Loading compiler Settings");
             Settings = await CompilerSettings.Load(MO2ProfileDir);
             Settings.IncludedGames = Settings.IncludedGames.Add(CompilingGame.Game);
-            
+
             Info("Looking for other profiles");
             var otherProfilesPath = MO2ProfileDir.Combine("otherprofiles.txt");
             SelectedProfiles = new HashSet<string>();
-            if (otherProfilesPath.Exists) SelectedProfiles = (await otherProfilesPath.ReadAllLinesAsync()).ToHashSet();
+            if (otherProfilesPath.Exists)
+            {
+                SelectedProfiles = (await otherProfilesPath.ReadAllLinesAsync()).ToHashSet();
+            }
+
             SelectedProfiles.Add(MO2Profile!);
 
             Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p)));
 
-            Utils.Log($"Compiling Game: {CompilingGame}");
+            Utils.Log($"Compiling Game: {CompilingGame.Game}");
-            Utils.Log($"Games from setting files:");
+            Utils.Log("Games from setting files:");
             foreach (var game in Settings.IncludedGames)
             {
                 Utils.Log($"- {game}");
             }
 
             Utils.Log($"VFS File Location: {VFSCacheName}");
-            Utils.Log($"MO2 Folder: {MO2Folder}");
+            Utils.Log($"MO2 Folder: {SourcePath}");
-            Utils.Log($"Downloads Folder: {MO2DownloadsFolder}");
+            Utils.Log($"Downloads Folder: {DownloadsPath}");
             Utils.Log($"Game Folder: {GamePath}");
-            
+
-            var watcher = new DiskSpaceWatcher(cancel, new []{MO2Folder, MO2DownloadsFolder, GamePath, AbsolutePath.EntryPoint}, (long)2 << 31,
+            var watcher = new DiskSpaceWatcher(cancel,
+                new[] {SourcePath, DownloadsPath, GamePath, AbsolutePath.EntryPoint}, (long)2 << 31,
                 drive =>
                 {
                     Utils.Log($"Aborting due to low space on {drive.Name}");
@@ -120,40 +98,42 @@ namespace Wabbajack.Lib
                 });
             var watcherTask = watcher.Start();
 
-            if (cancel.IsCancellationRequested) return false;
+            if (cancel.IsCancellationRequested)
-            
+            {
+                return false;
+            }
+
             List<AbsolutePath> roots;
             if (UseGamePaths)
             {
-                roots = new List<AbsolutePath>
+                roots = new List<AbsolutePath> {SourcePath, GamePath, DownloadsPath};
-                {
-                    MO2Folder, GamePath, MO2DownloadsFolder
-                };
                 roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation()));
             }
             else
             {
-                roots = new List<AbsolutePath>
+                roots = new List<AbsolutePath> {SourcePath, DownloadsPath};
-                {
-                    MO2Folder, MO2DownloadsFolder
-                };
-                
             }
 
             // TODO: make this generic so we can add more paths
 
-            var lootPath = (AbsolutePath)Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
+            var lootPath = (AbsolutePath)Path.Combine(
+                Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
                 "LOOT");
             IEnumerable<RawSourceFile> lootFiles = new List<RawSourceFile>();
             if (lootPath.Exists)
             {
-                roots.Add((AbsolutePath)lootPath);
+                roots.Add(lootPath);
             }
+
             UpdateTracker.NextStep("Indexing folders");
 
-            if (cancel.IsCancellationRequested) return false;
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
             await VFS.AddRoots(roots);
-            
+
             if (lootPath.Exists)
             {
                 if (CompilingGame.MO2Name == null)
@@ -161,7 +141,7 @@ namespace Wabbajack.Lib
                     throw new ArgumentException("Compiling game had no MO2 name specified.");
                 }
 
-                var lootGameDirs = new []
+                var lootGameDirs = new[]
                 {
                     CompilingGame.MO2Name, // most of the games use the MO2 name
                     CompilingGame.MO2Name.Replace(" ", "") //eg: Fallout 4 -> Fallout4
@@ -180,70 +160,58 @@ namespace Wabbajack.Lib
                             Consts.LOOTFolderFilesDir.Combine(p.RelativeTo(lootPath))));
 
                     if (!lootFiles.Any())
+                    {
                         Utils.Log(
                             $"Found no LOOT user data for {CompilingGame.HumanFriendlyGameName} at {lootGameDir}!");
+                    }
                 }
             }
-            
+
-            if (cancel.IsCancellationRequested) return false;
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
             UpdateTracker.NextStep("Cleaning output folder");
             await ModListOutputFolder.DeleteDirectory();
-            
+
-            if (cancel.IsCancellationRequested) return false;
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
             UpdateTracker.NextStep("Inferring metas for game file downloads");
             await InferMetas();
 
-            if (cancel.IsCancellationRequested) return false;
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
             UpdateTracker.NextStep("Reindexing downloads after meta inferring");
-            await VFS.AddRoot(MO2DownloadsFolder);
+            await VFS.AddRoot(DownloadsPath);
-            
+
-            if (cancel.IsCancellationRequested) return false;
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
             UpdateTracker.NextStep("Pre-validating Archives");
-            
+
 
             // Find all Downloads
-            IndexedArchives = (await MO2DownloadsFolder.EnumerateFiles()
+            IndexedArchives = (await DownloadsPath.EnumerateFiles()
                 .Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
-                .PMap(Queue, async f => new IndexedArchive(VFS.Index.ByRootPath[f])
+                .PMap(Queue,
-                {
+                    async f => new IndexedArchive(VFS.Index.ByRootPath[f])
-                    Name = (string)f.FileName,
-                    IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(),
-                    Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync()
-                })).ToList();
-
-
-            if (UseGamePaths)
-            {
-                foreach (var ag in Settings.IncludedGames)
-                {
-                    try
                     {
-                        var files = await ClientAPI.GetExistingGameFiles(Queue, ag);
+                        Name = (string)f.FileName,
-                        Utils.Log($"Including {files.Length} stock game files from {ag} as download sources");
+                        IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(),
-                        GameHashes[ag] = files.Select(f => f.Hash).ToHashSet();
+                        Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync()
+                    })).ToList();
 
-                        IndexedArchives.AddRange(files.Select(f =>
-                        {
-                            var meta = f.State.GetMetaIniString();
-                            var ini = meta.LoadIniString();
-                            var state = (GameFileSourceDownloader.State)f.State;
-                            return new IndexedArchive(
-                                VFS.Index.ByRootPath[ag.MetaData().GameLocation().Combine(state.GameFile)])
-                            {
-                                IniData = ini, Meta = meta,
-                            };
-                        }));
-                    }
-                    catch (Exception e)
-                    {
-                        Utils.Error(e, "Unable to find existing game files, skipping.");
-                    }
-                }
 
-                GamesWithHashes = GameHashes.SelectMany(g => g.Value.Select(h => (g, h)))
+            await IndexGameFileHashes();
-                    .GroupBy(gh => gh.h)
-                    .ToDictionary(gh => gh.Key, gh => gh.Select(p => p.g.Key).ToArray());
-            }
 
             IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();
 
@@ -252,14 +220,16 @@ namespace Wabbajack.Lib
             UpdateTracker.NextStep("Finding Install Files");
             ModListOutputFolder.CreateDirectory();
 
-            var mo2Files = MO2Folder.EnumerateFiles()
+            var mo2Files = SourcePath.EnumerateFiles()
                 .Where(p => p.IsFile)
                 .Select(p =>
                 {
                     if (!VFS.Index.ByRootPath.ContainsKey(p))
+                    {
                         Utils.Log($"WELL THERE'S YOUR PROBLEM: {p} {VFS.Index.ByRootPath.Count}");
-                    
+                    }
-                    return new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(MO2Folder));
+
+                    return new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(SourcePath));
                 });
 
             // If Game Folder Files exists, ignore the game folder
@@ -274,14 +244,19 @@ namespace Wabbajack.Lib
 
             Info($"Found {AllFiles.Count} files to build into mod list");
 
-            if (cancel.IsCancellationRequested) return false;
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
             UpdateTracker.NextStep("Verifying destinations");
 
             var dups = AllFiles.GroupBy(f => f.Path)
                 .Where(fs => fs.Count() > 1)
                 .Select(fs =>
                 {
-                    Utils.Log($"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}");
+                    Utils.Log(
+                        $"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}");
                     return fs;
                 }).ToList();
 
@@ -290,10 +265,14 @@ namespace Wabbajack.Lib
                 Error($"Found {dups.Count} duplicates, exiting");
             }
 
-            if (cancel.IsCancellationRequested) return false;
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
             UpdateTracker.NextStep("Loading INIs");
 
-            ModInis.SetTo(MO2Folder.Combine(Consts.MO2ModFolderName)
+            ModInis.SetTo(SourcePath.Combine(Consts.MO2ModFolderName)
                 .EnumerateDirectories()
                 .Select(f =>
                 {
@@ -306,19 +285,30 @@ namespace Wabbajack.Lib
 
             ArchivesByFullPath = IndexedArchives.ToDictionary(a => a.File.AbsoluteName);
 
-            if (cancel.IsCancellationRequested) return false;
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
             var stack = MakeStack();
             UpdateTracker.NextStep("Running Compilation Stack");
             var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f));
 
             // Add the extra files that were generated by the stack
-            if (cancel.IsCancellationRequested) return false;
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
             UpdateTracker.NextStep($"Adding {ExtraFiles.Count} that were generated by the stack");
             results = results.Concat(ExtraFiles).ToArray();
 
             var noMatch = results.OfType<NoMatch>().ToArray();
             PrintNoMatches(noMatch);
-            if (CheckForNoMatchExit(noMatch)) return false;
+            if (CheckForNoMatchExit(noMatch))
+            {
+                return false;
+            }
 
             foreach (var ignored in results.OfType<IgnoredDirectly>())
             {
@@ -334,10 +324,10 @@ namespace Wabbajack.Lib
 
             UpdateTracker.NextStep("Building Patches");
             await BuildPatches();
-            
+
             UpdateTracker.NextStep("Gathering Archives");
             await GatherArchives();
-            
+
             UpdateTracker.NextStep("Including Archive Metadata");
             await IncludeArchiveMetadata();
 
@@ -357,10 +347,10 @@ namespace Wabbajack.Lib
                 Readme = ModlistReadme ?? "",
                 Image = ModListImage != default ? ModListImage.FileName : default,
                 Website = !string.IsNullOrWhiteSpace(ModListWebsite) ? new Uri(ModListWebsite) : null,
-                Version = ModlistVersion ?? new Version(1,0,0,0),
+                Version = ModlistVersion ?? new Version(1, 0, 0, 0),
                 IsNSFW = ModlistIsNSFW
             };
-            
+
             UpdateTracker.NextStep("Including required files");
             await InlineFiles();
 
@@ -381,89 +371,17 @@ namespace Wabbajack.Lib
             return true;
         }
 
-        public Dictionary<Game, HashSet<Hash>> GameHashes { get; set; } = new Dictionary<Game, HashSet<Hash>>();
-        public Dictionary<Hash, Game[]> GamesWithHashes { get; set; } = new Dictionary<Hash, Game[]>();
-
-        public bool UseGamePaths { get; set; } = true;
-
-        private async Task CleanInvalidArchivesAndFillState()
-        {
-            var remove = (await IndexedArchives.PMap(Queue, async a =>
-            {
-                try
-                {
-                    a.State = (await ResolveArchive(a)).State;
-                    return null;
-                }
-                catch
-                {
-                    return a;
-                }
-            })).NotNull().ToHashSet();
-
-            if (remove.Count == 0)
-                return;
-
-            Utils.Log(
-                $"Removing {remove.Count} archives from the compilation state, this is probably not an issue but reference this if you have compilation failures");
-            remove.Do(r => Utils.Log($"Resolution failed for: ({r.File.Size} {r.File.Hash}) {r.File.FullPath}"));
-            IndexedArchives.RemoveAll(a => remove.Contains(a));
-        }
-
-        private async Task InferMetas()
-        {
-            async Task<bool> HasInvalidMeta(AbsolutePath filename)
-            {
-                var metaname = filename.WithExtension(Consts.MetaFileExtension);
-                if (!metaname.Exists) return true;
-                try
-                {
-                    return await DownloadDispatcher.ResolveArchive(metaname.LoadIniFile()) == null;
-                }
-                catch (Exception e)
-                {
-                    Utils.ErrorThrow(e, $"Exception while checking meta {filename}");
-                    return false;
-                }
-            }
-
-            var to_find = (await MO2DownloadsFolder.EnumerateFiles()
-                .Where(f => f.Extension != Consts.MetaFileExtension && f.Extension !=Consts.HashFileExtension)
-                .PMap(Queue, async f => await HasInvalidMeta(f) ? f : default))
-                .Where(f => f.Exists)
-                .ToList();
-
-            if (to_find.Count == 0) return;
-
-            Utils.Log($"Attempting to infer {to_find.Count} metas from the server.");
-
-            await to_find.PMap(Queue, async f =>
-            {
-                var vf = VFS.Index.ByRootPath[f];
-
-                var meta = await ClientAPI.InferDownloadState(vf.Hash);
-
-                if (meta == null)
-                {
-                    await vf.AbsoluteName.WithExtension(Consts.MetaFileExtension).WriteAllLinesAsync(
-                        "[General]", 
-                        "unknownArchive=true");
-                    return;
-                }
-
-                Utils.Log($"Inferred .meta for {vf.FullPath.FileName}, writing to disk");
-                await vf.AbsoluteName.WithExtension(Consts.MetaFileExtension).WriteAllTextAsync(meta.GetMetaIniString());
-            });
-        }
-
         private async Task IncludeArchiveMetadata()
         {
             Utils.Log($"Including {SelectedArchives.Count} .meta files for downloads");
             await SelectedArchives.PMap(Queue, async a =>
             {
-                if (a.State is GameFileSourceDownloader.State) return;
+                if (a.State is GameFileSourceDownloader.State)
-                
+                {
-                var source = MO2DownloadsFolder.Combine(a.Name + Consts.MetaFileExtension);
+                    return;
+                }
+
+                var source = DownloadsPath.Combine(a.Name + Consts.MetaFileExtension);
                 var ini = a.State.GetMetaIniString();
                 var (id, fullPath) = await IncludeString(ini);
                 InstallDirectives.Add(new ArchiveMeta
@@ -487,111 +405,9 @@ namespace Wabbajack.Lib
             ExtraFiles = new ConcurrentBag<Directive>();
         }
 
-        /// <summary>
-        ///     Fills in the Patch fields in files that require them
-        /// </summary>
-        private async Task BuildPatches()
-        {
-            Info("Gathering patch files");
-
-            var toBuild = InstallDirectives.OfType<PatchedFromArchive>()
-                .Where(p => p.Choices.Length > 0)
-                .SelectMany(p => p.Choices.Select(c => new PatchedFromArchive
-                    {
-                        To = p.To,
-                        Hash = p.Hash,
-                        ArchiveHashPath = c.MakeRelativePaths(),
-                        FromFile = c,
-                        Size = p.Size,
-                    }))
-                .ToArray();
-
-            if (toBuild.Length == 0) return;
-
-            // Extract all the source files
-            var indexed = toBuild.GroupBy(f => (VFS.Index.FileForArchiveHashPath(f.ArchiveHashPath)))
-                .ToDictionary(f => f.Key);
-            await VFS.Extract(Queue, indexed.Keys.ToHashSet(), 
-                async (vf, sf) =>
-            {
-                // For each, extract the destination
-                var matches = indexed[vf];
-                using var iqueue = new WorkQueue(1);
-                foreach (var match in matches)
-                {
-                    var destFile = FindDestFile(match.To);
-                    // Build the patch
-                    await VFS.Extract(iqueue, new[] {destFile}.ToHashSet(), 
-                        async (destvf, destsfn) =>
-                    {
-                        Info($"Patching {match.To}");
-                        Status($"Patching {match.To}");
-                        await using var srcStream = await sf.GetStream();
-                        await using var destStream = await destsfn.GetStream();
-                        var patchSize = await Utils.CreatePatchCached(srcStream, vf.Hash, destStream, destvf.Hash);
-                        Info($"Patch size {patchSize} for {match.To}");
-                    });
-                }
-
-            });
- 
-            // Load in the patches
-            await InstallDirectives.OfType<PatchedFromArchive>()
-                .Where(p => p.PatchID == default)
-                .PMap(Queue, async pfa =>
-                {
-                    var patches = pfa.Choices
-                        .Select(c => (Utils.TryGetPatch(c.Hash, pfa.Hash, out var data), data, c))
-                        .ToArray();
-
-                    // Pick the best patch
-                    if (patches.All(p => p.Item1))
-                    {
-                        var (_, bytes, file) = IncludePatches.PickPatch(this, patches);
-                        pfa.FromFile = file;
-                        pfa.FromHash = file.Hash;
-                        pfa.ArchiveHashPath = file.MakeRelativePaths();
-                        pfa.PatchID = await IncludeFile(bytes!);
-                    }
-                });
-
-            var firstFailedPatch = InstallDirectives.OfType<PatchedFromArchive>().FirstOrDefault(f => f.PatchID == default);
-            if (firstFailedPatch != null)
-            {
-                Utils.Log($"Missing data from failed patch, starting data dump");
-                Utils.Log($"Dest File: {firstFailedPatch.To}");
-                Utils.Log($"Options ({firstFailedPatch.Choices.Length}:");
-                foreach (var choice in firstFailedPatch.Choices)
-                {
-                    Utils.Log($"  {choice.FullPath}");
-                }
-                Error(
-                    $"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}");
-            }
-        }
-
-        private VirtualFile FindDestFile(RelativePath to)
-        {
-            var abs = to.RelativeTo(MO2Folder);
-            if (abs.Exists)
-                return VFS.Index.ByRootPath[abs];
-
-            if (to.StartsWith(Consts.BSACreationDir))
-            {
-                var bsaId = (RelativePath)((string)to).Split('\\')[1];
-                var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsaId);
-                var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray());
-
-                return VFS.Index.ByRootPath[MO2Folder.Combine(bsa.To)].Children.First(c => c.RelativeName == find);
-            }
-
-            throw new ArgumentException($"Couldn't load data for {to}");
-        }
-
         public override IEnumerable<ICompilationStep> GetStack()
         {
             return MakeStack();
-
         }
 
         /// <summary>
@@ -609,12 +425,12 @@ namespace Wabbajack.Lib
                 new IncludePropertyFiles(this),
                 //new IncludeSteamWorkshopItems(this),
                 new IgnoreSaveFiles(this),
-                new IgnoreStartsWith(this,"logs\\"),
+                new IgnoreStartsWith(this, "logs\\"),
                 new IgnoreStartsWith(this, "downloads\\"),
-                new IgnoreStartsWith(this,"webcache\\"),
+                new IgnoreStartsWith(this, "webcache\\"),
                 new IgnoreStartsWith(this, "overwrite\\"),
                 new IgnoreStartsWith(this, "crashDumps\\"),
-                new IgnorePathContains(this,"temporary_logs"),
+                new IgnorePathContains(this, "temporary_logs"),
                 new IgnorePathContains(this, "GPUCache"),
                 new IgnorePathContains(this, "SSEEdit Cache"),
                 new IgnoreOtherProfiles(this),
@@ -625,7 +441,7 @@ namespace Wabbajack.Lib
                 new IncludeLootFiles(this),
                 new IgnoreStartsWith(this, Path.Combine((string)Consts.GameFolderFilesDir, "Data")),
                 new IgnoreStartsWith(this, Path.Combine((string)Consts.GameFolderFilesDir, "Papyrus Compiler")),
-                new IgnoreStartsWith(this, Path.Combine((string)Consts.GameFolderFilesDir, "Skyrim")),                
+                new IgnoreStartsWith(this, Path.Combine((string)Consts.GameFolderFilesDir, "Skyrim")),
                 new IgnoreRegex(this, Consts.GameFolderFilesDir + "\\\\.*\\.bsa"),
                 new IncludeRegex(this, "^[^\\\\]*\\.bat$"),
                 new IncludeModIniData(this),
@@ -633,7 +449,8 @@ namespace Wabbajack.Lib
                 new IncludeTaggedMods(this, Consts.WABBAJACK_INCLUDE),
                 new IgnoreEndsWith(this, ".pyc"),
                 new IgnoreEndsWith(this, ".log"),
-                new DeconstructBSAs(this), // Deconstruct BSAs before building patches so we don't generate massive patch files
+                new DeconstructBSAs(
+                    this), // Deconstruct BSAs before building patches so we don't generate massive patch files
                 new IncludePatches(this),
                 new IncludeDummyESPs(this),
 
@@ -647,12 +464,11 @@ namespace Wabbajack.Lib
                 // Theme file MO2 downloads somehow
                 new IgnoreEndsWith(this, "splash.png"),
                 // File to force MO2 into portable mode
-                new IgnoreEndsWith(this, "portable.txt"), 
+                new IgnoreEndsWith(this, "portable.txt"),
                 new IgnoreEndsWith(this, ".bin"),
                 new IgnoreEndsWith(this, ".refcache"),
                 //Include custom categories  
                 new IncludeRegex(this, "categories.dat$"),
-
                 new IgnoreWabbajackInstallCruft(this),
 
                 //new PatchStockESMs(this),
@@ -660,7 +476,6 @@ namespace Wabbajack.Lib
                 new IncludeAllConfigs(this),
                 new zEditIntegration.IncludeZEditPatches(this),
                 new IncludeTaggedMods(this, Consts.WABBAJACK_NOMATCH_INCLUDE),
-
                 new DropAll(this)
             };
         }

Wabbajack.Lib/NativeCompiler.cs

@@ -0,0 +1,289 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Lib.CompilationSteps;
+using Wabbajack.Lib.Validation;
+using Wabbajack.VirtualFileSystem;
+
+namespace Wabbajack.Lib
+{
+    public class NativeCompiler : ACompiler
+    {
+        public NativeCompiler(NativeCompilerSettings settings, AbsolutePath sourcePath, AbsolutePath downloadsPath, AbsolutePath outputModListPath) 
+            : base(3, settings.ModListName, sourcePath, downloadsPath, outputModListPath)
+        {
+            CompilingGame = settings.CompilingGame.MetaData();
+            GamePath = CompilingGame.GameLocation();
+            NativeSettings = settings;
+        }
+
+        public NativeCompilerSettings NativeSettings { get; set; }
+
+        protected override async Task<bool> _Begin(CancellationToken cancel)
+        {
+            await Metrics.Send("begin_compiling", ModListName ?? "unknown");
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
+            DesiredThreads.OnNext(DiskThreads);
+            FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;
+
+            UpdateTracker.Reset();
+            UpdateTracker.NextStep("Gathering information");
+
+            Utils.Log($"Compiling Game: {CompilingGame.Game}");
+            Utils.Log("Games from setting files:");
+            foreach (var game in Settings.IncludedGames)
+            {
+                Utils.Log($"- {game}");
+            }
+
+            Utils.Log($"VFS File Location: {VFSCacheName}");
+            Utils.Log($"MO2 Folder: {SourcePath}");
+            Utils.Log($"Downloads Folder: {DownloadsPath}");
+            Utils.Log($"Game Folder: {GamePath}");
+
+            var watcher = new DiskSpaceWatcher(cancel,
+                new[] {SourcePath, DownloadsPath, GamePath, AbsolutePath.EntryPoint}, (long)2 << 31,
+                drive =>
+                {
+                    Utils.Log($"Aborting due to low space on {drive.Name}");
+                    Abort();
+                });
+            var watcherTask = watcher.Start();
+
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
+            List<AbsolutePath> roots = new List<AbsolutePath> {SourcePath, GamePath, DownloadsPath};
+            roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation()));
+
+            UpdateTracker.NextStep("Indexing folders");
+
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
+            await VFS.AddRoots(roots);
+
+            UpdateTracker.NextStep("Cleaning output folder");
+            await ModListOutputFolder.DeleteDirectory();
+
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
+            UpdateTracker.NextStep("Inferring metas for game file downloads");
+            await InferMetas();
+
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
+            UpdateTracker.NextStep("Reindexing downloads after meta inferring");
+            await VFS.AddRoot(DownloadsPath);
+
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
+            UpdateTracker.NextStep("Pre-validating Archives");
+
+
+            // Find all Downloads
+            IndexedArchives = (await DownloadsPath.EnumerateFiles()
+                .Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
+                .PMap(Queue,
+                    async f => new IndexedArchive(VFS.Index.ByRootPath[f])
+                    {
+                        Name = (string)f.FileName,
+                        IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(),
+                        Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync()
+                    })).ToList();
+
+
+            await IndexGameFileHashes();
+
+            IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();
+
+            await CleanInvalidArchivesAndFillState();
+
+            UpdateTracker.NextStep("Finding Install Files");
+            ModListOutputFolder.CreateDirectory();
+
+            var mo2Files = SourcePath.EnumerateFiles()
+                .Where(p => p.IsFile)
+                .Select(p =>
+                {
+                    if (!VFS.Index.ByRootPath.ContainsKey(p))
+                    {
+                        Utils.Log($"WELL THERE'S YOUR PROBLEM: {p} {VFS.Index.ByRootPath.Count}");
+                    }
+
+                    return new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(SourcePath));
+                });
+
+            // If Game Folder Files exists, ignore the game folder
+            IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
+                .OrderBy(f => f.NestingFactor)
+                .GroupBy(f => f.Hash)
+                .ToDictionary(f => f.Key, f => f.AsEnumerable());
+            
+            AllFiles.SetTo(mo2Files
+                .DistinctBy(f => f.Path));
+
+            Info($"Found {AllFiles.Count} files to build into mod list");
+
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
+            UpdateTracker.NextStep("Verifying destinations");
+
+            var dups = AllFiles.GroupBy(f => f.Path)
+                .Where(fs => fs.Count() > 1)
+                .Select(fs =>
+                {
+                    Utils.Log(
+                        $"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}");
+                    return fs;
+                }).ToList();
+
+            if (dups.Count > 0)
+            {
+                Error($"Found {dups.Count} duplicates, exiting");
+            }
+
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
+            UpdateTracker.NextStep("Loading INIs");
+
+            ArchivesByFullPath = IndexedArchives.ToDictionary(a => a.File.AbsoluteName);
+
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+
+            var stack = MakeStack();
+            UpdateTracker.NextStep("Running Compilation Stack");
+            var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f));
+
+            // Add the extra files that were generated by the stack
+            if (cancel.IsCancellationRequested)
+            {
+                return false;
+            }
+            var noMatch = results.OfType<NoMatch>().ToArray();
+            PrintNoMatches(noMatch);
+            if (CheckForNoMatchExit(noMatch))
+            {
+                return false;
+            }
+
+            foreach (var ignored in results.OfType<IgnoredDirectly>())
+            {
+                Utils.Log($"Ignored {ignored.To} because {ignored.Reason}");
+            }
+
+            InstallDirectives.SetTo(results.Where(i => !(i is IgnoredDirectly)));
+
+            Info("Getting Nexus api_key, please click authorize if a browser window appears");
+
+            UpdateTracker.NextStep("Building Patches");
+            await BuildPatches();
+
+            UpdateTracker.NextStep("Gathering Archives");
+            await GatherArchives();
+
+            UpdateTracker.NextStep("Gathering Metadata");
+            await GatherMetaData();
+
+            ModList = new ModList
+            {
+                GameType = CompilingGame.Game,
+                WabbajackVersion = Consts.CurrentMinimumWabbajackVersion,
+                Archives = SelectedArchives.ToList(),
+                ModManager = ModManager.MO2,
+                Directives = InstallDirectives,
+                Name = ModListName ?? "untitled",
+                Author = ModListAuthor ?? "",
+                Description = ModListDescription ?? "",
+                Readme = ModlistReadme ?? "",
+                Image = ModListImage != default ? ModListImage.FileName : default,
+                Website = !string.IsNullOrWhiteSpace(ModListWebsite) ? new Uri(ModListWebsite) : null,
+                Version = ModlistVersion ?? new Version(1, 0, 0, 0),
+                IsNSFW = ModlistIsNSFW
+            };
+
+            UpdateTracker.NextStep("Including required files");
+            await InlineFiles();
+
+            UpdateTracker.NextStep("Running Validation");
+
+            await ValidateModlist.RunValidation(ModList);
+            UpdateTracker.NextStep("Generating Report");
+
+            GenerateManifest();
+
+            UpdateTracker.NextStep("Exporting Modlist");
+            await ExportModList();
+
+            ResetMembers();
+
+            UpdateTracker.NextStep("Done Building Modlist");
+
+            return true;
+        }
+        
+        /// <summary>
+        ///     Clear references to lists that hold a lot of data.
+        /// </summary>
+        private void ResetMembers()
+        {
+            AllFiles = new List<RawSourceFile>();
+            InstallDirectives = new List<Directive>();
+            SelectedArchives = new List<Archive>();
+        }
+
+        public override AbsolutePath GamePath { get; }
+        public override IEnumerable<ICompilationStep> GetStack()
+        {
+            return MakeStack();
+        }
+
+        public override IEnumerable<ICompilationStep> MakeStack()
+        {
+            List<ICompilationStep> steps = NativeSettings.CompilationSteps.Select(InterpretStep).ToList();
+            steps.Add(new DropAll(this));
+            return steps;
+        }
+
+        public ICompilationStep InterpretStep(string[] step)
+        {
+            return step[0] switch
+            {
+                "IgnoreStartsWith" => new IgnoreStartsWith(this, step[1]),
+                "IncludeConfigs" => new IncludeAllConfigs(this),
+                "IncludeDirectMatches" => new DirectMatch(this),
+                "IncludePatches" => new IncludePatches(this),
+                _ => throw new ArgumentException($"No interpretation for step {step[0]}")
+            };
+        }
+    }
+}

Wabbajack.Lib/NativeCompilerSettings.cs

@@ -0,0 +1,14 @@
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib
+{
+    public class NativeCompilerSettings : CompilerSettings
+    {
+        public Game CompilingGame { get; set; }
+
+        public string ModListName { get; set; } = "untitled";
+        
+        public string[][] CompilationSteps = new string[0][];
+
+    }
+}

Wabbajack.Lib/zEditIntegration.cs

@@ -50,21 +50,21 @@ namespace Wabbajack.Lib
                             return false;
                         }
 
-                        if (settings.managerPath != _mo2Compiler.MO2Folder)
+                        if (settings.managerPath != _mo2Compiler.SourcePath)
                         {
-                            Utils.Log($"zEdit settings file {f}: managerPath is not {_mo2Compiler.MO2Folder} but {settings.managerPath}!");
+                            Utils.Log($"zEdit settings file {f}: managerPath is not {_mo2Compiler.SourcePath} but {settings.managerPath}!");
                             return false;
                         }
 
-                        if (settings.modsPath != _mo2Compiler.MO2Folder.Combine(Consts.MO2ModFolderName))
+                        if (settings.modsPath != _mo2Compiler.SourcePath.Combine(Consts.MO2ModFolderName))
                         {
-                            Utils.Log($"zEdit settings file {f}: modsPath is not {_mo2Compiler.MO2Folder}\\{Consts.MO2ModFolderName} but {settings.modsPath}!");
+                            Utils.Log($"zEdit settings file {f}: modsPath is not {_mo2Compiler.SourcePath}\\{Consts.MO2ModFolderName} but {settings.modsPath}!");
                             return false;
                         }
 
-                        if (settings.mergePath != _mo2Compiler.MO2Folder.Combine(Consts.MO2ModFolderName))
+                        if (settings.mergePath != _mo2Compiler.SourcePath.Combine(Consts.MO2ModFolderName))
                         {
-                            Utils.Log($"zEdit settings file {f}: modsPath is not {_mo2Compiler.MO2Folder}\\{Consts.MO2ModFolderName} but {settings.modsPath}!");
+                            Utils.Log($"zEdit settings file {f}: modsPath is not {_mo2Compiler.SourcePath}\\{Consts.MO2ModFolderName} but {settings.modsPath}!");
                             return false;
                         }
 
@@ -103,7 +103,7 @@ namespace Wabbajack.Lib
 
                 _mergesIndexed =
                     merges.ToDictionary(
-                        m => _mo2Compiler.MO2Folder.Combine((string)Consts.MO2ModFolderName, m.Key.name, m.Key.filename),
+                        m => _mo2Compiler.SourcePath.Combine((string)Consts.MO2ModFolderName, m.Key.name, m.Key.filename),
                         m => m.First());
 
                 _disabled = false;
@@ -180,12 +180,12 @@ namespace Wabbajack.Lib
 
                     return new SourcePatch
                     {
-                        RelativePath = absPath.RelativeTo(_mo2Compiler.MO2Folder),
+                        RelativePath = absPath.RelativeTo(_mo2Compiler.SourcePath),
                         Hash = hash
                     };
                 }));
 
-                var srcData = (await result.Sources.SelectAsync(async f => await _mo2Compiler.MO2Folder.Combine(f.RelativePath).ReadAllBytesAsync()).ToList())
+                var srcData = (await result.Sources.SelectAsync(async f => await _mo2Compiler.SourcePath.Combine(f.RelativePath).ReadAllBytesAsync()).ToList())
                     .ConcatArrays();
 
                 var dstData = await source.AbsolutePath.ReadAllBytesAsync();

Wabbajack.Test/ACompilerTest.cs

@@ -36,7 +36,8 @@ namespace Wabbajack.Test
         protected async Task<MO2Compiler> ConfigureAndRunCompiler(string profile, bool useGameFiles= false)
         {
             var compiler = new MO2Compiler(
-                mo2Folder: utils.MO2Folder,
+                sourcePath: utils.SourcePath,
+                downloadsPath: utils.DownloadsPath,
                 mo2Profile: profile,
                 outputFile: OutputFile(profile));
             compiler.UseGamePaths = useGameFiles;
@@ -51,13 +52,35 @@ namespace Wabbajack.Test
             await Install(compiler);
             return compiler.ModList;
         }
+        
+        protected async Task<NativeCompiler> ConfigureAndRunCompiler(AbsolutePath configPath, bool useGameFiles= false)
+        {
+            var settings = configPath.FromJson<NativeCompilerSettings>();
+            var profile = utils.AddProfile();
+
+            var compiler = new NativeCompiler(
+                settings: settings,
+                sourcePath: utils.SourcePath,
+                downloadsPath: utils.DownloadsPath,
+                outputModListPath: OutputFile(profile)) 
+                {UseGamePaths = useGameFiles};
+            Assert.True(await compiler.Begin());
+            return compiler;
+        }
+        protected async Task<ModList> CompileAndInstall(AbsolutePath settingsPath, bool useGameFiles = false)
+        {
+            var compiler = await ConfigureAndRunCompiler(settingsPath, useGameFiles: useGameFiles);
+            Utils.Log("Finished Compiling");
+            await Install(compiler);
+            return compiler.ModList;
+        }
 
         private static AbsolutePath OutputFile(string profile)
         {
             return ((RelativePath)profile).RelativeToEntryPoint().WithExtension(Consts.ModListExtension);
         }
 
-        protected async Task Install(MO2Compiler compiler)
+        protected async Task Install(ACompiler compiler)
         {
             Utils.Log("Loading Modlist");
             var modlist = AInstaller.LoadFromFile(compiler.ModListOutputFile);
@@ -65,8 +88,8 @@ namespace Wabbajack.Test
             var installer = new MO2Installer(
                 archive: compiler.ModListOutputFile,
                 modList: modlist,
-                outputFolder: utils.InstallFolder,
+                outputFolder: utils.InstallPath,
-                downloadFolder: utils.DownloadsFolder,
+                downloadFolder: utils.DownloadsPath,
                 parameters: CreateDummySystemParameters());
             installer.WarnOnOverwrite = false;
             installer.GameFolder = utils.GameFolder;

Wabbajack.Test/EndToEndTests.cs

@@ -52,7 +52,7 @@ namespace Wabbajack.Test
             await DownloadAndInstall(
                 "https://github.com/ModOrganizer2/modorganizer/releases/download/v2.2.1/Mod.Organizer.2.2.1.7z",
                 "Mod.Organizer.2.2.1.7z");
-            await utils.DownloadsFolder.Combine("Mod.Organizer.2.2.1.7z.meta").WriteAllLinesAsync(
+            await utils.DownloadsPath.Combine("Mod.Organizer.2.2.1.7z.meta").WriteAllLinesAsync(
                     "[General]",
                     "directURL=https://github.com/ModOrganizer2/modorganizer/releases/download/v2.2.1/Mod.Organizer.2.2.1.7z"
                 );
@@ -75,7 +75,7 @@ namespace Wabbajack.Test
                 $"matchAll= {modfiles[2].Download.FileName}"
             );
             
-            await utils.MO2Folder.Combine("startup.bat").WriteAllLinesAsync(
+            await utils.SourcePath.Combine("startup.bat").WriteAllLinesAsync(
                 "ModOrganizer2.exe SKSE"
             );
 
@@ -83,13 +83,13 @@ namespace Wabbajack.Test
             await CompileAndInstall(profile);
             await utils.VerifyAllFiles();
 
-            await utils.InstallFolder.Combine(Consts.LOOTFolderFilesDir).DeleteDirectory();
+            await utils.InstallPath.Combine(Consts.LOOTFolderFilesDir).DeleteDirectory();
 
             var compiler = new MO2Compiler(
-                mo2Folder: utils.InstallFolder,
+                sourcePath: utils.InstallPath,
+                downloadsPath: utils.DownloadsPath,
                 mo2Profile: profile,
                 outputFile: profile.RelativeTo(AbsolutePath.EntryPoint).WithExtension(Consts.ModListExtension));
-            compiler.MO2DownloadsFolder = utils.DownloadsFolder;
             Assert.True(await compiler.Begin());
 
         }
@@ -105,12 +105,12 @@ namespace Wabbajack.Test
                 await state.Download(new Archive(state: null!) { Name = "Unknown"}, src);
             }
 
-            utils.DownloadsFolder.CreateDirectory();
+            utils.DownloadsPath.CreateDirectory();
 
-            var destFile = utils.DownloadsFolder.Combine(filename);
+            var destFile = utils.DownloadsPath.Combine(filename);
             await src.CopyToAsync(destFile);
 
-            var modFolder = modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName);
+            var modFolder = modName == null ? utils.SourcePath : utils.ModsPath.Combine(modName);
             await FileExtractor2.ExtractAll(Queue, src, modFolder);
             return (destFile, modFolder);
         }
@@ -140,12 +140,12 @@ namespace Wabbajack.Test
                 await state.Download(src);
             }
             
-            utils.DownloadsFolder.CreateDirectory();
+            utils.DownloadsPath.CreateDirectory();
 
-            var dest = utils.DownloadsFolder.Combine(file.file_name);
+            var dest = utils.DownloadsPath.Combine(file.file_name);
             await src.CopyToAsync(dest);
 
-            var modFolder = utils.ModsFolder.Combine(modName);
+            var modFolder = utils.ModsPath.Combine(modName);
             await FileExtractor2.ExtractAll(Queue, src, modFolder);
             
             await dest.WithExtension(Consts.MetaFileExtension).WriteAllTextAsync(ini);
@@ -165,8 +165,8 @@ namespace Wabbajack.Test
             var installer = new MO2Installer(
                 archive: compiler.ModListOutputFile, 
                 modList: modlist,
-                outputFolder: utils.InstallFolder,
+                outputFolder: utils.InstallPath,
-                downloadFolder: utils.DownloadsFolder,
+                downloadFolder: utils.DownloadsPath,
                 parameters: ACompilerTest.CreateDummySystemParameters())
             {
                 UseCompression = true
@@ -178,7 +178,8 @@ namespace Wabbajack.Test
         private async Task<MO2Compiler> ConfigureAndRunCompiler(string profile)
         {
             var compiler = new MO2Compiler(
-                mo2Folder: utils.MO2Folder,
+                sourcePath: utils.SourcePath,
+                downloadsPath: utils.DownloadsPath,
                 mo2Profile: profile,
                 outputFile: profile.RelativeTo(AbsolutePath.EntryPoint).WithExtension(Consts.ModListExtension));
             Assert.True(await compiler.Begin());

Wabbajack.Test/SanityTests.cs

@@ -54,7 +54,7 @@ namespace Wabbajack.Test
             await utils.AddManualDownload(
                 new Dictionary<string, byte[]> {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}});
             
-            await utils.DownloadsFolder.Combine("some_other_file.7z").WriteAllTextAsync("random data");
+            await utils.DownloadsPath.Combine("some_other_file.7z").WriteAllTextAsync("random data");
 
             await CompileAndInstall(profile);
 
@@ -90,14 +90,14 @@ namespace Wabbajack.Test
 
             await utils.Configure();
 
-            utils.MO2Folder.Combine(Consts.GameFolderFilesDir).CreateDirectory();
+            utils.SourcePath.Combine(Consts.GameFolderFilesDir).CreateDirectory();
 
             await utils.AddManualDownload(
                 new Dictionary<string, byte[]> {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}});
 
             await CompileAndInstall(profile);
 
-            Assert.False(utils.InstallFolder.Combine(Consts.GameFolderFilesDir, (RelativePath)@"enbstuff\test.pex").IsFile);
+            Assert.False(utils.InstallPath.Combine(Consts.GameFolderFilesDir, (RelativePath)@"enbstuff\test.pex").IsFile);
         }
 
         [Fact]
@@ -188,12 +188,12 @@ namespace Wabbajack.Test
             var profile = utils.AddProfile();
             var mod = await utils.AddMod("dummy");
 
-            var saveFolder = utils.MO2Folder.Combine("profiles", profile, "saves");
+            var saveFolder = utils.SourcePath.Combine("profiles", profile, "saves");
             saveFolder.CreateDirectory();
             await saveFolder.Combine("incompilation").WriteAllTextAsync("ignore this");
 
-            var installSaveFolderThisProfile = utils.InstallFolder.Combine("profiles", profile, "saves");
+            var installSaveFolderThisProfile = utils.InstallPath.Combine("profiles", profile, "saves");
-            var installSaveFolderOtherProfile = utils.InstallFolder.Combine("profiles", "Other Profile", "saves");
+            var installSaveFolderOtherProfile = utils.InstallPath.Combine("profiles", "Other Profile", "saves");
             installSaveFolderThisProfile.CreateDirectory();
             installSaveFolderOtherProfile.CreateDirectory();
 
@@ -215,7 +215,7 @@ namespace Wabbajack.Test
             var mod = await utils.AddMod("dummy");
 
             await utils.Configure();
-            await utils.MO2Folder.Combine("profiles", profile, "somegameprefs.ini").WriteAllLinesAsync(
+            await utils.SourcePath.Combine("profiles", profile, "somegameprefs.ini").WriteAllLinesAsync(
                 // Beth inis are messy, let's make ours just as messy to catch some parse failures
                 "[Display]",
                 "foo=4",
@@ -231,7 +231,7 @@ namespace Wabbajack.Test
 
             var modlist = await CompileAndInstall(profile);
 
-            var ini = utils.InstallFolder.Combine("profiles", profile, "somegameprefs.ini").LoadIniFile();
+            var ini = utils.InstallPath.Combine("profiles", profile, "somegameprefs.ini").LoadIniFile();
 
             var sysinfo = CreateDummySystemParameters();
 
@@ -511,7 +511,7 @@ namespace Wabbajack.Test
             await new CompilerSettings()
             {
                 IncludedGames = new []{Game.Morrowind}
-            }.ToJsonAsync(utils.MO2Folder.Combine("profiles", profile, CompilerSettings.FileName), true);
+            }.ToJsonAsync(utils.SourcePath.Combine("profiles", profile, CompilerSettings.FileName), true);
 
             Game.SkyrimSpecialEdition.MetaData().CanSourceFrom = new[] {Game.Morrowind, Game.Skyrim};
             
@@ -540,7 +540,7 @@ namespace Wabbajack.Test
             await utils.VerifyInstalledFile(mod, @"Data\SkyrimSE\Update.esm.old");
             await utils.VerifyInstalledFile(mod, @"Data\SkyrimSE\Update.esm");
             
-            Assert.False(utils.InstallFolder.Combine(Consts.GameFolderFilesDir).IsDirectory);
+            Assert.False(utils.InstallPath.Combine(Consts.GameFolderFilesDir).IsDirectory);
             
         }
 
@@ -554,8 +554,8 @@ namespace Wabbajack.Test
 
             await utils.Configure();
 
-            utils.MO2Folder.Combine(Consts.GameFolderFilesDir).CreateDirectory();
+            utils.SourcePath.Combine(Consts.GameFolderFilesDir).CreateDirectory();
-            await utils.MO2Folder.Combine(Consts.GameFolderFilesDir).Combine("dx4242.dll")
+            await utils.SourcePath.Combine(Consts.GameFolderFilesDir).Combine("dx4242.dll")
                 .WriteAllBytesAsync(utils.RandomData());
 
             await utils.AddManualDownload(
@@ -580,7 +580,7 @@ namespace Wabbajack.Test
             var disabledMod = await utils.AddMod();
             var disabledTestPex = await utils.AddModFile(disabledMod, @"Data\scripts\disabledTestPex.pex", 10);
 
-            await disabledMod.RelativeTo(utils.ModsFolder).Combine("meta.ini").WriteAllLinesAsync(
+            await disabledMod.RelativeTo(utils.ModsPath).Combine("meta.ini").WriteAllLinesAsync(
                 "[General]",
                 $"notes={Consts.WABBAJACK_ALWAYS_ENABLE}");
 
@@ -602,7 +602,7 @@ namespace Wabbajack.Test
             await utils.VerifyInstalledFile(enabledMod, @"Data\scripts\enabledTestPex.pex");
             await utils.VerifyInstalledFile(disabledMod, @"Data\scripts\disabledTestPex.pex");
 
-            var modlistTxt = await utils.InstallFolder.Combine("profiles", profile, "modlist.txt").ReadAllLinesAsync();
+            var modlistTxt = await utils.InstallPath.Combine("profiles", profile, "modlist.txt").ReadAllLinesAsync();
             Assert.Equal(new string[]
             {
                 $"-{disabledMod}",
@@ -610,5 +610,47 @@ namespace Wabbajack.Test
             }, modlistTxt.ToArray());
         }
 
+        [Fact]
+        public async Task CanCompileFromNativeSource()
+        {
+            utils.CreatePaths();
+
+            var gameFolder = Game.SkyrimSpecialEdition.MetaData().GameLocation();
+            await gameFolder.Combine("SkyrimSE.exe").CopyToAsync(utils.SourcePath.Combine("SkyrimSE.exe"));
+
+            var some_dds = utils.SourcePath.Combine("some_file.dds");
+            await some_dds.WriteAllBytesAsync(utils.RandomData());
+
+            var blerg = utils.SourcePath.Combine("file1.blerg");
+            await blerg.WriteAllBytesAsync(utils.RandomData());
+            
+            await utils.AddManualDownload(
+                new Dictionary<string, byte[]>
+                {
+                    {"file1.blerg", await some_dds.ReadAllBytesAsync()},
+                });
+
+            var settings = new NativeCompilerSettings
+            {
+                CompilingGame = Game.SkyrimSpecialEdition,
+                CompilationSteps = new []
+                {
+                    new []{"IgnoreStartsWith", "downloads"},
+                    new []{"IncludeConfigs"},
+                    new []{"IncludeDirectMatches"},
+                    new []{"IncludePatches"}
+                }
+            };
+
+            var settingsPath = utils.SourcePath.Combine("native_compiler_settings.json");
+            await settings.ToJsonAsync(utils.SourcePath.Combine("native_compiler_settings.json"), true);
+
+            await CompileAndInstall(settingsPath, true);
+            
+            Assert.Equal(await some_dds.FileHashAsync(), await utils.InstallPath.Combine("some_file.dds").FileHashAsync());
+            Assert.Equal(await gameFolder.Combine("SkyrimSE.exe").FileHashAsync(), 
+                await utils.InstallPath.Combine("SkyrimSE.exe").FileHashAsync());
+        }
+
     }
 }

Wabbajack.Test/TestUtils.cs

@@ -32,11 +32,11 @@ namespace Wabbajack.Test
         public AbsolutePath TestFolder => WorkingDirectory.Combine(ID);
         public AbsolutePath GameFolder => WorkingDirectory.Combine(ID, "game_folder");
 
-        public AbsolutePath MO2Folder => WorkingDirectory.Combine(ID, "mo2_folder");
+        public AbsolutePath SourcePath => WorkingDirectory.Combine(ID, "source_folder");
-        public AbsolutePath ModsFolder => MO2Folder.Combine(Consts.MO2ModFolderName);
+        public AbsolutePath ModsPath => SourcePath.Combine(Consts.MO2ModFolderName);
-        public AbsolutePath DownloadsFolder => MO2Folder.Combine("downloads");
+        public AbsolutePath DownloadsPath => SourcePath.Combine("downloads");
 
-        public AbsolutePath InstallFolder => TestFolder.Combine("installed");
+        public AbsolutePath InstallPath => TestFolder.Combine("installed");
 
         public HashSet<string> Profiles = new HashSet<string>();
 
@@ -44,20 +44,20 @@ namespace Wabbajack.Test
 
         public async Task Configure(IEnumerable<(string ModName, bool IsEnabled)> enabledMods = null)
         {
-            await MO2Folder.Combine("ModOrganizer.ini").WriteAllLinesAsync(
+            await SourcePath.Combine("ModOrganizer.ini").WriteAllLinesAsync(
                 "[General]", 
                 $"gameName={Game.MetaData().MO2Name}", 
                 $"gamePath={((string)GameFolder).Replace("\\", "\\\\")}", 
-                $"download_directory={DownloadsFolder}");
+                $"download_directory={DownloadsPath}");
 
-            DownloadsFolder.CreateDirectory();
+            DownloadsPath.CreateDirectory();
             GameFolder.Combine("Data").CreateDirectory();
 
             if (enabledMods == null)
             {
                 Profiles.Do(profile =>
                 {
-                    MO2Folder.Combine("profiles", profile, "modlist.txt").WriteAllLinesAsync(
+                    SourcePath.Combine("profiles", profile, "modlist.txt").WriteAllLinesAsync(
                         Mods.Select(s => $"+{s}").ToArray());
                 });
             }
@@ -65,7 +65,7 @@ namespace Wabbajack.Test
             {
                 Profiles.Do(profile =>
                 {
-                    MO2Folder.Combine("profiles", profile, "modlist.txt").WriteAllLinesAsync(
+                    SourcePath.Combine("profiles", profile, "modlist.txt").WriteAllLinesAsync(
                         enabledMods.Select(s => $"{(s.IsEnabled ? "+" : "-")}{s.ModName}").ToArray());
                 });
             }
@@ -74,7 +74,7 @@ namespace Wabbajack.Test
         public string AddProfile(string name = null)
         {
             string profile_name = name ?? RandomName();
-            MO2Folder.Combine("profiles", profile_name).CreateDirectory();
+            SourcePath.Combine("profiles", profile_name).CreateDirectory();
             Profiles.Add(profile_name);
             return profile_name;
         }
@@ -82,7 +82,7 @@ namespace Wabbajack.Test
         public async Task<string> AddMod(string name = null)
         {
             string mod_name = name ?? RandomName();
-            var mod_folder = MO2Folder.Combine(Consts.MO2ModFolderName, (RelativePath)mod_name);
+            var mod_folder = SourcePath.Combine(Consts.MO2ModFolderName, (RelativePath)mod_name);
             mod_folder.CreateDirectory();
             await mod_folder.Combine("meta.ini").WriteAllTextAsync("[General]");
             Mods.Add(mod_name);
@@ -99,7 +99,7 @@ namespace Wabbajack.Test
         /// <returns></returns>
         public async Task<AbsolutePath> AddModFile(string mod_name, string path, int random_fill=128)
         {
-            var full_path = ModsFolder.Combine(mod_name, path);
+            var full_path = ModsPath.Combine(mod_name, path);
             full_path.Parent.CreateDirectory();
             await GenerateRandomFileData(full_path, random_fill);
             return full_path;
@@ -161,16 +161,17 @@ namespace Wabbajack.Test
         {
             var name = RandomName() + ".zip";
 
-            await using FileStream fs = await DownloadsFolder.Combine(name).Create();
+            await using FileStream fs = await DownloadsPath.Combine(name).Create();
             using ZipArchive archive = new ZipArchive(fs, ZipArchiveMode.Create);
-            contents.Do(kv =>
+            foreach (var (key, value) in contents)
             {
-                var entry = archive.CreateEntry(kv.Key);
+                Utils.Log($"Adding {value.Length.ToFileSizeString()} entry {key}");
-                using var os = entry.Open();
+                var entry = archive.CreateEntry(key);
-                os.Write(kv.Value, 0, kv.Value.Length);
+                await using var os = entry.Open();
-            });
+                await os.WriteAsync(value, 0, value.Length);
+            }
 
-            await DownloadsFolder.Combine(name + Consts.MetaFileExtension).WriteAllLinesAsync(
+            await DownloadsPath.Combine(name + Consts.MetaFileExtension).WriteAllLinesAsync(
                     "[General]",
                     "manualURL=<TESTING>"
                 );
@@ -180,10 +181,10 @@ namespace Wabbajack.Test
 
         public async Task VerifyInstalledFile(string mod, string file)
         {
-            var src = MO2Folder.Combine((string)Consts.MO2ModFolderName, mod, file);
+            var src = SourcePath.Combine((string)Consts.MO2ModFolderName, mod, file);
             Assert.True(src.Exists);
 
-            var dest = InstallFolder.Combine((string)Consts.MO2ModFolderName, mod, file);
+            var dest = InstallPath.Combine((string)Consts.MO2ModFolderName, mod, file);
             Assert.True(dest.Exists, $"Destination {dest} doesn't exist");
 
             var srcData = await src.ReadAllBytesAsync();
@@ -203,7 +204,7 @@ namespace Wabbajack.Test
             var src = GameFolder.Combine(file);
             Assert.True(src.Exists);
 
-            var dest = InstallFolder.Combine((string)Consts.GameFolderFilesDir, file);
+            var dest = InstallPath.Combine((string)Consts.GameFolderFilesDir, file);
             Assert.True(dest.Exists);
 
             var srcData = await src.ReadAllBytesAsync();
@@ -219,7 +220,7 @@ namespace Wabbajack.Test
         }
         public AbsolutePath PathOfInstalledFile(string mod, string file)
         {
-            return InstallFolder.Combine((string)Consts.MO2ModFolderName, mod, file);
+            return InstallPath.Combine((string)Consts.MO2ModFolderName, mod, file);
         }
 
         public async ValueTask VerifyAllFiles(bool gameFileShouldNotExistInGameFolder = true)
@@ -228,32 +229,32 @@ namespace Wabbajack.Test
             {
                 foreach (var file in Game.MetaData().RequiredFiles!)
                 {
-                    Assert.False(InstallFolder.Combine(Consts.GameFolderFilesDir, (RelativePath)file).Exists);
+                    Assert.False(InstallPath.Combine(Consts.GameFolderFilesDir, (RelativePath)file).Exists);
                 }
             }
 
 
             var skipFiles = new []{"portable.txt"}.Select(e => (RelativePath)e).ToHashSet();
-            foreach (var destFile in InstallFolder.EnumerateFiles())
+            foreach (var destFile in InstallPath.EnumerateFiles())
             {
-                var relFile = destFile.RelativeTo(InstallFolder);
+                var relFile = destFile.RelativeTo(InstallPath);
-                if (destFile.InFolder(Consts.LOOTFolderFilesDir.RelativeTo(MO2Folder)) || destFile.InFolder(Consts.GameFolderFilesDir.RelativeTo(MO2Folder)))
+                if (destFile.InFolder(Consts.LOOTFolderFilesDir.RelativeTo(SourcePath)) || destFile.InFolder(Consts.GameFolderFilesDir.RelativeTo(SourcePath)))
                     continue;
                 
                 if (!skipFiles.Contains(relFile)) 
-                    Assert.True(MO2Folder.Combine(relFile).Exists, $"Only in Destination: {relFile}");
+                    Assert.True(SourcePath.Combine(relFile).Exists, $"Only in Destination: {relFile}");
             }
 
             var skipExtensions = new []{".txt", ".ini"}.Select(e => new Extension(e)).ToHashSet();
 
-            foreach (var srcFile in MO2Folder.EnumerateFiles())
+            foreach (var srcFile in SourcePath.EnumerateFiles())
             {
-                var relFile = srcFile.RelativeTo(MO2Folder);
+                var relFile = srcFile.RelativeTo(SourcePath);
 
                 if (relFile.StartsWith("downloads\\"))
                     continue;
 
-                var destFile = InstallFolder.Combine(relFile);
+                var destFile = InstallPath.Combine(relFile);
                 Assert.True(destFile.Exists, $"Only in Source: {relFile}");
 
                 if (!skipExtensions.Contains(srcFile.Extension))
@@ -271,5 +272,12 @@ namespace Wabbajack.Test
             await GenerateRandomFileData(fullPath, i);
             return fullPath;
         }
+
+        public void CreatePaths()
+        {
+            SourcePath.CreateDirectory();
+            DownloadsPath.CreateDirectory();
+            InstallPath.CreateDirectory();
+        }
     }
 }

Wabbajack.VirtualFileSystem/Context.cs

@@ -236,7 +236,7 @@ namespace Wabbajack.VirtualFileSystem
                 }
 
             }
-            await filesByParent[top].PMap(queue, async file => await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName)));
+            await filesByParent[top].PMap(queue, async file => await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName) {CanMove = false}));
         }
 
         #region KnownFiles

Wabbajack/View Models/Compilers/MO2CompilerVM.cs

@@ -10,6 +10,7 @@ using System.Threading.Tasks;
 using DynamicData;
 using Wabbajack.Common;
 using Wabbajack.Lib;
+using WebSocketSharp;
 
 namespace Wabbajack
 {
@@ -49,7 +50,7 @@ namespace Wabbajack
                 PathType = FilePickerVM.PathTypeOptions.File,
                 PromptTitle = "Select a Modlist"
             };
-            ModListLocation.Filters.Add(new CommonFileDialogFilter("MO2 Profile (modlist.txt)", ".txt"));
+            ModListLocation.Filters.Add(new CommonFileDialogFilter("MO2 Profile (modlist.txt) or Native Settings (native_compiler_settings.json)", ".txt,.json"));
 
             DownloadLocation = new FilePickerVM()
             {
@@ -63,8 +64,18 @@ namespace Wabbajack
                 {
                     try
                     {
-                        var profileFolder = loc.Parent;
+                        if (loc.FileName == Consts.ModListTxt)
-                        return profileFolder.Parent.Parent;
+                        {
+                            var profileFolder = loc.Parent;
+                            return profileFolder.Parent.Parent;
+                        }
+
+                        if (loc.FileName == Consts.NativeSettingsJson)
+                        {
+                            return loc.Parent;
+                        }
+
+                        return default;
                     }
                     catch (Exception)
                     {
@@ -77,6 +88,11 @@ namespace Wabbajack
                 {
                     try
                     {
+                        if (loc.FileName == Consts.NativeSettingsJson)
+                        {
+                            var settings = loc.FromJson<NativeCompilerSettings>();
+                            return settings.ModListName;
+                        }
                         return (string)loc.Parent.FileName;
                     }
                     catch (Exception)
@@ -179,24 +195,45 @@ namespace Wabbajack
 
             try
             {
-                using (ActiveCompilation = new MO2Compiler(
+                ACompiler compiler;
-                    mo2Folder: Mo2Folder,
+                
-                    mo2Profile: MOProfile,
+                if (ModListLocation.TargetPath.FileName == Consts.NativeSettingsJson)
-                    outputFile: outputFile)
                 {
-                    ModListName = ModlistSettings.ModListName,
+                    var settings = ModListLocation.TargetPath.FromJson<NativeCompilerSettings>();
-                    ModListAuthor = ModlistSettings.AuthorText,
+                    compiler = new NativeCompiler(settings, Mo2Folder, DownloadLocation.TargetPath, outputFile)
-                    ModListDescription = ModlistSettings.Description,
+                    {
-                    ModListImage = ModlistSettings.ImagePath.TargetPath,
+                        ModListName = ModlistSettings.ModListName,
-                    ModListWebsite = ModlistSettings.Website,
+                        ModListAuthor = ModlistSettings.AuthorText,
-                    ModlistReadme = ModlistSettings.Readme,
+                        ModListDescription = ModlistSettings.Description,
-                    MO2DownloadsFolder = DownloadLocation.TargetPath,
+                        ModListImage = ModlistSettings.ImagePath.TargetPath,
-                    ModlistVersion = ModlistSettings.Version,
+                        ModListWebsite = ModlistSettings.Website,
-                    ModlistIsNSFW = ModlistSettings.IsNSFW
+                        ModlistReadme = ModlistSettings.Readme,
-                })
+                        ModlistVersion = ModlistSettings.Version,
+                        ModlistIsNSFW = ModlistSettings.IsNSFW
+                    };
+                }
+                else
+                {
+                    compiler = new MO2Compiler(
+                        sourcePath: Mo2Folder,
+                        downloadsPath: DownloadLocation.TargetPath,
+                        mo2Profile: MOProfile,
+                        outputFile: outputFile)
+                    {
+                        ModListName = ModlistSettings.ModListName,
+                        ModListAuthor = ModlistSettings.AuthorText,
+                        ModListDescription = ModlistSettings.Description,
+                        ModListImage = ModlistSettings.ImagePath.TargetPath,
+                        ModListWebsite = ModlistSettings.Website,
+                        ModlistReadme = ModlistSettings.Readme,
+                        ModlistVersion = ModlistSettings.Version,
+                        ModlistIsNSFW = ModlistSettings.IsNSFW
+                    };
+                }
+                using (ActiveCompilation = compiler
+)
                 {
                     Parent.MWVM.Settings.Performance.SetProcessorSettings(ActiveCompilation);
-
                     var success = await ActiveCompilation.Begin();
                     return GetResponse<ModList>.Create(success, ActiveCompilation.ModList);
                 }