move compilation steps into separate files and abstract behind an interface

Wabbajack.Lib/CompilationSteps/ACompilationStep.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public abstract class ACompilationStep : ICompilationStep
+    {
+        protected Compiler _compiler;
+
+        public ACompilationStep(Compiler compiler)
+        {
+            _compiler = compiler;
+        }
+
+        public abstract Directive Run(RawSourceFile source);
+    }
+}

Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Alphaleonis.Win32.Filesystem;
+using Compression.BSA;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    class DeconstructBSAs : ACompilationStep
+    {
+        private readonly List<ICompilationStep> _microstack;
+        private readonly List<ICompilationStep> _microstackWithInclude;
+        private readonly IEnumerable<string> _include_directly;
+
+        public DeconstructBSAs(Compiler compiler) : base(compiler)
+        {
+             _include_directly = _compiler.ModInis.Where(kv =>
+                {
+                    var general = kv.Value.General;
+                    if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE)) return true;
+                    if (general.comments != null && general.comments.Contains(Consts.WABBAJACK_INCLUDE)) return true;
+                    return false;
+                })
+                .Select(kv => $"mods\\{kv.Key}\\")
+                .ToList();
+
+            _microstack = new List<ICompilationStep>
+            {
+                new DirectMatch(_compiler),
+                new IncludePatches(_compiler),
+                new DropAll(_compiler)
+            };
+
+            _microstackWithInclude = new List<ICompilationStep>
+            {
+                new DirectMatch(_compiler),
+                new IncludePatches(_compiler),
+                new IncludeAll(_compiler)
+            };
+
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path).ToLower())) return null;
+
+            var defaultInclude = false;
+            if (source.Path.StartsWith("mods"))
+            {
+                if (_include_directly.Any(path => source.Path.StartsWith(path)))
+                {
+                    defaultInclude = true;
+                }
+            }
+
+            var source_files = source.File.FileInArchive;
+
+            var stack = defaultInclude ? _microstackWithInclude : _microstack;
+
+            var id = Guid.NewGuid().ToString();
+
+            var matches = source_files.PMap(e => Compiler.RunStack(stack, new RawSourceFile(e)
+            {
+                Path = Path.Combine(Consts.BSACreationDir, id, e.Paths.Last())
+            }));
+
+
+            foreach (var match in matches)
+            {
+                if (match is IgnoredDirectly) 
+                    Utils.Error($"File required for BSA {source.Path} creation doesn't exist: {match.To}");
+                _compiler.ExtraFiles.Add(match);
+            }
+            
+            CreateBSA directive;
+            using (var bsa = BSADispatch.OpenRead(source.AbsolutePath))
+            {
+                directive = new CreateBSA
+                {
+                    To = source.Path,
+                    TempID = id,
+                    State = bsa.State,
+                    FileStates = bsa.Files.Select(f => f.State).ToList()
+                };
+            }
+
+            return directive;
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/DirectMatch.cs

@@ -0,0 +1,29 @@
+using System.Linq;
+using Alphaleonis.Win32.Filesystem;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    class DirectMatch : ACompilationStep
+    {
+        public DirectMatch(Compiler compiler) : base(compiler)
+        {
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!_compiler.IndexedFiles.TryGetValue(source.Hash, out var found)) return null;
+            var result = source.EvolveTo<FromArchive>();
+
+            var match = found.Where(f =>
+                                Path.GetFileName(f.Paths[f.Paths.Length - 1]) == Path.GetFileName(source.Path))
+                            .OrderBy(f => f.Paths.Length)
+                            .FirstOrDefault() 
+                        ?? found.OrderBy(f => f.Paths.Length).FirstOrDefault();
+
+            result.ArchiveHashPath = match.MakeRelativePaths();
+
+            return result;
+
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/DropAll.cs

@@ -0,0 +1,19 @@
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class DropAll : ACompilationStep
+    {
+        public DropAll(Compiler compiler) : base(compiler)
+        {
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            var result = source.EvolveTo<NoMatch>();
+            result.Reason = "No Match in Stack";
+            Utils.Log($"No match for: {source.Path}");
+            return result;
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IStackStep.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public interface ICompilationStep
+    {
+        Directive Run(RawSourceFile source);
+    }
+}

Wabbajack.Lib/CompilationSteps/IgnoreDisabledMods.cs

@@ -0,0 +1,49 @@
+using System.Collections.Generic;
+using System.Linq;
+using Alphaleonis.Win32.Filesystem;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IgnoreDisabledMods : ACompilationStep
+    {
+        private readonly IEnumerable<string> _allEnabledMods;
+
+        public IgnoreDisabledMods(Compiler compiler) : base(compiler)
+        {
+            var alwaysEnabled = _compiler.ModInis.Where(f => IsAlwaysEnabled(f.Value)).Select(f => f.Key).ToHashSet();
+
+            _allEnabledMods = _compiler.SelectedProfiles
+                .SelectMany(p => File.ReadAllLines(Path.Combine(_compiler.MO2Folder, "profiles", p, "modlist.txt")))
+                .Where(line => line.StartsWith("+") || line.EndsWith("_separator"))
+                .Select(line => line.Substring(1))
+                .Concat(alwaysEnabled)
+                .Select(line => Path.Combine("mods", line) + "\\")
+                .ToList();
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!source.Path.StartsWith("mods") || _allEnabledMods.Any(mod => source.Path.StartsWith(mod)))
+                return null;
+            var r = source.EvolveTo<IgnoredDirectly>();
+            r.Reason = "Disabled Mod";
+            return r;
+        }
+
+
+        private static bool IsAlwaysEnabled(dynamic data)
+        {
+            if (data == null)
+                return false;
+            if (data.General != null && data.General.notes != null &&
+                data.General.notes.Contains(
+                    Consts.WABBAJACK_ALWAYS_ENABLE))
+                return true;
+            if (data.General != null && data.General.comments != null &&
+                data.General.notes.Contains(Consts.WABBAJACK_ALWAYS_ENABLE))
+                return true;
+            return false;
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IgnoreEndsWith.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IgnoreEndsWith : ACompilationStep
+    {
+        private readonly string _reason;
+        private readonly string _postfix;
+
+        public IgnoreEndsWith(Compiler compiler, string postfix) : base(compiler)
+        {
+            _postfix = postfix;
+            _reason = $"Ignored because path ends with {postfix}";
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!source.Path.EndsWith(_postfix)) return null;
+            var result = source.EvolveTo<IgnoredDirectly>();
+            result.Reason = _reason;
+            return result;
+
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs

@@ -0,0 +1,23 @@
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IgnoreGameFiles : ACompilationStep
+    {
+        private readonly string _startDir;
+
+        public IgnoreGameFiles(Compiler compiler) : base(compiler)
+        {
+            _startDir = Consts.GameFolderFilesDir + "\\";
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!source.Path.StartsWith(_startDir)) return null;
+            var i = source.EvolveTo<IgnoredDirectly>();
+            i.Reason = "Default game file";
+            return i;
+
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IgnorePathContains.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IgnorePathContains : ACompilationStep
+    {
+        private readonly string _pattern;
+        private readonly string _reason;
+
+        public IgnorePathContains(Compiler compiler, string pattern) : base(compiler)
+        {
+            _pattern = $"\\{pattern.Trim('\\')}\\";
+            _reason = $"Ignored because path contains {_pattern}";
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!source.Path.Contains(_pattern)) return null;
+            var result = source.EvolveTo<IgnoredDirectly>();
+            result.Reason = _reason;
+            return result;
+
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IgnoreRegex.cs

@@ -0,0 +1,27 @@
+using System.Text.RegularExpressions;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IgnoreRegex : ACompilationStep
+    {
+        private readonly string _reason;
+        private string _pattern;
+        private readonly Regex _regex;
+
+        public IgnoreRegex(Compiler compiler, string pattern) : base(compiler)
+        {
+            _pattern = pattern;
+            _reason = $"Ignored because path matches regex {pattern}";
+            _regex = new Regex(pattern);
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!_regex.IsMatch(source.Path)) return null;
+            var result = source.EvolveTo<IgnoredDirectly>();
+            result.Reason = _reason;
+            return result;
+
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IgnoreStartsWith.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IgnoreStartsWith : ACompilationStep
+    {
+        private readonly string _reason;
+        private readonly string _prefix;
+
+        public IgnoreStartsWith(Compiler compiler, string prefix) : base(compiler)
+        {
+            _prefix = prefix;
+            _reason = string.Format("Ignored because path starts with {0}", _prefix);
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (source.Path.StartsWith(_prefix))
+            {
+                var result = source.EvolveTo<IgnoredDirectly>();
+                result.Reason = _reason;
+                return result;
+            }
+            return null;
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IgnoreWabbajackInstallCruft.cs

@@ -0,0 +1,28 @@
+using System.Collections.Generic;
+using System.Linq;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    class IgnoreWabbajackInstallCruft : ACompilationStep
+    {
+        private readonly HashSet<string> _cruftFiles;
+
+        public IgnoreWabbajackInstallCruft(Compiler compiler) : base(compiler)
+        { 
+            _cruftFiles = new HashSet<string>
+            {
+                "7z.dll", "7z.exe", "vfs_staged_files\\", "nexus.key_cache", "patch_cache\\",
+                Consts.NexusCacheDirectory + "\\"
+            };
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!_cruftFiles.Any(f => source.Path.StartsWith(f))) return null;
+            var result = source.EvolveTo<IgnoredDirectly>();
+            result.Reason = "Wabbajack Cruft file";
+            return result;
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IncludeAll.cs

@@ -0,0 +1,18 @@
+using Alphaleonis.Win32.Filesystem;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IncludeAll : ACompilationStep
+    {
+        public IncludeAll(Compiler compiler) : base(compiler)
+        {
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            var inline = source.EvolveTo<InlineFile>();
+            inline.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath));
+            return inline;
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IncludeAllConfigs.cs

@@ -0,0 +1,20 @@
+using Alphaleonis.Win32.Filesystem;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    class IncludeAllConfigs : ACompilationStep
+    {
+        public IncludeAllConfigs(Compiler compiler) : base(compiler)
+        {
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!Consts.ConfigFileExtensions.Contains(Path.GetExtension(source.Path))) return null;
+            var result = source.EvolveTo<InlineFile>();
+            result.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath));
+            return result;
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IncludeDummyESPs.cs

@@ -0,0 +1,30 @@
+using Alphaleonis.Win32.Filesystem;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IncludeDummyESPs : ACompilationStep
+    {
+        public IncludeDummyESPs(Compiler compiler) : base(compiler)
+        {
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (Path.GetExtension(source.AbsolutePath) != ".esp" &&
+                Path.GetExtension(source.AbsolutePath) != ".esm") return null;
+            
+            var bsa = Path.Combine(Path.GetDirectoryName(source.AbsolutePath),
+                Path.GetFileNameWithoutExtension(source.AbsolutePath) + ".bsa");
+            var bsaTextures = Path.Combine(Path.GetDirectoryName(source.AbsolutePath),
+                Path.GetFileNameWithoutExtension(source.AbsolutePath) + " - Textures.bsa");
+            var espSize = new FileInfo(source.AbsolutePath).Length;
+            
+            if (espSize > 250 || (!File.Exists(bsa) && !File.Exists(bsaTextures))) return null;
+            
+            var inline = source.EvolveTo<InlineFile>();
+            inline.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath));
+            return inline;
+
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IncludeLOOTFiles.cs

@@ -0,0 +1,24 @@
+using Alphaleonis.Win32.Filesystem;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IncludeLootFiles : ACompilationStep
+    {
+        private readonly string _prefix;
+
+        public IncludeLootFiles(Compiler compiler) : base(compiler)
+        {
+            _prefix = Consts.LOOTFolderFilesDir + "\\";
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!source.Path.StartsWith(_prefix)) return null;
+            var result = source.EvolveTo<InlineFile>();
+            result.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath).ToBase64());
+            return result;
+
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IncludeModIniData.cs

@@ -0,0 +1,20 @@
+using Alphaleonis.Win32.Filesystem;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    class IncludeModIniData : ACompilationStep
+    {
+        public IncludeModIniData(Compiler compiler) : base(compiler)
+        {
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!source.Path.StartsWith("mods\\") || !source.Path.EndsWith("\\meta.ini")) return null;
+            var e = source.EvolveTo<InlineFile>();
+            e.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath));
+            return e;
+
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IncludeOtherProfiles.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IgnoreOtherProfiles : ACompilationStep
+    {
+        private readonly IEnumerable<string> _profiles;
+
+        public IgnoreOtherProfiles(Compiler compiler) : base(compiler)
+        {
+            _profiles = _compiler.SelectedProfiles
+                .Select(p => Path.Combine("profiles", p) + "\\")
+                .ToList();
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!source.Path.StartsWith("profiles\\")) return null;
+            if (_profiles.Any(profile => source.Path.StartsWith(profile))) return null;
+            var c = source.EvolveTo<IgnoredDirectly>();
+            c.Reason = "File not for selected profiles";
+            return c;
+
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IncludePatches.cs

@@ -0,0 +1,41 @@
+using System.Collections.Generic;
+using System.Linq;
+using Alphaleonis.Win32.Filesystem;
+using VFS;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IncludePatches : ACompilationStep
+    {
+        private readonly Dictionary<string, IGrouping<string, VirtualFile>> _indexed;
+
+        public IncludePatches(Compiler compiler) : base(compiler)
+        {
+            _indexed = _compiler.IndexedFiles.Values
+                .SelectMany(f => f)
+                .GroupBy(f => Path.GetFileName(Enumerable.Last(f.Paths)).ToLower())
+                .ToDictionary(f => f.Key);
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!_indexed.TryGetValue(Path.GetFileName(Enumerable.Last(source.File.Paths).ToLower()), out var value))
+                return null;
+
+            var found = value.OrderByDescending(f => (f.TopLevelArchive ?? f).LastModified).First();
+
+            var e = source.EvolveTo<PatchedFromArchive>();
+            e.ArchiveHashPath = found.MakeRelativePaths();
+            e.To = source.Path;
+            e.Hash = source.File.Hash;
+
+            Utils.TryGetPatch(found.Hash, source.File.Hash, out var data);
+
+            if (data != null)
+                e.PatchID = _compiler.IncludeFile(data);
+
+            return e;
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs

@@ -0,0 +1,43 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    class IncludePropertyFiles : ACompilationStep
+    {
+        public IncludePropertyFiles(Compiler compiler) : base(compiler)
+        {
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            var files = new HashSet<string>
+            {
+                _compiler.ModListImage, _compiler.ModListReadme
+            };
+            if (!files.Any(f => source.AbsolutePath.Equals(f))) return null;
+            if (!File.Exists(source.AbsolutePath)) return null;
+            var isBanner = source.AbsolutePath == _compiler.ModListImage;
+            //var isReadme = source.AbsolutePath == ModListReadme;
+            var result = source.EvolveTo<PropertyFile>();
+            result.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath));
+            if (isBanner)
+            {
+                result.Type = PropertyType.Banner;
+                _compiler.ModListImage = result.SourceDataID;
+            }
+            else
+            {
+                result.Type = PropertyType.Readme;
+                _compiler.ModListReadme = result.SourceDataID;
+            }
+            return result;
+        }
+
+
+    }
+}

Wabbajack.Lib/CompilationSteps/IncludeRegex.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IncludeRegex : ACompilationStep
+    {
+        private readonly Regex _regex;
+
+        public IncludeRegex(Compiler compiler, string pattern) : base(compiler)
+        {
+            _regex = new Regex(pattern);
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!_regex.IsMatch(source.Path)) return null;
+            
+            var result = source.EvolveTo<InlineFile>();
+            result.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath));
+            return result;
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IncludeStubbedConfigfiles.cs

@@ -0,0 +1,42 @@
+using System.Text;
+using Alphaleonis.Win32.Filesystem;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IncludeStubbedConfigFiles : ACompilationStep
+    {
+        public IncludeStubbedConfigFiles(Compiler compiler) : base(compiler)
+        {
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            return Consts.ConfigFileExtensions.Contains(Path.GetExtension(source.Path)) ? RemapFile(source) : null;
+        }
+
+        private Directive RemapFile(RawSourceFile source)
+        {
+            var data = File.ReadAllText(source.AbsolutePath);
+            var originalData = data;
+
+            data = data.Replace(_compiler.GamePath, Consts.GAME_PATH_MAGIC_BACK);
+            data = data.Replace(_compiler.GamePath.Replace("\\", "\\\\"), Consts.GAME_PATH_MAGIC_DOUBLE_BACK);
+            data = data.Replace(_compiler.GamePath.Replace("\\", "/"), Consts.GAME_PATH_MAGIC_FORWARD);
+
+            data = data.Replace(_compiler.MO2Folder, Consts.MO2_PATH_MAGIC_BACK);
+            data = data.Replace(_compiler.MO2Folder.Replace("\\", "\\\\"), Consts.MO2_PATH_MAGIC_DOUBLE_BACK);
+            data = data.Replace(_compiler.MO2Folder.Replace("\\", "/"), Consts.MO2_PATH_MAGIC_FORWARD);
+
+            data = data.Replace(_compiler.MO2DownloadsFolder, Consts.DOWNLOAD_PATH_MAGIC_BACK);
+            data = data.Replace(_compiler.MO2DownloadsFolder.Replace("\\", "\\\\"), Consts.DOWNLOAD_PATH_MAGIC_DOUBLE_BACK);
+            data = data.Replace(_compiler.MO2DownloadsFolder.Replace("\\", "/"), Consts.DOWNLOAD_PATH_MAGIC_FORWARD);
+
+            if (data == originalData)
+                return null;
+            var result = source.EvolveTo<RemappedInlineFile>();
+            result.SourceDataID = _compiler.IncludeFile(Encoding.UTF8.GetBytes(data));
+            return result;
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs

@@ -0,0 +1,39 @@
+using System.Collections.Generic;
+using System.Linq;
+using Alphaleonis.Win32.Filesystem;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    class IncludeTaggedMods : ACompilationStep
+    {
+        private readonly IEnumerable<string> _includeDirectly;
+        private string _tag;
+
+
+        public IncludeTaggedMods(Compiler compiler, string tag) : base(compiler)
+        {
+            _tag = tag;
+            _includeDirectly = _compiler.ModInis.Where(kv =>
+            {
+                var general = kv.Value.General;
+                if (general.notes != null && general.notes.Contains(_tag))
+                    return true;
+                return general.comments != null && general.comments.Contains(_tag);
+            }).Select(kv => $"mods\\{kv.Key}\\");
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!source.Path.StartsWith("mods")) return null;
+            foreach (var modpath in _includeDirectly)
+            {
+                if (!source.Path.StartsWith(modpath)) continue;
+                var result = source.EvolveTo<InlineFile>();
+                result.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath));
+                return result;
+            }
+
+            return null;
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/IncludeThisProfile.cs

@@ -0,0 +1,39 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Alphaleonis.Win32.Filesystem;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IncludeThisProfile : ACompilationStep
+    {
+        private readonly IEnumerable<string> _correctProfiles;
+
+        public IncludeThisProfile(Compiler compiler) : base(compiler)
+        {
+            _correctProfiles = _compiler.SelectedProfiles.Select(p => Path.Combine("profiles", p) + "\\").ToList();
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (_correctProfiles.Any(p => source.Path.StartsWith(p)))
+            {
+                var data = source.Path.EndsWith("\\modlist.txt") ? ReadAndCleanModlist(source.AbsolutePath) : File.ReadAllBytes(source.AbsolutePath);
+
+                var e = source.EvolveTo<InlineFile>();
+                e.SourceDataID = _compiler.IncludeFile(data);
+                return e;
+            }
+
+            return null;
+        }
+        private static byte[] ReadAndCleanModlist(string absolutePath)
+        {
+            var lines = File.ReadAllLines(absolutePath);
+            lines = (from line in lines
+                where !(line.StartsWith("-") && !line.EndsWith("_separator"))
+                select line).ToArray();
+            return Encoding.UTF8.GetBytes(string.Join("\r\n", lines));
+        }
+    }
+}

Wabbajack.Lib/CompilationSteps/PatchStockESMs.cs

@@ -0,0 +1,40 @@
+using System.IO;
+using Wabbajack.Common;
+using File = Alphaleonis.Win32.Filesystem.File;
+using Path = Alphaleonis.Win32.Filesystem.Path;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    class PatchStockESMs : ACompilationStep
+    {
+        public PatchStockESMs(Compiler compiler) : base(compiler)
+        {
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            var filename = Path.GetFileName(source.Path);
+            var gameFile = Path.Combine(_compiler.GamePath, "Data", filename);
+            if (!Consts.GameESMs.Contains(filename) || !source.Path.StartsWith("mods\\") ||
+                !File.Exists(gameFile)) return null;
+            
+            Utils.Log(
+                $"A ESM named {filename} was found in a mod that shares a name with a core game ESMs, it is assumed this is a cleaned ESM and it will be binary patched.");
+            var result = source.EvolveTo<CleanedESM>();
+            result.SourceESMHash = _compiler.VFS.Lookup(gameFile).Hash;
+
+            Utils.Status($"Generating patch of {filename}");
+            using (var ms = new MemoryStream())
+            {
+                Utils.CreatePatch(File.ReadAllBytes(gameFile), File.ReadAllBytes(source.AbsolutePath), ms);
+                var data = ms.ToArray();
+                result.SourceDataID = _compiler.IncludeFile(data);
+                Utils.Log($"Generated a {data.Length} byte patch for {filename}");
+
+            }
+            
+            return result;
+
+        }
+    }
+}

Wabbajack.Lib/Compiler.cs

@@ -1,6 +1,7 @@
 using CommonMark;
 using Compression.BSA;
 using System;
+using System.CodeDom;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
@@ -12,6 +13,7 @@ using System.Text;
 using System.Text.RegularExpressions;
 using VFS;
 using Wabbajack.Common;
+using Wabbajack.Lib.CompilationSteps;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.NexusApi;
 using Wabbajack.Lib.Validation;
@@ -498,12 +500,12 @@ namespace Wabbajack.Lib
         }
 
 
-        private Directive RunStack(IEnumerable<Func<RawSourceFile, Directive>> stack, RawSourceFile source)
+        public static Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
         {
-            Status($"Compiling {source.Path}");
+            Utils.Status($"Compiling {source.Path}");
-            foreach (var f in stack)
+            foreach (var step in stack)
             {
-                var result = f(source);
+                var result = step.Run(source);
                 if (result != null) return result;
             }
 
@@ -517,631 +519,65 @@ namespace Wabbajack.Lib
         ///     result included into the pack
         /// </summary>
         /// <returns></returns>
-        private IEnumerable<Func<RawSourceFile, Directive>> MakeStack()
+        private IEnumerable<ICompilationStep> MakeStack()
         {
             Info("Generating compilation stack");
-            return new List<Func<RawSourceFile, Directive>>
+            return new List<ICompilationStep>
             {
-                IncludePropertyFiles(),
+                new IncludePropertyFiles(this),
-                IgnoreStartsWith("logs\\"),
+                new IgnoreStartsWith(this,"logs\\"),
-                IncludeRegex("^downloads\\\\.*\\.meta"),
+                new IncludeRegex(this, "^downloads\\\\.*\\.meta"),
-                IgnoreStartsWith("downloads\\"),
+                new IgnoreStartsWith(this, "downloads\\"),
-                IgnoreStartsWith("webcache\\"),
+                new IgnoreStartsWith(this,"webcache\\"),
-                IgnoreStartsWith("overwrite\\"),
+                new IgnoreStartsWith(this, "overwrite\\"),
-                IgnorePathContains("temporary_logs"),
+                new IgnorePathContains(this,"temporary_logs"),
-                IgnorePathContains("GPUCache"),
+                new IgnorePathContains(this, "GPUCache"),
-                IgnorePathContains("SSEEdit Cache"),
+                new IgnorePathContains(this, "SSEEdit Cache"),
-                IgnoreEndsWith(".pyc"),
+                new IgnoreEndsWith(this, ".pyc"),
-                IgnoreEndsWith(".log"),
+                new IgnoreEndsWith(this, ".log"),
-                IgnoreOtherProfiles(),
+                new IgnoreOtherProfiles(this),
-                IgnoreDisabledMods(),
+                new IgnoreDisabledMods(this),
-                IncludeThisProfile(),
+                new IncludeThisProfile(this),
                 // Ignore the ModOrganizer.ini file it contains info created by MO2 on startup
-                IncludeStubbedConfigFiles(),
+                new IncludeStubbedConfigFiles(this),
-                IncludeLootFiles(),
+                new IncludeLootFiles(this),
-                IgnoreStartsWith(Path.Combine(Consts.GameFolderFilesDir, "Data")),
+                new IgnoreStartsWith(this, Path.Combine(Consts.GameFolderFilesDir, "Data")),
-                IgnoreStartsWith(Path.Combine(Consts.GameFolderFilesDir, "Papyrus Compiler")),
+                new IgnoreStartsWith(this, Path.Combine(Consts.GameFolderFilesDir, "Papyrus Compiler")),
-                IgnoreStartsWith(Path.Combine(Consts.GameFolderFilesDir, "Skyrim")),
+                new IgnoreStartsWith(this, Path.Combine(Consts.GameFolderFilesDir, "Skyrim")),
-                IgnoreRegex(Consts.GameFolderFilesDir + "\\\\.*\\.bsa"),
+                new IgnoreRegex(this, Consts.GameFolderFilesDir + "\\\\.*\\.bsa"),
-                IncludeModIniData(),
+                new IncludeModIniData(this),
-                DirectMatch(),
+                new DirectMatch(this),
-                IncludeTaggedFiles(Consts.WABBAJACK_INCLUDE),
+                new IncludeTaggedMods(this, Consts.WABBAJACK_INCLUDE),
-                DeconstructBSAs(), // Deconstruct BSAs before building patches so we don't generate massive patch files
+                new DeconstructBSAs(this), // Deconstruct BSAs before building patches so we don't generate massive patch files
-                IncludePatches(),
+                new IncludePatches(this),
-                IncludeDummyESPs(),
+                new IncludeDummyESPs(this),
 
 
                 // If we have no match at this point for a game folder file, skip them, we can't do anything about them
-                IgnoreGameFiles(),
+                new IgnoreGameFiles(this),
 
                 // There are some types of files that will error the compilation, because they're created on-the-fly via tools
                 // so if we don't have a match by this point, just drop them.
-                IgnoreEndsWith(".ini"),
+                new IgnoreEndsWith(this, ".ini"),
-                IgnoreEndsWith(".html"),
+                new IgnoreEndsWith(this, ".html"),
-                IgnoreEndsWith(".txt"),
+                new IgnoreEndsWith(this, ".txt"),
                 // Don't know why, but this seems to get copied around a bit
-                IgnoreEndsWith("HavokBehaviorPostProcess.exe"),
+                new IgnoreEndsWith(this, "HavokBehaviorPostProcess.exe"),
                 // Theme file MO2 downloads somehow
-                IgnoreEndsWith("splash.png"),
+                new IgnoreEndsWith(this, "splash.png"),
 
-                IgnoreEndsWith(".bin"),
+                new IgnoreEndsWith(this, ".bin"),
-                IgnoreEndsWith(".refcache"),
+                new IgnoreEndsWith(this, ".refcache"),
 
-                IgnoreWabbajackInstallCruft(),
+                new IgnoreWabbajackInstallCruft(this),
 
-                PatchStockESMs(),
+                new PatchStockESMs(this),
 
-                IncludeAllConfigs(),
+                new IncludeAllConfigs(this),
-                IncludeTaggedFiles(Consts.WABBAJACK_NOMATCH_INCLUDE),
+                new IncludeTaggedMods(this, Consts.WABBAJACK_NOMATCH_INCLUDE),
-                zEditIntegration.IncludezEditPatches(this),
+                new zEditIntegration.IncludeZEditPatches(this),
 
-                DropAll()
+                new DropAll(this)
-            };
-        }
-
-        private Func<RawSourceFile, Directive> IncludePropertyFiles()
-        {
-            return source =>
-            {
-                var files = new HashSet<string>
-                {
-                    ModListImage, ModListReadme
-                };
-                if (!files.Any(f => source.AbsolutePath.Equals(f))) return null;
-                if (!File.Exists(source.AbsolutePath)) return null;
-                var isBanner = source.AbsolutePath == ModListImage;
-                //var isReadme = source.AbsolutePath == ModListReadme;
-                var result = source.EvolveTo<PropertyFile>();
-                result.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath));
-                if (isBanner)
-                {
-                    result.Type = PropertyType.Banner;
-                    ModListImage = result.SourceDataID;
-                }
-                else
-                {
-                    result.Type = PropertyType.Readme;
-                    ModListReadme = result.SourceDataID;
-                }
-                return result;
-            };
-        }
-
-        private Func<RawSourceFile, Directive> IgnoreWabbajackInstallCruft()
-        {
-            var cruft_files = new HashSet<string>
-            {
-                "7z.dll", "7z.exe", "vfs_staged_files\\", "nexus.key_cache", "patch_cache\\",
-                Consts.NexusCacheDirectory + "\\"
-            };
-            return source =>
-            {
-                if (!cruft_files.Any(f => source.Path.StartsWith(f))) return null;
-                var result = source.EvolveTo<IgnoredDirectly>();
-                result.Reason = "Wabbajack Cruft file";
-                return result;
-            };
-        }
-
-        private Func<RawSourceFile, Directive> IncludeAllConfigs()
-        {
-            return source =>
-            {
-                if (!Consts.ConfigFileExtensions.Contains(Path.GetExtension(source.Path))) return null;
-                var result = source.EvolveTo<InlineFile>();
-                result.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath));
-                return result;
-            };
-        }
-
-        private Func<RawSourceFile, Directive> PatchStockESMs()
-        {
-            return source =>
-            {
-                var filename = Path.GetFileName(source.Path);
-                var game_file = Path.Combine(GamePath, "Data", filename);
-                if (Consts.GameESMs.Contains(filename) && source.Path.StartsWith("mods\\") && File.Exists(game_file))
-                {
-                    Info(
-                        $"A ESM named {filename} was found in a mod that shares a name with a core game ESMs, it is assumed this is a cleaned ESM and it will be binary patched.");
-                    var result = source.EvolveTo<CleanedESM>();
-                    result.SourceESMHash = VFS.Lookup(game_file).Hash;
-
-                    Status($"Generating patch of {filename}");
-                    using (var ms = new MemoryStream())
-                    {
-                        Utils.CreatePatch(File.ReadAllBytes(game_file), File.ReadAllBytes(source.AbsolutePath), ms);
-                        var data = ms.ToArray();
-                        result.SourceDataID = IncludeFile(data);
-                        Info($"Generated a {data.Length} byte patch for {filename}");
-
-                    }
-
-
-                    return result;
-                }
-
-                return null;
-            };
-        }
-
-
-        private Func<RawSourceFile, Directive> IncludeLootFiles()
-        {
-            var prefix = Consts.LOOTFolderFilesDir + "\\";
-            return source =>
-            {
-                if (source.Path.StartsWith(prefix))
-                {
-                    var result = source.EvolveTo<InlineFile>();
-                    result.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath).ToBase64());
-                    return result;
-                }
-
-                return null;
-            };
-        }
-
-        private Func<RawSourceFile, Directive> IncludeStubbedConfigFiles()
-        {
-            return source =>
-            {
-                if (Consts.ConfigFileExtensions.Contains(Path.GetExtension(source.Path)))
-                    return RemapFile(source, GamePath);
-                return null;
-            };
-        }
-
-        private Directive RemapFile(RawSourceFile source, string gamePath)
-        {
-            var data = File.ReadAllText(source.AbsolutePath);
-            var original_data = data;
-
-            data = data.Replace(GamePath, Consts.GAME_PATH_MAGIC_BACK);
-            data = data.Replace(GamePath.Replace("\\", "\\\\"), Consts.GAME_PATH_MAGIC_DOUBLE_BACK);
-            data = data.Replace(GamePath.Replace("\\", "/"), Consts.GAME_PATH_MAGIC_FORWARD);
-
-            data = data.Replace(MO2Folder, Consts.MO2_PATH_MAGIC_BACK);
-            data = data.Replace(MO2Folder.Replace("\\", "\\\\"), Consts.MO2_PATH_MAGIC_DOUBLE_BACK);
-            data = data.Replace(MO2Folder.Replace("\\", "/"), Consts.MO2_PATH_MAGIC_FORWARD);
-
-            data = data.Replace(MO2DownloadsFolder, Consts.DOWNLOAD_PATH_MAGIC_BACK);
-            data = data.Replace(MO2DownloadsFolder.Replace("\\", "\\\\"), Consts.DOWNLOAD_PATH_MAGIC_DOUBLE_BACK);
-            data = data.Replace(MO2DownloadsFolder.Replace("\\", "/"), Consts.DOWNLOAD_PATH_MAGIC_FORWARD);
-
-            if (data == original_data)
-                return null;
-            var result = source.EvolveTo<RemappedInlineFile>();
-            result.SourceDataID = IncludeFile(Encoding.UTF8.GetBytes(data));
-            return result;
-        }
-
-        private Func<RawSourceFile, Directive> IgnorePathContains(string v)
-        {
-            v = $"\\{v.Trim('\\')}\\";
-            var reason = $"Ignored because path contains {v}";
-            return source =>
-            {
-                if (source.Path.Contains(v))
-                {
-                    var result = source.EvolveTo<IgnoredDirectly>();
-                    result.Reason = reason;
-                    return result;
-                }
-
-                return null;
-            };
-        }
-
-
-        /// <summary>
-        ///     If a user includes WABBAJACK_INCLUDE directly in the notes or comments of a mod, the contents of that
-        ///     mod will be inlined into the installer. USE WISELY.
-        /// </summary>
-        /// <returns></returns>
-        private Func<RawSourceFile, Directive> IncludeTaggedFiles(string tag)
-        {
-            var include_directly = ModInis.Where(kv =>
-            {
-                var general = kv.Value.General;
-                if (general.notes != null && general.notes.Contains(tag))
-                    return true;
-                if (general.comments != null && general.comments.Contains(tag))
-                    return true;
-                return false;
-            }).Select(kv => $"mods\\{kv.Key}\\");
-
-
-            return source =>
-            {
-                if (source.Path.StartsWith("mods"))
-                    foreach (var modpath in include_directly)
-                        if (source.Path.StartsWith(modpath))
-                        {
-                            var result = source.EvolveTo<InlineFile>();
-                            result.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath));
-                            return result;
-                        }
-
-                return null;
-            };
-        }
-
-
-        /// <summary>
-        ///     Some tools like the Cathedral Asset Optimizer will create dummy ESPs whos only existance is to make
-        ///     sure a BSA with the same name is loaded. We don't have a good way to detect these, but if an ESP is
-        ///     less than 100 bytes in size and shares a name with a BSA it's a pretty good chance that it's a dummy
-        ///     and the contents are generated.
-        /// </summary>
-        /// <returns></returns>
-        private Func<RawSourceFile, Directive> IncludeDummyESPs()
-        {
-            return source =>
-            {
-                if (Path.GetExtension(source.AbsolutePath) == ".esp" || Path.GetExtension(source.AbsolutePath) == ".esm")
-                {
-                    var bsa = Path.Combine(Path.GetDirectoryName(source.AbsolutePath),
-                        Path.GetFileNameWithoutExtension(source.AbsolutePath) + ".bsa");
-                    var bsa_textures = Path.Combine(Path.GetDirectoryName(source.AbsolutePath),
-                        Path.GetFileNameWithoutExtension(source.AbsolutePath) + " - Textures.bsa");
-                    var esp_size = new FileInfo(source.AbsolutePath).Length;
-                    if (esp_size <= 250 && (File.Exists(bsa) || File.Exists(bsa_textures)))
-                    {
-                        var inline = source.EvolveTo<InlineFile>();
-                        inline.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath));
-                        return inline;
-                    }
-                }
-
-                return null;
-            };
-        }
-
-
-        /// <summary>
-        ///     This function will search for a way to create a BSA in the installed mod list by assembling it from files
-        ///     found in archives. To do this we hash all the files in side the BSA then try to find matches and patches for
-        ///     all of the files.
-        /// </summary>
-        /// <returns></returns>
-        private Func<RawSourceFile, Directive> DeconstructBSAs()
-        {
-            var include_directly = ModInis.Where(kv =>
-            {
-                var general = kv.Value.General;
-                if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE))
-                    return true;
-                if (general.comments != null && general.comments.Contains(Consts.WABBAJACK_INCLUDE))
-                    return true;
-                return false;
-            }).Select(kv => $"mods\\{kv.Key}\\");
-
-            var microstack = new List<Func<RawSourceFile, Directive>>
-            {
-                DirectMatch(),
-                IncludePatches(),
-                DropAll()
-            };
-
-            var microstack_with_include = new List<Func<RawSourceFile, Directive>>
-            {
-                DirectMatch(),
-                IncludePatches(),
-                IncludeALL()
-            };
-
-
-            return source =>
-            {
-                if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path).ToLower())) return null;
-
-                var default_include = false;
-                if (source.Path.StartsWith("mods"))
-                    foreach (var modpath in include_directly)
-                        if (source.Path.StartsWith(modpath))
-                        {
-                            default_include = true;
-                            break;
-                        }
-
-                var source_files = source.File.FileInArchive;
-
-                var stack = default_include ? microstack_with_include : microstack;
-
-                var id = Guid.NewGuid().ToString();
-
-                var matches = source_files.PMap(e => RunStack(stack, new RawSourceFile(e)
-                {
-                    Path = Path.Combine(Consts.BSACreationDir, id, e.Paths.Last())
-                }));
-
-
-                foreach (var match in matches)
-                {
-                    if (match is IgnoredDirectly) Error($"File required for BSA {source.Path} creation doesn't exist: {match.To}");
-                    ExtraFiles.Add(match);
-                }
-
-                ;
-
-                CreateBSA directive;
-                using (var bsa = BSADispatch.OpenRead(source.AbsolutePath))
-                {
-                    directive = new CreateBSA
-                    {
-                        To = source.Path,
-                        TempID = id,
-                        State = bsa.State,
-                        FileStates = bsa.Files.Select(f => f.State).ToList()
-                    };
-                }
-
-                return directive;
-            };
-        }
-
-        private Func<RawSourceFile, Directive> IncludeALL()
-        {
-            return source =>
-            {
-                var inline = source.EvolveTo<InlineFile>();
-                inline.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath));
-                return inline;
-            };
-        }
-
-        private Func<RawSourceFile, Directive> IgnoreDisabledMods()
-        {
-            var always_enabled = ModInis.Where(f => IsAlwaysEnabled(f.Value)).Select(f => f.Key).ToHashSet();
-
-            var all_enabled_mods = SelectedProfiles
-                .SelectMany(p => File.ReadAllLines(Path.Combine(MO2Folder, "profiles", p, "modlist.txt")))
-                .Where(line => line.StartsWith("+") || line.EndsWith("_separator"))
-                .Select(line => line.Substring(1))
-                .Concat(always_enabled)
-                .Select(line => Path.Combine("mods", line) + "\\")
-                .ToList();
-
-            return source =>
-            {
-                if (!source.Path.StartsWith("mods") || all_enabled_mods.Any(mod => source.Path.StartsWith(mod)))
-                    return null;
-                var r = source.EvolveTo<IgnoredDirectly>();
-                r.Reason = "Disabled Mod";
-                return r;
-            };
-        }
-
-        private static bool IsAlwaysEnabled(dynamic data)
-        {
-            if (data == null)
-                return false;
-            if (data.General != null && data.General.notes != null &&
-                data.General.notes.Contains(Consts.WABBAJACK_ALWAYS_ENABLE))
-                return true;
-            if (data.General != null && data.General.comments != null &&
-                data.General.notes.Contains(Consts.WABBAJACK_ALWAYS_ENABLE))
-                return true;
-            return false;
-        }
-
-        /// <summary>
-        ///     This matches files based purely on filename, and then creates a binary patch.
-        ///     In practice this is fine, because a single file tends to only come from one archive.
-        /// </summary>
-        /// <returns></returns>
-        private Func<RawSourceFile, Directive> IncludePatches()
-        {
-            var indexed = IndexedFiles.Values
-                .SelectMany(f => f)
-                .GroupBy(f => Path.GetFileName(f.Paths.Last()).ToLower())
-                .ToDictionary(f => f.Key);
-
-            return source =>
-            {
-                if (!indexed.TryGetValue(Path.GetFileName(source.File.Paths.Last().ToLower()), out var value))
-                    return null;
-
-                var found = value.OrderByDescending(f => (f.TopLevelArchive ?? f).LastModified).First();
-
-                var e = source.EvolveTo<PatchedFromArchive>();
-                e.ArchiveHashPath = found.MakeRelativePaths();
-                e.To = source.Path;
-                e.Hash = source.File.Hash;
-
-                Utils.TryGetPatch(found.Hash, source.File.Hash, out var data);
-
-                if (data != null)
-                    e.PatchID = IncludeFile(data);
-
-                return e;
-            };
-        }
-
-        private Func<RawSourceFile, Directive> IncludeModIniData()
-        {
-            return source =>
-            {
-                if (source.Path.StartsWith("mods\\") && source.Path.EndsWith("\\meta.ini"))
-                {
-                    var e = source.EvolveTo<InlineFile>();
-                    e.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath));
-                    return e;
-                }
-
-                return null;
-            };
-        }
-
-        private Func<RawSourceFile, Directive> IgnoreGameFiles()
-        {
-            var start_dir = Consts.GameFolderFilesDir + "\\";
-            return source =>
-            {
-                if (source.Path.StartsWith(start_dir))
-                {
-                    var i = source.EvolveTo<IgnoredDirectly>();
-                    i.Reason = "Default game file";
-                    return i;
-                }
-
-                return null;
-            };
-        }
-
-        private Func<RawSourceFile, Directive> IncludeThisProfile()
-        {
-            var correct_profiles = SelectedProfiles.Select(p => Path.Combine("profiles", p) + "\\").ToList();
-
-            return source =>
-            {
-                if (correct_profiles.Any(p => source.Path.StartsWith(p)))
-                {
-                    byte[] data;
-                    if (source.Path.EndsWith("\\modlist.txt"))
-                        data = ReadAndCleanModlist(source.AbsolutePath);
-                    else
-                        data = File.ReadAllBytes(source.AbsolutePath);
-
-                    var e = source.EvolveTo<InlineFile>();
-                    e.SourceDataID = IncludeFile(data);
-                    return e;
-                }
-
-                return null;
-            };
-        }
-
-        private byte[] ReadAndCleanModlist(string absolutePath)
-        {
-            var lines = File.ReadAllLines(absolutePath);
-            lines = (from line in lines
-                     where !(line.StartsWith("-") && !line.EndsWith("_separator"))
-                     select line).ToArray();
-            return Encoding.UTF8.GetBytes(string.Join("\r\n", lines));
-        }
-
-        private Func<RawSourceFile, Directive> IgnoreOtherProfiles()
-        {
-            var profiles = SelectedProfiles
-                .Select(p => Path.Combine("profiles", p) + "\\")
-                .ToList();
-
-            return source =>
-            {
-                if (source.Path.StartsWith("profiles\\"))
-                {
-                    if (profiles.Any(profile => source.Path.StartsWith(profile))) return null;
-                    var c = source.EvolveTo<IgnoredDirectly>();
-                    c.Reason = "File not for selected profiles";
-                    return c;
-                }
-
-                return null;
-            };
-        }
-
-        private Func<RawSourceFile, Directive> IgnoreEndsWith(string v)
-        {
-            var reason = string.Format("Ignored because path ends with {0}", v);
-            return source =>
-            {
-                if (source.Path.EndsWith(v))
-                {
-                    var result = source.EvolveTo<IgnoredDirectly>();
-                    result.Reason = reason;
-                    return result;
-                }
-
-                return null;
-            };
-        }
-
-        private Func<RawSourceFile, Directive> IgnoreRegex(string p)
-        {
-            var reason = string.Format("Ignored because path matches regex {0}", p);
-            var regex = new Regex(p);
-            return source =>
-            {
-                if (regex.IsMatch(source.Path))
-                {
-                    var result = source.EvolveTo<IgnoredDirectly>();
-                    result.Reason = reason;
-                    return result;
-                }
-
-                return null;
-            };
-        }
-
-
-        private Func<RawSourceFile, Directive> IncludeRegex(string pattern)
-        {
-            var regex = new Regex(pattern);
-            return source =>
-            {
-                if (regex.IsMatch(source.Path))
-                {
-                    var result = source.EvolveTo<InlineFile>();
-                    result.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath));
-                    return result;
-                }
-
-                return null;
-            };
-        }
-
-
-        private Func<RawSourceFile, Directive> DropAll()
-        {
-            return source =>
-            {
-                var result = source.EvolveTo<NoMatch>();
-                result.Reason = "No Match in Stack";
-                Info($"No match for: {source.Path}");
-                return result;
-            };
-        }
-
-        private Func<RawSourceFile, Directive> DirectMatch()
-        {
-            return source =>
-            {
-                if (IndexedFiles.TryGetValue(source.Hash, out var found))
-                {
-                    var result = source.EvolveTo<FromArchive>();
-
-                    var match = found.Where(f =>
-                            Path.GetFileName(f.Paths[f.Paths.Length - 1]) == Path.GetFileName(source.Path))
-                        .OrderBy(f => f.Paths.Length)
-                        .FirstOrDefault();
-
-                    if (match == null)
-                        match = found.OrderBy(f => f.Paths.Length).FirstOrDefault();
-
-                    result.ArchiveHashPath = match.MakeRelativePaths();
-
-                    return result;
-                }
-
-                return null;
-            };
-        }
-
-        private Func<RawSourceFile, Directive> IgnoreStartsWith(string v)
-        {
-            var reason = string.Format("Ignored because path starts with {0}", v);
-            return source =>
-            {
-                if (source.Path.StartsWith(v))
-                {
-                    var result = source.EvolveTo<IgnoredDirectly>();
-                    result.Reason = reason;
-                    return result;
-                }
-
-                return null;
             };
         }
 

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -119,6 +119,31 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="CerasConfig.cs" />
+    <Compile Include="CompilationSteps\ACompilationStep.cs" />
+    <Compile Include="CompilationSteps\DeconstructBSAs.cs" />
+    <Compile Include="CompilationSteps\DirectMatch.cs" />
+    <Compile Include="CompilationSteps\DropAll.cs" />
+    <Compile Include="CompilationSteps\IgnoreDisabledMods.cs" />
+    <Compile Include="CompilationSteps\IgnoreEndsWith.cs" />
+    <Compile Include="CompilationSteps\IgnoreGameFiles.cs" />
+    <Compile Include="CompilationSteps\IgnorePathContains.cs" />
+    <Compile Include="CompilationSteps\IgnoreRegex.cs" />
+    <Compile Include="CompilationSteps\IgnoreStartsWith.cs" />
+    <Compile Include="CompilationSteps\IgnoreWabbajackInstallCruft.cs" />
+    <Compile Include="CompilationSteps\IncludeAll.cs" />
+    <Compile Include="CompilationSteps\IncludeAllConfigs.cs" />
+    <Compile Include="CompilationSteps\IncludeDummyESPs.cs" />
+    <Compile Include="CompilationSteps\IncludeLootFiles.cs" />
+    <Compile Include="CompilationSteps\IncludeModIniData.cs" />
+    <Compile Include="CompilationSteps\IncludeOtherProfiles.cs" />
+    <Compile Include="CompilationSteps\IncludePatches.cs" />
+    <Compile Include="CompilationSteps\IncludePropertyFiles.cs" />
+    <Compile Include="CompilationSteps\IncludeRegex.cs" />
+    <Compile Include="CompilationSteps\IncludeStubbedConfigFiles.cs" />
+    <Compile Include="CompilationSteps\IncludeTaggedMods.cs" />
+    <Compile Include="CompilationSteps\IncludeThisProfile.cs" />
+    <Compile Include="CompilationSteps\IStackStep.cs" />
+    <Compile Include="CompilationSteps\PatchStockESMs.cs" />
     <Compile Include="Compiler.cs" />
     <Compile Include="Data.cs" />
     <Compile Include="Downloaders\AbstractDownloadState.cs" />

Wabbajack.Lib/zEditIntegration.cs

@@ -4,6 +4,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using Wabbajack.Common;
+using Wabbajack.Lib.CompilationSteps;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
 using File = Alphaleonis.Win32.Filesystem.File;
 using Path = Alphaleonis.Win32.Filesystem.Path;
@@ -29,75 +30,76 @@ namespace Wabbajack.Lib
             return null;
         }
 
-        public static Func<RawSourceFile, Directive> IncludezEditPatches(Compiler compiler)
+        public class IncludeZEditPatches : ACompilationStep
         {
-            var zEditPath = FindzEditPath(compiler);
+            private Dictionary<string, zEditMerge> _mergesIndexed;
-            var havezEdit = zEditPath != null;
 
-            Utils.Log(havezEdit ? $"Found zEdit at {zEditPath}" : $"zEdit not detected, disabling zEdit routines");
+            public IncludeZEditPatches(Compiler compiler) : base(compiler)
-
-            if (!havezEdit) return source => { return null; };
-
-            var merges = Directory.EnumerateFiles(Path.Combine(zEditPath, "profiles"),
-                    DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive)
-                .Where(f => f.EndsWith("\\merges.json"))
-                .SelectMany(f => f.FromJSON<List<zEditMerge>>())
-                .GroupBy(f => (f.name, f.filename));
-
-            merges.Where(m => m.Count() > 1)
-                .Do(m =>
-                {
-                    Utils.Warning(
-                        $"WARNING, you have two patches named {m.Key.name}\\{m.Key.filename} in your zEdit profiles. We'll pick one at random, this probably isn't what you want.");
-                });
-
-            var mergesIndexed =
-                merges.ToDictionary(
-                    m => Path.Combine(compiler.MO2Folder, "mods", m.Key.name, m.Key.filename),
-                    m => m.First());
-
-
-
-            return source =>
             {
-                if (mergesIndexed.TryGetValue(source.AbsolutePath, out var merge))
+                var zEditPath = FindzEditPath(compiler);
+                var havezEdit = zEditPath != null;
+
+                Utils.Log(havezEdit ? $"Found zEdit at {zEditPath}" : $"zEdit not detected, disabling zEdit routines");
+
+                if (!havezEdit)
                 {
-                    var result = source.EvolveTo<MergedPatch>();
+                    _mergesIndexed = new Dictionary<string, zEditMerge>();
-                    result.Sources = merge.plugins.Select(f =>
+                    return;
-                    {
-                        var abs_path = Path.Combine(f.dataFolder, f.filename);
-                        if (!File.Exists(abs_path))
-                            throw new InvalidDataException(
-                                $"File {abs_path} is required to build {merge.filename} but it doesn't exist");
-
-                        return new SourcePatch
-                        {
-                            RelativePath = abs_path.RelativeTo(compiler.MO2Folder),
-                            Hash = compiler.VFS[abs_path].Hash
-                        };
-                    }).ToList();
-
-                    var src_data = merge.plugins.Select(f => File.ReadAllBytes(Path.Combine(f.dataFolder, f.filename)))
-                        .ConcatArrays();
-
-                    var dst_data = File.ReadAllBytes(source.AbsolutePath);
-
-                    using (var ms = new MemoryStream())
-                    {
-                        Utils.CreatePatch(src_data, dst_data, ms);
-                        result.PatchID = compiler.IncludeFile(ms.ToArray());
-                    }
-
-                    return result;
-
                 }
-                return null;
-            };
 
+                var merges = Directory.EnumerateFiles(Path.Combine(zEditPath, "profiles"),
+                        DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive)
+                    .Where(f => f.EndsWith("\\merges.json"))
+                    .SelectMany(f => f.FromJSON<List<zEditMerge>>())
+                    .GroupBy(f => (f.name, f.filename));
 
+                merges.Where(m => m.Count() > 1)
+                    .Do(m =>
+                    {
+                        Utils.Warning(
+                            $"WARNING, you have two patches named {m.Key.name}\\{m.Key.filename} in your zEdit profiles. We'll pick one at random, this probably isn't what you want.");
+                    });
+
+                _mergesIndexed =
+                    merges.ToDictionary(
+                        m => Path.Combine(compiler.MO2Folder, "mods", m.Key.name, m.Key.filename),
+                        m => m.First());
+            }
+
+            public override Directive Run(RawSourceFile source)
+            {
+                if (!_mergesIndexed.TryGetValue(source.AbsolutePath, out var merge)) return null;
+                var result = source.EvolveTo<MergedPatch>();
+                result.Sources = merge.plugins.Select(f =>
+                {
+                    var abs_path = Path.Combine(f.dataFolder, f.filename);
+                    if (!File.Exists(abs_path))
+                        throw new InvalidDataException(
+                            $"File {abs_path} is required to build {merge.filename} but it doesn't exist");
+
+                    return new SourcePatch
+                    {
+                        RelativePath = abs_path.RelativeTo(_compiler.MO2Folder),
+                        Hash = _compiler.VFS[abs_path].Hash
+                    };
+                }).ToList();
+
+                var src_data = merge.plugins.Select(f => File.ReadAllBytes(Path.Combine(f.dataFolder, f.filename)))
+                    .ConcatArrays();
+
+                var dst_data = File.ReadAllBytes(source.AbsolutePath);
+
+                using (var ms = new MemoryStream())
+                {
+                    Utils.CreatePatch(src_data, dst_data, ms);
+                    result.PatchID = _compiler.IncludeFile(ms.ToArray());
+                }
+
+                return result;
+
+            }
         }
 
-
         class zEditMerge
         {
             public string name;