diff --git a/Wabbajack.Lib/CompilationSteps/ACompilationStep.cs b/Wabbajack.Lib/CompilationSteps/ACompilationStep.cs new file mode 100644 index 00000000..885efb43 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/ACompilationStep.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Wabbajack.Lib.CompilationSteps +{ + public abstract class ACompilationStep : ICompilationStep + { + protected Compiler _compiler; + + public ACompilationStep(Compiler compiler) + { + _compiler = compiler; + } + + public abstract Directive Run(RawSourceFile source); + } +} diff --git a/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs b/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs new file mode 100644 index 00000000..84f5b70e --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Alphaleonis.Win32.Filesystem; +using Compression.BSA; +using Wabbajack.Common; + +namespace Wabbajack.Lib.CompilationSteps +{ + class DeconstructBSAs : ACompilationStep + { + private readonly List _microstack; + private readonly List _microstackWithInclude; + private readonly IEnumerable _include_directly; + + public DeconstructBSAs(Compiler compiler) : base(compiler) + { + _include_directly = _compiler.ModInis.Where(kv => + { + var general = kv.Value.General; + if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE)) return true; + if (general.comments != null && general.comments.Contains(Consts.WABBAJACK_INCLUDE)) return true; + return false; + }) + .Select(kv => $"mods\\{kv.Key}\\") + .ToList(); + + _microstack = new List + { + new DirectMatch(_compiler), + new IncludePatches(_compiler), + new DropAll(_compiler) + }; + + _microstackWithInclude = new List + { + new DirectMatch(_compiler), + new IncludePatches(_compiler), + new IncludeAll(_compiler) + }; + + } + + public override Directive Run(RawSourceFile source) + { + if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path).ToLower())) return null; + + var defaultInclude = false; + if (source.Path.StartsWith("mods")) + { + if (_include_directly.Any(path => source.Path.StartsWith(path))) + { + defaultInclude = true; + } + } + + var source_files = source.File.FileInArchive; + + var stack = defaultInclude ? _microstackWithInclude : _microstack; + + var id = Guid.NewGuid().ToString(); + + var matches = source_files.PMap(e => Compiler.RunStack(stack, new RawSourceFile(e) + { + Path = Path.Combine(Consts.BSACreationDir, id, e.Paths.Last()) + })); + + + foreach (var match in matches) + { + if (match is IgnoredDirectly) + Utils.Error($"File required for BSA {source.Path} creation doesn't exist: {match.To}"); + _compiler.ExtraFiles.Add(match); + } + + CreateBSA directive; + using (var bsa = BSADispatch.OpenRead(source.AbsolutePath)) + { + directive = new CreateBSA + { + To = source.Path, + TempID = id, + State = bsa.State, + FileStates = bsa.Files.Select(f => f.State).ToList() + }; + } + + return directive; + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/DirectMatch.cs b/Wabbajack.Lib/CompilationSteps/DirectMatch.cs new file mode 100644 index 00000000..7021162a --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/DirectMatch.cs @@ -0,0 +1,29 @@ +using System.Linq; +using Alphaleonis.Win32.Filesystem; + +namespace Wabbajack.Lib.CompilationSteps +{ + class DirectMatch : ACompilationStep + { + public DirectMatch(Compiler compiler) : base(compiler) + { + } + + public override Directive Run(RawSourceFile source) + { + if (!_compiler.IndexedFiles.TryGetValue(source.Hash, out var found)) return null; + var result = source.EvolveTo(); + + var match = found.Where(f => + Path.GetFileName(f.Paths[f.Paths.Length - 1]) == Path.GetFileName(source.Path)) + .OrderBy(f => f.Paths.Length) + .FirstOrDefault() + ?? found.OrderBy(f => f.Paths.Length).FirstOrDefault(); + + result.ArchiveHashPath = match.MakeRelativePaths(); + + return result; + + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/DropAll.cs b/Wabbajack.Lib/CompilationSteps/DropAll.cs new file mode 100644 index 00000000..f2b6a31b --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/DropAll.cs @@ -0,0 +1,19 @@ +using Wabbajack.Common; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class DropAll : ACompilationStep + { + public DropAll(Compiler compiler) : base(compiler) + { + } + + public override Directive Run(RawSourceFile source) + { + var result = source.EvolveTo(); + result.Reason = "No Match in Stack"; + Utils.Log($"No match for: {source.Path}"); + return result; + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IStackStep.cs b/Wabbajack.Lib/CompilationSteps/IStackStep.cs new file mode 100644 index 00000000..8ed94405 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IStackStep.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Wabbajack.Lib.CompilationSteps +{ + public interface ICompilationStep + { + Directive Run(RawSourceFile source); + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreDisabledMods.cs b/Wabbajack.Lib/CompilationSteps/IgnoreDisabledMods.cs new file mode 100644 index 00000000..12bf4f9b --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IgnoreDisabledMods.cs @@ -0,0 +1,49 @@ +using System.Collections.Generic; +using System.Linq; +using Alphaleonis.Win32.Filesystem; +using Wabbajack.Common; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IgnoreDisabledMods : ACompilationStep + { + private readonly IEnumerable _allEnabledMods; + + public IgnoreDisabledMods(Compiler compiler) : base(compiler) + { + var alwaysEnabled = _compiler.ModInis.Where(f => IsAlwaysEnabled(f.Value)).Select(f => f.Key).ToHashSet(); + + _allEnabledMods = _compiler.SelectedProfiles + .SelectMany(p => File.ReadAllLines(Path.Combine(_compiler.MO2Folder, "profiles", p, "modlist.txt"))) + .Where(line => line.StartsWith("+") || line.EndsWith("_separator")) + .Select(line => line.Substring(1)) + .Concat(alwaysEnabled) + .Select(line => Path.Combine("mods", line) + "\\") + .ToList(); + } + + public override Directive Run(RawSourceFile source) + { + if (!source.Path.StartsWith("mods") || _allEnabledMods.Any(mod => source.Path.StartsWith(mod))) + return null; + var r = source.EvolveTo(); + r.Reason = "Disabled Mod"; + return r; + } + + + private static bool IsAlwaysEnabled(dynamic data) + { + if (data == null) + return false; + if (data.General != null && data.General.notes != null && + data.General.notes.Contains( + Consts.WABBAJACK_ALWAYS_ENABLE)) + return true; + if (data.General != null && data.General.comments != null && + data.General.notes.Contains(Consts.WABBAJACK_ALWAYS_ENABLE)) + return true; + return false; + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreEndsWith.cs b/Wabbajack.Lib/CompilationSteps/IgnoreEndsWith.cs new file mode 100644 index 00000000..f4fefc14 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IgnoreEndsWith.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IgnoreEndsWith : ACompilationStep + { + private readonly string _reason; + private readonly string _postfix; + + public IgnoreEndsWith(Compiler compiler, string postfix) : base(compiler) + { + _postfix = postfix; + _reason = $"Ignored because path ends with {postfix}"; + } + + public override Directive Run(RawSourceFile source) + { + if (!source.Path.EndsWith(_postfix)) return null; + var result = source.EvolveTo(); + result.Reason = _reason; + return result; + + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs b/Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs new file mode 100644 index 00000000..5db81608 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs @@ -0,0 +1,23 @@ +using Wabbajack.Common; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IgnoreGameFiles : ACompilationStep + { + private readonly string _startDir; + + public IgnoreGameFiles(Compiler compiler) : base(compiler) + { + _startDir = Consts.GameFolderFilesDir + "\\"; + } + + public override Directive Run(RawSourceFile source) + { + if (!source.Path.StartsWith(_startDir)) return null; + var i = source.EvolveTo(); + i.Reason = "Default game file"; + return i; + + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IgnorePathContains.cs b/Wabbajack.Lib/CompilationSteps/IgnorePathContains.cs new file mode 100644 index 00000000..ac8b752c --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IgnorePathContains.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IgnorePathContains : ACompilationStep + { + private readonly string _pattern; + private readonly string _reason; + + public IgnorePathContains(Compiler compiler, string pattern) : base(compiler) + { + _pattern = $"\\{pattern.Trim('\\')}\\"; + _reason = $"Ignored because path contains {_pattern}"; + } + + public override Directive Run(RawSourceFile source) + { + if (!source.Path.Contains(_pattern)) return null; + var result = source.EvolveTo(); + result.Reason = _reason; + return result; + + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreRegex.cs b/Wabbajack.Lib/CompilationSteps/IgnoreRegex.cs new file mode 100644 index 00000000..4636dcbb --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IgnoreRegex.cs @@ -0,0 +1,27 @@ +using System.Text.RegularExpressions; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IgnoreRegex : ACompilationStep + { + private readonly string _reason; + private string _pattern; + private readonly Regex _regex; + + public IgnoreRegex(Compiler compiler, string pattern) : base(compiler) + { + _pattern = pattern; + _reason = $"Ignored because path matches regex {pattern}"; + _regex = new Regex(pattern); + } + + public override Directive Run(RawSourceFile source) + { + if (!_regex.IsMatch(source.Path)) return null; + var result = source.EvolveTo(); + result.Reason = _reason; + return result; + + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreStartsWith.cs b/Wabbajack.Lib/CompilationSteps/IgnoreStartsWith.cs new file mode 100644 index 00000000..c0154232 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IgnoreStartsWith.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IgnoreStartsWith : ACompilationStep + { + private readonly string _reason; + private readonly string _prefix; + + public IgnoreStartsWith(Compiler compiler, string prefix) : base(compiler) + { + _prefix = prefix; + _reason = string.Format("Ignored because path starts with {0}", _prefix); + } + + public override Directive Run(RawSourceFile source) + { + if (source.Path.StartsWith(_prefix)) + { + var result = source.EvolveTo(); + result.Reason = _reason; + return result; + } + return null; + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreWabbajackInstallCruft.cs b/Wabbajack.Lib/CompilationSteps/IgnoreWabbajackInstallCruft.cs new file mode 100644 index 00000000..272f2446 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IgnoreWabbajackInstallCruft.cs @@ -0,0 +1,28 @@ +using System.Collections.Generic; +using System.Linq; +using Wabbajack.Common; + +namespace Wabbajack.Lib.CompilationSteps +{ + class IgnoreWabbajackInstallCruft : ACompilationStep + { + private readonly HashSet _cruftFiles; + + public IgnoreWabbajackInstallCruft(Compiler compiler) : base(compiler) + { + _cruftFiles = new HashSet + { + "7z.dll", "7z.exe", "vfs_staged_files\\", "nexus.key_cache", "patch_cache\\", + Consts.NexusCacheDirectory + "\\" + }; + } + + public override Directive Run(RawSourceFile source) + { + if (!_cruftFiles.Any(f => source.Path.StartsWith(f))) return null; + var result = source.EvolveTo(); + result.Reason = "Wabbajack Cruft file"; + return result; + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IncludeAll.cs b/Wabbajack.Lib/CompilationSteps/IncludeAll.cs new file mode 100644 index 00000000..f9ce8cff --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IncludeAll.cs @@ -0,0 +1,18 @@ +using Alphaleonis.Win32.Filesystem; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IncludeAll : ACompilationStep + { + public IncludeAll(Compiler compiler) : base(compiler) + { + } + + public override Directive Run(RawSourceFile source) + { + var inline = source.EvolveTo(); + inline.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath)); + return inline; + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IncludeAllConfigs.cs b/Wabbajack.Lib/CompilationSteps/IncludeAllConfigs.cs new file mode 100644 index 00000000..b34fb4e7 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IncludeAllConfigs.cs @@ -0,0 +1,20 @@ +using Alphaleonis.Win32.Filesystem; +using Wabbajack.Common; + +namespace Wabbajack.Lib.CompilationSteps +{ + class IncludeAllConfigs : ACompilationStep + { + public IncludeAllConfigs(Compiler compiler) : base(compiler) + { + } + + public override Directive Run(RawSourceFile source) + { + if (!Consts.ConfigFileExtensions.Contains(Path.GetExtension(source.Path))) return null; + var result = source.EvolveTo(); + result.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath)); + return result; + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IncludeDummyESPs.cs b/Wabbajack.Lib/CompilationSteps/IncludeDummyESPs.cs new file mode 100644 index 00000000..9df66e48 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IncludeDummyESPs.cs @@ -0,0 +1,30 @@ +using Alphaleonis.Win32.Filesystem; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IncludeDummyESPs : ACompilationStep + { + public IncludeDummyESPs(Compiler compiler) : base(compiler) + { + } + + public override Directive Run(RawSourceFile source) + { + if (Path.GetExtension(source.AbsolutePath) != ".esp" && + Path.GetExtension(source.AbsolutePath) != ".esm") return null; + + var bsa = Path.Combine(Path.GetDirectoryName(source.AbsolutePath), + Path.GetFileNameWithoutExtension(source.AbsolutePath) + ".bsa"); + var bsaTextures = Path.Combine(Path.GetDirectoryName(source.AbsolutePath), + Path.GetFileNameWithoutExtension(source.AbsolutePath) + " - Textures.bsa"); + var espSize = new FileInfo(source.AbsolutePath).Length; + + if (espSize > 250 || (!File.Exists(bsa) && !File.Exists(bsaTextures))) return null; + + var inline = source.EvolveTo(); + inline.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath)); + return inline; + + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IncludeLOOTFiles.cs b/Wabbajack.Lib/CompilationSteps/IncludeLOOTFiles.cs new file mode 100644 index 00000000..417defae --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IncludeLOOTFiles.cs @@ -0,0 +1,24 @@ +using Alphaleonis.Win32.Filesystem; +using Wabbajack.Common; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IncludeLootFiles : ACompilationStep + { + private readonly string _prefix; + + public IncludeLootFiles(Compiler compiler) : base(compiler) + { + _prefix = Consts.LOOTFolderFilesDir + "\\"; + } + + public override Directive Run(RawSourceFile source) + { + if (!source.Path.StartsWith(_prefix)) return null; + var result = source.EvolveTo(); + result.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath).ToBase64()); + return result; + + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IncludeModIniData.cs b/Wabbajack.Lib/CompilationSteps/IncludeModIniData.cs new file mode 100644 index 00000000..111dbbcf --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IncludeModIniData.cs @@ -0,0 +1,20 @@ +using Alphaleonis.Win32.Filesystem; + +namespace Wabbajack.Lib.CompilationSteps +{ + class IncludeModIniData : ACompilationStep + { + public IncludeModIniData(Compiler compiler) : base(compiler) + { + } + + public override Directive Run(RawSourceFile source) + { + if (!source.Path.StartsWith("mods\\") || !source.Path.EndsWith("\\meta.ini")) return null; + var e = source.EvolveTo(); + e.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath)); + return e; + + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IncludeOtherProfiles.cs b/Wabbajack.Lib/CompilationSteps/IncludeOtherProfiles.cs new file mode 100644 index 00000000..5d720c73 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IncludeOtherProfiles.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IgnoreOtherProfiles : ACompilationStep + { + private readonly IEnumerable _profiles; + + public IgnoreOtherProfiles(Compiler compiler) : base(compiler) + { + _profiles = _compiler.SelectedProfiles + .Select(p => Path.Combine("profiles", p) + "\\") + .ToList(); + } + + public override Directive Run(RawSourceFile source) + { + if (!source.Path.StartsWith("profiles\\")) return null; + if (_profiles.Any(profile => source.Path.StartsWith(profile))) return null; + var c = source.EvolveTo(); + c.Reason = "File not for selected profiles"; + return c; + + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IncludePatches.cs b/Wabbajack.Lib/CompilationSteps/IncludePatches.cs new file mode 100644 index 00000000..f410b27f --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IncludePatches.cs @@ -0,0 +1,41 @@ +using System.Collections.Generic; +using System.Linq; +using Alphaleonis.Win32.Filesystem; +using VFS; +using Wabbajack.Common; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IncludePatches : ACompilationStep + { + private readonly Dictionary> _indexed; + + public IncludePatches(Compiler compiler) : base(compiler) + { + _indexed = _compiler.IndexedFiles.Values + .SelectMany(f => f) + .GroupBy(f => Path.GetFileName(Enumerable.Last(f.Paths)).ToLower()) + .ToDictionary(f => f.Key); + } + + public override Directive Run(RawSourceFile source) + { + if (!_indexed.TryGetValue(Path.GetFileName(Enumerable.Last(source.File.Paths).ToLower()), out var value)) + return null; + + var found = value.OrderByDescending(f => (f.TopLevelArchive ?? f).LastModified).First(); + + var e = source.EvolveTo(); + e.ArchiveHashPath = found.MakeRelativePaths(); + e.To = source.Path; + e.Hash = source.File.Hash; + + Utils.TryGetPatch(found.Hash, source.File.Hash, out var data); + + if (data != null) + e.PatchID = _compiler.IncludeFile(data); + + return e; + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs b/Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs new file mode 100644 index 00000000..6d0ac334 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; + +namespace Wabbajack.Lib.CompilationSteps +{ + class IncludePropertyFiles : ACompilationStep + { + public IncludePropertyFiles(Compiler compiler) : base(compiler) + { + } + + public override Directive Run(RawSourceFile source) + { + var files = new HashSet + { + _compiler.ModListImage, _compiler.ModListReadme + }; + if (!files.Any(f => source.AbsolutePath.Equals(f))) return null; + if (!File.Exists(source.AbsolutePath)) return null; + var isBanner = source.AbsolutePath == _compiler.ModListImage; + //var isReadme = source.AbsolutePath == ModListReadme; + var result = source.EvolveTo(); + result.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath)); + if (isBanner) + { + result.Type = PropertyType.Banner; + _compiler.ModListImage = result.SourceDataID; + } + else + { + result.Type = PropertyType.Readme; + _compiler.ModListReadme = result.SourceDataID; + } + return result; + } + + + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IncludeRegex.cs b/Wabbajack.Lib/CompilationSteps/IncludeRegex.cs new file mode 100644 index 00000000..30c25c16 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IncludeRegex.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IncludeRegex : ACompilationStep + { + private readonly Regex _regex; + + public IncludeRegex(Compiler compiler, string pattern) : base(compiler) + { + _regex = new Regex(pattern); + } + + public override Directive Run(RawSourceFile source) + { + if (!_regex.IsMatch(source.Path)) return null; + + var result = source.EvolveTo(); + result.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath)); + return result; + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IncludeStubbedConfigfiles.cs b/Wabbajack.Lib/CompilationSteps/IncludeStubbedConfigfiles.cs new file mode 100644 index 00000000..650cf725 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IncludeStubbedConfigfiles.cs @@ -0,0 +1,42 @@ +using System.Text; +using Alphaleonis.Win32.Filesystem; +using Wabbajack.Common; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IncludeStubbedConfigFiles : ACompilationStep + { + public IncludeStubbedConfigFiles(Compiler compiler) : base(compiler) + { + } + + public override Directive Run(RawSourceFile source) + { + return Consts.ConfigFileExtensions.Contains(Path.GetExtension(source.Path)) ? RemapFile(source) : null; + } + + private Directive RemapFile(RawSourceFile source) + { + var data = File.ReadAllText(source.AbsolutePath); + var originalData = data; + + data = data.Replace(_compiler.GamePath, Consts.GAME_PATH_MAGIC_BACK); + data = data.Replace(_compiler.GamePath.Replace("\\", "\\\\"), Consts.GAME_PATH_MAGIC_DOUBLE_BACK); + data = data.Replace(_compiler.GamePath.Replace("\\", "/"), Consts.GAME_PATH_MAGIC_FORWARD); + + data = data.Replace(_compiler.MO2Folder, Consts.MO2_PATH_MAGIC_BACK); + data = data.Replace(_compiler.MO2Folder.Replace("\\", "\\\\"), Consts.MO2_PATH_MAGIC_DOUBLE_BACK); + data = data.Replace(_compiler.MO2Folder.Replace("\\", "/"), Consts.MO2_PATH_MAGIC_FORWARD); + + data = data.Replace(_compiler.MO2DownloadsFolder, Consts.DOWNLOAD_PATH_MAGIC_BACK); + data = data.Replace(_compiler.MO2DownloadsFolder.Replace("\\", "\\\\"), Consts.DOWNLOAD_PATH_MAGIC_DOUBLE_BACK); + data = data.Replace(_compiler.MO2DownloadsFolder.Replace("\\", "/"), Consts.DOWNLOAD_PATH_MAGIC_FORWARD); + + if (data == originalData) + return null; + var result = source.EvolveTo(); + result.SourceDataID = _compiler.IncludeFile(Encoding.UTF8.GetBytes(data)); + return result; + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs b/Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs new file mode 100644 index 00000000..6c6881e2 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs @@ -0,0 +1,39 @@ +using System.Collections.Generic; +using System.Linq; +using Alphaleonis.Win32.Filesystem; + +namespace Wabbajack.Lib.CompilationSteps +{ + class IncludeTaggedMods : ACompilationStep + { + private readonly IEnumerable _includeDirectly; + private string _tag; + + + public IncludeTaggedMods(Compiler compiler, string tag) : base(compiler) + { + _tag = tag; + _includeDirectly = _compiler.ModInis.Where(kv => + { + var general = kv.Value.General; + if (general.notes != null && general.notes.Contains(_tag)) + return true; + return general.comments != null && general.comments.Contains(_tag); + }).Select(kv => $"mods\\{kv.Key}\\"); + } + + public override Directive Run(RawSourceFile source) + { + if (!source.Path.StartsWith("mods")) return null; + foreach (var modpath in _includeDirectly) + { + if (!source.Path.StartsWith(modpath)) continue; + var result = source.EvolveTo(); + result.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath)); + return result; + } + + return null; + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IncludeThisProfile.cs b/Wabbajack.Lib/CompilationSteps/IncludeThisProfile.cs new file mode 100644 index 00000000..7fbf8839 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IncludeThisProfile.cs @@ -0,0 +1,39 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Alphaleonis.Win32.Filesystem; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IncludeThisProfile : ACompilationStep + { + private readonly IEnumerable _correctProfiles; + + public IncludeThisProfile(Compiler compiler) : base(compiler) + { + _correctProfiles = _compiler.SelectedProfiles.Select(p => Path.Combine("profiles", p) + "\\").ToList(); + } + + public override Directive Run(RawSourceFile source) + { + if (_correctProfiles.Any(p => source.Path.StartsWith(p))) + { + var data = source.Path.EndsWith("\\modlist.txt") ? ReadAndCleanModlist(source.AbsolutePath) : File.ReadAllBytes(source.AbsolutePath); + + var e = source.EvolveTo(); + e.SourceDataID = _compiler.IncludeFile(data); + return e; + } + + return null; + } + private static byte[] ReadAndCleanModlist(string absolutePath) + { + var lines = File.ReadAllLines(absolutePath); + lines = (from line in lines + where !(line.StartsWith("-") && !line.EndsWith("_separator")) + select line).ToArray(); + return Encoding.UTF8.GetBytes(string.Join("\r\n", lines)); + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/PatchStockESMs.cs b/Wabbajack.Lib/CompilationSteps/PatchStockESMs.cs new file mode 100644 index 00000000..2f77521d --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/PatchStockESMs.cs @@ -0,0 +1,40 @@ +using System.IO; +using Wabbajack.Common; +using File = Alphaleonis.Win32.Filesystem.File; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.Lib.CompilationSteps +{ + class PatchStockESMs : ACompilationStep + { + public PatchStockESMs(Compiler compiler) : base(compiler) + { + } + + public override Directive Run(RawSourceFile source) + { + var filename = Path.GetFileName(source.Path); + var gameFile = Path.Combine(_compiler.GamePath, "Data", filename); + if (!Consts.GameESMs.Contains(filename) || !source.Path.StartsWith("mods\\") || + !File.Exists(gameFile)) return null; + + Utils.Log( + $"A ESM named {filename} was found in a mod that shares a name with a core game ESMs, it is assumed this is a cleaned ESM and it will be binary patched."); + var result = source.EvolveTo(); + result.SourceESMHash = _compiler.VFS.Lookup(gameFile).Hash; + + Utils.Status($"Generating patch of {filename}"); + using (var ms = new MemoryStream()) + { + Utils.CreatePatch(File.ReadAllBytes(gameFile), File.ReadAllBytes(source.AbsolutePath), ms); + var data = ms.ToArray(); + result.SourceDataID = _compiler.IncludeFile(data); + Utils.Log($"Generated a {data.Length} byte patch for {filename}"); + + } + + return result; + + } + } +} diff --git a/Wabbajack.Lib/Compiler.cs b/Wabbajack.Lib/Compiler.cs index 98aeced9..c1ab4edf 100644 --- a/Wabbajack.Lib/Compiler.cs +++ b/Wabbajack.Lib/Compiler.cs @@ -1,6 +1,7 @@ using CommonMark; using Compression.BSA; using System; +using System.CodeDom; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; @@ -12,6 +13,7 @@ using System.Text; using System.Text.RegularExpressions; using VFS; using Wabbajack.Common; +using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; @@ -498,12 +500,12 @@ namespace Wabbajack.Lib } - private Directive RunStack(IEnumerable> stack, RawSourceFile source) + public static Directive RunStack(IEnumerable stack, RawSourceFile source) { - Status($"Compiling {source.Path}"); - foreach (var f in stack) + Utils.Status($"Compiling {source.Path}"); + foreach (var step in stack) { - var result = f(source); + var result = step.Run(source); if (result != null) return result; } @@ -517,631 +519,65 @@ namespace Wabbajack.Lib /// result included into the pack /// /// - private IEnumerable> MakeStack() + private IEnumerable MakeStack() { Info("Generating compilation stack"); - return new List> + return new List { - IncludePropertyFiles(), - IgnoreStartsWith("logs\\"), - IncludeRegex("^downloads\\\\.*\\.meta"), - IgnoreStartsWith("downloads\\"), - IgnoreStartsWith("webcache\\"), - IgnoreStartsWith("overwrite\\"), - IgnorePathContains("temporary_logs"), - IgnorePathContains("GPUCache"), - IgnorePathContains("SSEEdit Cache"), - IgnoreEndsWith(".pyc"), - IgnoreEndsWith(".log"), - IgnoreOtherProfiles(), - IgnoreDisabledMods(), - IncludeThisProfile(), + new IncludePropertyFiles(this), + new IgnoreStartsWith(this,"logs\\"), + new IncludeRegex(this, "^downloads\\\\.*\\.meta"), + new IgnoreStartsWith(this, "downloads\\"), + new IgnoreStartsWith(this,"webcache\\"), + new IgnoreStartsWith(this, "overwrite\\"), + new IgnorePathContains(this,"temporary_logs"), + new IgnorePathContains(this, "GPUCache"), + new IgnorePathContains(this, "SSEEdit Cache"), + new IgnoreEndsWith(this, ".pyc"), + new IgnoreEndsWith(this, ".log"), + new IgnoreOtherProfiles(this), + new IgnoreDisabledMods(this), + new IncludeThisProfile(this), // Ignore the ModOrganizer.ini file it contains info created by MO2 on startup - IncludeStubbedConfigFiles(), - IncludeLootFiles(), - IgnoreStartsWith(Path.Combine(Consts.GameFolderFilesDir, "Data")), - IgnoreStartsWith(Path.Combine(Consts.GameFolderFilesDir, "Papyrus Compiler")), - IgnoreStartsWith(Path.Combine(Consts.GameFolderFilesDir, "Skyrim")), - IgnoreRegex(Consts.GameFolderFilesDir + "\\\\.*\\.bsa"), - IncludeModIniData(), - DirectMatch(), - IncludeTaggedFiles(Consts.WABBAJACK_INCLUDE), - DeconstructBSAs(), // Deconstruct BSAs before building patches so we don't generate massive patch files - IncludePatches(), - IncludeDummyESPs(), + new IncludeStubbedConfigFiles(this), + new IncludeLootFiles(this), + new IgnoreStartsWith(this, Path.Combine(Consts.GameFolderFilesDir, "Data")), + new IgnoreStartsWith(this, Path.Combine(Consts.GameFolderFilesDir, "Papyrus Compiler")), + new IgnoreStartsWith(this, Path.Combine(Consts.GameFolderFilesDir, "Skyrim")), + new IgnoreRegex(this, Consts.GameFolderFilesDir + "\\\\.*\\.bsa"), + new IncludeModIniData(this), + new DirectMatch(this), + new IncludeTaggedMods(this, Consts.WABBAJACK_INCLUDE), + new DeconstructBSAs(this), // Deconstruct BSAs before building patches so we don't generate massive patch files + new IncludePatches(this), + new IncludeDummyESPs(this), // If we have no match at this point for a game folder file, skip them, we can't do anything about them - IgnoreGameFiles(), + new IgnoreGameFiles(this), // There are some types of files that will error the compilation, because they're created on-the-fly via tools // so if we don't have a match by this point, just drop them. - IgnoreEndsWith(".ini"), - IgnoreEndsWith(".html"), - IgnoreEndsWith(".txt"), + new IgnoreEndsWith(this, ".ini"), + new IgnoreEndsWith(this, ".html"), + new IgnoreEndsWith(this, ".txt"), // Don't know why, but this seems to get copied around a bit - IgnoreEndsWith("HavokBehaviorPostProcess.exe"), + new IgnoreEndsWith(this, "HavokBehaviorPostProcess.exe"), // Theme file MO2 downloads somehow - IgnoreEndsWith("splash.png"), + new IgnoreEndsWith(this, "splash.png"), - IgnoreEndsWith(".bin"), - IgnoreEndsWith(".refcache"), + new IgnoreEndsWith(this, ".bin"), + new IgnoreEndsWith(this, ".refcache"), - IgnoreWabbajackInstallCruft(), + new IgnoreWabbajackInstallCruft(this), - PatchStockESMs(), + new PatchStockESMs(this), - IncludeAllConfigs(), - IncludeTaggedFiles(Consts.WABBAJACK_NOMATCH_INCLUDE), - zEditIntegration.IncludezEditPatches(this), + new IncludeAllConfigs(this), + new IncludeTaggedMods(this, Consts.WABBAJACK_NOMATCH_INCLUDE), + new zEditIntegration.IncludeZEditPatches(this), - DropAll() - }; - } - - private Func IncludePropertyFiles() - { - return source => - { - var files = new HashSet - { - ModListImage, ModListReadme - }; - if (!files.Any(f => source.AbsolutePath.Equals(f))) return null; - if (!File.Exists(source.AbsolutePath)) return null; - var isBanner = source.AbsolutePath == ModListImage; - //var isReadme = source.AbsolutePath == ModListReadme; - var result = source.EvolveTo(); - result.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath)); - if (isBanner) - { - result.Type = PropertyType.Banner; - ModListImage = result.SourceDataID; - } - else - { - result.Type = PropertyType.Readme; - ModListReadme = result.SourceDataID; - } - return result; - }; - } - - private Func IgnoreWabbajackInstallCruft() - { - var cruft_files = new HashSet - { - "7z.dll", "7z.exe", "vfs_staged_files\\", "nexus.key_cache", "patch_cache\\", - Consts.NexusCacheDirectory + "\\" - }; - return source => - { - if (!cruft_files.Any(f => source.Path.StartsWith(f))) return null; - var result = source.EvolveTo(); - result.Reason = "Wabbajack Cruft file"; - return result; - }; - } - - private Func IncludeAllConfigs() - { - return source => - { - if (!Consts.ConfigFileExtensions.Contains(Path.GetExtension(source.Path))) return null; - var result = source.EvolveTo(); - result.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath)); - return result; - }; - } - - private Func PatchStockESMs() - { - return source => - { - var filename = Path.GetFileName(source.Path); - var game_file = Path.Combine(GamePath, "Data", filename); - if (Consts.GameESMs.Contains(filename) && source.Path.StartsWith("mods\\") && File.Exists(game_file)) - { - Info( - $"A ESM named {filename} was found in a mod that shares a name with a core game ESMs, it is assumed this is a cleaned ESM and it will be binary patched."); - var result = source.EvolveTo(); - result.SourceESMHash = VFS.Lookup(game_file).Hash; - - Status($"Generating patch of {filename}"); - using (var ms = new MemoryStream()) - { - Utils.CreatePatch(File.ReadAllBytes(game_file), File.ReadAllBytes(source.AbsolutePath), ms); - var data = ms.ToArray(); - result.SourceDataID = IncludeFile(data); - Info($"Generated a {data.Length} byte patch for {filename}"); - - } - - - return result; - } - - return null; - }; - } - - - private Func IncludeLootFiles() - { - var prefix = Consts.LOOTFolderFilesDir + "\\"; - return source => - { - if (source.Path.StartsWith(prefix)) - { - var result = source.EvolveTo(); - result.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath).ToBase64()); - return result; - } - - return null; - }; - } - - private Func IncludeStubbedConfigFiles() - { - return source => - { - if (Consts.ConfigFileExtensions.Contains(Path.GetExtension(source.Path))) - return RemapFile(source, GamePath); - return null; - }; - } - - private Directive RemapFile(RawSourceFile source, string gamePath) - { - var data = File.ReadAllText(source.AbsolutePath); - var original_data = data; - - data = data.Replace(GamePath, Consts.GAME_PATH_MAGIC_BACK); - data = data.Replace(GamePath.Replace("\\", "\\\\"), Consts.GAME_PATH_MAGIC_DOUBLE_BACK); - data = data.Replace(GamePath.Replace("\\", "/"), Consts.GAME_PATH_MAGIC_FORWARD); - - data = data.Replace(MO2Folder, Consts.MO2_PATH_MAGIC_BACK); - data = data.Replace(MO2Folder.Replace("\\", "\\\\"), Consts.MO2_PATH_MAGIC_DOUBLE_BACK); - data = data.Replace(MO2Folder.Replace("\\", "/"), Consts.MO2_PATH_MAGIC_FORWARD); - - data = data.Replace(MO2DownloadsFolder, Consts.DOWNLOAD_PATH_MAGIC_BACK); - data = data.Replace(MO2DownloadsFolder.Replace("\\", "\\\\"), Consts.DOWNLOAD_PATH_MAGIC_DOUBLE_BACK); - data = data.Replace(MO2DownloadsFolder.Replace("\\", "/"), Consts.DOWNLOAD_PATH_MAGIC_FORWARD); - - if (data == original_data) - return null; - var result = source.EvolveTo(); - result.SourceDataID = IncludeFile(Encoding.UTF8.GetBytes(data)); - return result; - } - - private Func IgnorePathContains(string v) - { - v = $"\\{v.Trim('\\')}\\"; - var reason = $"Ignored because path contains {v}"; - return source => - { - if (source.Path.Contains(v)) - { - var result = source.EvolveTo(); - result.Reason = reason; - return result; - } - - return null; - }; - } - - - /// - /// If a user includes WABBAJACK_INCLUDE directly in the notes or comments of a mod, the contents of that - /// mod will be inlined into the installer. USE WISELY. - /// - /// - private Func IncludeTaggedFiles(string tag) - { - var include_directly = ModInis.Where(kv => - { - var general = kv.Value.General; - if (general.notes != null && general.notes.Contains(tag)) - return true; - if (general.comments != null && general.comments.Contains(tag)) - return true; - return false; - }).Select(kv => $"mods\\{kv.Key}\\"); - - - return source => - { - if (source.Path.StartsWith("mods")) - foreach (var modpath in include_directly) - if (source.Path.StartsWith(modpath)) - { - var result = source.EvolveTo(); - result.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath)); - return result; - } - - return null; - }; - } - - - /// - /// Some tools like the Cathedral Asset Optimizer will create dummy ESPs whos only existance is to make - /// sure a BSA with the same name is loaded. We don't have a good way to detect these, but if an ESP is - /// less than 100 bytes in size and shares a name with a BSA it's a pretty good chance that it's a dummy - /// and the contents are generated. - /// - /// - private Func IncludeDummyESPs() - { - return source => - { - if (Path.GetExtension(source.AbsolutePath) == ".esp" || Path.GetExtension(source.AbsolutePath) == ".esm") - { - var bsa = Path.Combine(Path.GetDirectoryName(source.AbsolutePath), - Path.GetFileNameWithoutExtension(source.AbsolutePath) + ".bsa"); - var bsa_textures = Path.Combine(Path.GetDirectoryName(source.AbsolutePath), - Path.GetFileNameWithoutExtension(source.AbsolutePath) + " - Textures.bsa"); - var esp_size = new FileInfo(source.AbsolutePath).Length; - if (esp_size <= 250 && (File.Exists(bsa) || File.Exists(bsa_textures))) - { - var inline = source.EvolveTo(); - inline.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath)); - return inline; - } - } - - return null; - }; - } - - - /// - /// This function will search for a way to create a BSA in the installed mod list by assembling it from files - /// found in archives. To do this we hash all the files in side the BSA then try to find matches and patches for - /// all of the files. - /// - /// - private Func DeconstructBSAs() - { - var include_directly = ModInis.Where(kv => - { - var general = kv.Value.General; - if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE)) - return true; - if (general.comments != null && general.comments.Contains(Consts.WABBAJACK_INCLUDE)) - return true; - return false; - }).Select(kv => $"mods\\{kv.Key}\\"); - - var microstack = new List> - { - DirectMatch(), - IncludePatches(), - DropAll() - }; - - var microstack_with_include = new List> - { - DirectMatch(), - IncludePatches(), - IncludeALL() - }; - - - return source => - { - if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path).ToLower())) return null; - - var default_include = false; - if (source.Path.StartsWith("mods")) - foreach (var modpath in include_directly) - if (source.Path.StartsWith(modpath)) - { - default_include = true; - break; - } - - var source_files = source.File.FileInArchive; - - var stack = default_include ? microstack_with_include : microstack; - - var id = Guid.NewGuid().ToString(); - - var matches = source_files.PMap(e => RunStack(stack, new RawSourceFile(e) - { - Path = Path.Combine(Consts.BSACreationDir, id, e.Paths.Last()) - })); - - - foreach (var match in matches) - { - if (match is IgnoredDirectly) Error($"File required for BSA {source.Path} creation doesn't exist: {match.To}"); - ExtraFiles.Add(match); - } - - ; - - CreateBSA directive; - using (var bsa = BSADispatch.OpenRead(source.AbsolutePath)) - { - directive = new CreateBSA - { - To = source.Path, - TempID = id, - State = bsa.State, - FileStates = bsa.Files.Select(f => f.State).ToList() - }; - } - - return directive; - }; - } - - private Func IncludeALL() - { - return source => - { - var inline = source.EvolveTo(); - inline.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath)); - return inline; - }; - } - - private Func IgnoreDisabledMods() - { - var always_enabled = ModInis.Where(f => IsAlwaysEnabled(f.Value)).Select(f => f.Key).ToHashSet(); - - var all_enabled_mods = SelectedProfiles - .SelectMany(p => File.ReadAllLines(Path.Combine(MO2Folder, "profiles", p, "modlist.txt"))) - .Where(line => line.StartsWith("+") || line.EndsWith("_separator")) - .Select(line => line.Substring(1)) - .Concat(always_enabled) - .Select(line => Path.Combine("mods", line) + "\\") - .ToList(); - - return source => - { - if (!source.Path.StartsWith("mods") || all_enabled_mods.Any(mod => source.Path.StartsWith(mod))) - return null; - var r = source.EvolveTo(); - r.Reason = "Disabled Mod"; - return r; - }; - } - - private static bool IsAlwaysEnabled(dynamic data) - { - if (data == null) - return false; - if (data.General != null && data.General.notes != null && - data.General.notes.Contains(Consts.WABBAJACK_ALWAYS_ENABLE)) - return true; - if (data.General != null && data.General.comments != null && - data.General.notes.Contains(Consts.WABBAJACK_ALWAYS_ENABLE)) - return true; - return false; - } - - /// - /// This matches files based purely on filename, and then creates a binary patch. - /// In practice this is fine, because a single file tends to only come from one archive. - /// - /// - private Func IncludePatches() - { - var indexed = IndexedFiles.Values - .SelectMany(f => f) - .GroupBy(f => Path.GetFileName(f.Paths.Last()).ToLower()) - .ToDictionary(f => f.Key); - - return source => - { - if (!indexed.TryGetValue(Path.GetFileName(source.File.Paths.Last().ToLower()), out var value)) - return null; - - var found = value.OrderByDescending(f => (f.TopLevelArchive ?? f).LastModified).First(); - - var e = source.EvolveTo(); - e.ArchiveHashPath = found.MakeRelativePaths(); - e.To = source.Path; - e.Hash = source.File.Hash; - - Utils.TryGetPatch(found.Hash, source.File.Hash, out var data); - - if (data != null) - e.PatchID = IncludeFile(data); - - return e; - }; - } - - private Func IncludeModIniData() - { - return source => - { - if (source.Path.StartsWith("mods\\") && source.Path.EndsWith("\\meta.ini")) - { - var e = source.EvolveTo(); - e.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath)); - return e; - } - - return null; - }; - } - - private Func IgnoreGameFiles() - { - var start_dir = Consts.GameFolderFilesDir + "\\"; - return source => - { - if (source.Path.StartsWith(start_dir)) - { - var i = source.EvolveTo(); - i.Reason = "Default game file"; - return i; - } - - return null; - }; - } - - private Func IncludeThisProfile() - { - var correct_profiles = SelectedProfiles.Select(p => Path.Combine("profiles", p) + "\\").ToList(); - - return source => - { - if (correct_profiles.Any(p => source.Path.StartsWith(p))) - { - byte[] data; - if (source.Path.EndsWith("\\modlist.txt")) - data = ReadAndCleanModlist(source.AbsolutePath); - else - data = File.ReadAllBytes(source.AbsolutePath); - - var e = source.EvolveTo(); - e.SourceDataID = IncludeFile(data); - return e; - } - - return null; - }; - } - - private byte[] ReadAndCleanModlist(string absolutePath) - { - var lines = File.ReadAllLines(absolutePath); - lines = (from line in lines - where !(line.StartsWith("-") && !line.EndsWith("_separator")) - select line).ToArray(); - return Encoding.UTF8.GetBytes(string.Join("\r\n", lines)); - } - - private Func IgnoreOtherProfiles() - { - var profiles = SelectedProfiles - .Select(p => Path.Combine("profiles", p) + "\\") - .ToList(); - - return source => - { - if (source.Path.StartsWith("profiles\\")) - { - if (profiles.Any(profile => source.Path.StartsWith(profile))) return null; - var c = source.EvolveTo(); - c.Reason = "File not for selected profiles"; - return c; - } - - return null; - }; - } - - private Func IgnoreEndsWith(string v) - { - var reason = string.Format("Ignored because path ends with {0}", v); - return source => - { - if (source.Path.EndsWith(v)) - { - var result = source.EvolveTo(); - result.Reason = reason; - return result; - } - - return null; - }; - } - - private Func IgnoreRegex(string p) - { - var reason = string.Format("Ignored because path matches regex {0}", p); - var regex = new Regex(p); - return source => - { - if (regex.IsMatch(source.Path)) - { - var result = source.EvolveTo(); - result.Reason = reason; - return result; - } - - return null; - }; - } - - - private Func IncludeRegex(string pattern) - { - var regex = new Regex(pattern); - return source => - { - if (regex.IsMatch(source.Path)) - { - var result = source.EvolveTo(); - result.SourceDataID = IncludeFile(File.ReadAllBytes(source.AbsolutePath)); - return result; - } - - return null; - }; - } - - - private Func DropAll() - { - return source => - { - var result = source.EvolveTo(); - result.Reason = "No Match in Stack"; - Info($"No match for: {source.Path}"); - return result; - }; - } - - private Func DirectMatch() - { - return source => - { - if (IndexedFiles.TryGetValue(source.Hash, out var found)) - { - var result = source.EvolveTo(); - - var match = found.Where(f => - Path.GetFileName(f.Paths[f.Paths.Length - 1]) == Path.GetFileName(source.Path)) - .OrderBy(f => f.Paths.Length) - .FirstOrDefault(); - - if (match == null) - match = found.OrderBy(f => f.Paths.Length).FirstOrDefault(); - - result.ArchiveHashPath = match.MakeRelativePaths(); - - return result; - } - - return null; - }; - } - - private Func IgnoreStartsWith(string v) - { - var reason = string.Format("Ignored because path starts with {0}", v); - return source => - { - if (source.Path.StartsWith(v)) - { - var result = source.EvolveTo(); - result.Reason = reason; - return result; - } - - return null; + new DropAll(this) }; } diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 8f1e52bc..3dfd8ba2 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -119,6 +119,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Wabbajack.Lib/zEditIntegration.cs b/Wabbajack.Lib/zEditIntegration.cs index 7b0a787b..feaca053 100644 --- a/Wabbajack.Lib/zEditIntegration.cs +++ b/Wabbajack.Lib/zEditIntegration.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using Wabbajack.Common; +using Wabbajack.Lib.CompilationSteps; using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; using Path = Alphaleonis.Win32.Filesystem.Path; @@ -29,75 +30,76 @@ namespace Wabbajack.Lib return null; } - public static Func IncludezEditPatches(Compiler compiler) + public class IncludeZEditPatches : ACompilationStep { - var zEditPath = FindzEditPath(compiler); - var havezEdit = zEditPath != null; + private Dictionary _mergesIndexed; - Utils.Log(havezEdit ? $"Found zEdit at {zEditPath}" : $"zEdit not detected, disabling zEdit routines"); - - if (!havezEdit) return source => { return null; }; - - var merges = Directory.EnumerateFiles(Path.Combine(zEditPath, "profiles"), - DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive) - .Where(f => f.EndsWith("\\merges.json")) - .SelectMany(f => f.FromJSON>()) - .GroupBy(f => (f.name, f.filename)); - - merges.Where(m => m.Count() > 1) - .Do(m => - { - Utils.Warning( - $"WARNING, you have two patches named {m.Key.name}\\{m.Key.filename} in your zEdit profiles. We'll pick one at random, this probably isn't what you want."); - }); - - var mergesIndexed = - merges.ToDictionary( - m => Path.Combine(compiler.MO2Folder, "mods", m.Key.name, m.Key.filename), - m => m.First()); - - - - return source => + public IncludeZEditPatches(Compiler compiler) : base(compiler) { - if (mergesIndexed.TryGetValue(source.AbsolutePath, out var merge)) + var zEditPath = FindzEditPath(compiler); + var havezEdit = zEditPath != null; + + Utils.Log(havezEdit ? $"Found zEdit at {zEditPath}" : $"zEdit not detected, disabling zEdit routines"); + + if (!havezEdit) { - var result = source.EvolveTo(); - result.Sources = merge.plugins.Select(f => - { - var abs_path = Path.Combine(f.dataFolder, f.filename); - if (!File.Exists(abs_path)) - throw new InvalidDataException( - $"File {abs_path} is required to build {merge.filename} but it doesn't exist"); - - return new SourcePatch - { - RelativePath = abs_path.RelativeTo(compiler.MO2Folder), - Hash = compiler.VFS[abs_path].Hash - }; - }).ToList(); - - var src_data = merge.plugins.Select(f => File.ReadAllBytes(Path.Combine(f.dataFolder, f.filename))) - .ConcatArrays(); - - var dst_data = File.ReadAllBytes(source.AbsolutePath); - - using (var ms = new MemoryStream()) - { - Utils.CreatePatch(src_data, dst_data, ms); - result.PatchID = compiler.IncludeFile(ms.ToArray()); - } - - return result; - + _mergesIndexed = new Dictionary(); + return; } - return null; - }; + var merges = Directory.EnumerateFiles(Path.Combine(zEditPath, "profiles"), + DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive) + .Where(f => f.EndsWith("\\merges.json")) + .SelectMany(f => f.FromJSON>()) + .GroupBy(f => (f.name, f.filename)); + merges.Where(m => m.Count() > 1) + .Do(m => + { + Utils.Warning( + $"WARNING, you have two patches named {m.Key.name}\\{m.Key.filename} in your zEdit profiles. We'll pick one at random, this probably isn't what you want."); + }); + + _mergesIndexed = + merges.ToDictionary( + m => Path.Combine(compiler.MO2Folder, "mods", m.Key.name, m.Key.filename), + m => m.First()); + } + + public override Directive Run(RawSourceFile source) + { + if (!_mergesIndexed.TryGetValue(source.AbsolutePath, out var merge)) return null; + var result = source.EvolveTo(); + result.Sources = merge.plugins.Select(f => + { + var abs_path = Path.Combine(f.dataFolder, f.filename); + if (!File.Exists(abs_path)) + throw new InvalidDataException( + $"File {abs_path} is required to build {merge.filename} but it doesn't exist"); + + return new SourcePatch + { + RelativePath = abs_path.RelativeTo(_compiler.MO2Folder), + Hash = _compiler.VFS[abs_path].Hash + }; + }).ToList(); + + var src_data = merge.plugins.Select(f => File.ReadAllBytes(Path.Combine(f.dataFolder, f.filename))) + .ConcatArrays(); + + var dst_data = File.ReadAllBytes(source.AbsolutePath); + + using (var ms = new MemoryStream()) + { + Utils.CreatePatch(src_data, dst_data, ms); + result.PatchID = _compiler.IncludeFile(ms.ToArray()); + } + + return result; + + } } - class zEditMerge { public string name;