Created VortexCompiler and updated MO2 Compiler

Wabbajack.Lib/ACompiler.cs

@@ -4,23 +4,29 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using VFS;
+using Wabbajack.Common;
 using Wabbajack.Lib.CompilationSteps;
 
 namespace Wabbajack.Lib
 {
     public abstract class ACompiler
     {
-        protected string GamePath;
+        public ModManager ModManager;
+        public Compiler _mo2Compiler;
+        public VortexCompiler _vortexCompiler;
 
-        protected string ModListOutputFolder;
-        protected string ModListOutputFile;
+        public string GamePath;
 
-        protected List<Directive> InstallDirectives;
-        protected List<RawSourceFile> AllFiles;
-        protected ModList ModList;
-        protected VirtualFileSystem VFS;
-        protected List<IndexedArchive> IndexedArchives;
-        protected Dictionary<string, IEnumerable<VirtualFile>> IndexedFiles;
+        public string ModListOutputFolder;
+        public string ModListOutputFile;
+
+        public List<Archive> SelectedArchives;
+        public List<Directive> InstallDirectives;
+        public List<RawSourceFile> AllFiles;
+        public ModList ModList;
+        public VirtualFileSystem VFS;
+        public List<IndexedArchive> IndexedArchives;
+        public Dictionary<string, IEnumerable<VirtualFile>> IndexedFiles;
 
         public abstract void Info(string msg);
         public abstract void Status(string msg);

Wabbajack.Lib/Compiler.cs

@@ -26,7 +26,7 @@ using Path = Alphaleonis.Win32.Filesystem.Path;
 
 namespace Wabbajack.Lib
 {
-    public class Compiler
+    public class Compiler : ACompiler
     {
         private string _mo2DownloadsFolder;
 
@@ -42,9 +42,25 @@ namespace Wabbajack.Lib
 
         public Compiler(string mo2_folder)
         {
+            _vortexCompiler = null;
+            _mo2Compiler = this;
+            ModManager = ModManager.MO2;
+
             MO2Folder = mo2_folder;
             MO2Ini = Path.Combine(MO2Folder, "ModOrganizer.ini").LoadIniFile();
             GamePath = ((string)MO2Ini.General.gamePath).Replace("\\\\", "\\");
+
+            ModListOutputFolder = "output_folder";
+            ModListOutputFile = MO2Profile + ExtensionManager.Extension;
+            
+            SelectedArchives = new List<Archive>();
+            InstallDirectives = new List<Directive>();
+            AllFiles = new List<RawSourceFile>();
+            ModList = new ModList();
+
+            VFS = VirtualFileSystem.VFS;
+            IndexedArchives = new List<IndexedArchive>();
+            IndexedFiles = new Dictionary<string, IEnumerable<VirtualFile>>();
         }
 
         public dynamic MO2Ini { get; }
@@ -70,55 +86,43 @@ namespace Wabbajack.Lib
 
         public string MO2ProfileDir => Path.Combine(MO2Folder, "profiles", MO2Profile);
 
-        public string ModListOutputFolder => "output_folder";
-        public string ModListOutputFile => MO2Profile + ExtensionManager.Extension;
-
-        public List<Directive> InstallDirectives { get; private set; }
         internal UserStatus User { get; private set; }
-        public List<Archive> SelectedArchives { get; private set; }
-        public List<RawSourceFile> AllFiles { get; private set; }
-        public ModList ModList { get; private set; }
         public ConcurrentBag<Directive> ExtraFiles { get; private set; }
         public Dictionary<string, dynamic> ModInis { get; private set; }
 
-        public VirtualFileSystem VFS => VirtualFileSystem.VFS;
-
-        public List<IndexedArchive> IndexedArchives { get; private set; }
-        public Dictionary<string, IEnumerable<VirtualFile>> IndexedFiles { get; private set; }
-
         public HashSet<string> SelectedProfiles { get; set; } = new HashSet<string>();
 
-        public void Info(string msg)
+        public override void Info(string msg)
         {
             Utils.Log(msg);
         }
 
-        public void Status(string msg)
+        public override void Status(string msg)
         {
             WorkQueue.Report(msg, 0);
         }
 
-        private void Error(string msg)
+        public override void Error(string msg)
         {
             Utils.Log(msg);
             throw new Exception(msg);
         }
 
-        internal string IncludeFile(byte[] data)
+        internal override string IncludeFile(byte[] data)
         {
             var id = Guid.NewGuid().ToString();
             File.WriteAllBytes(Path.Combine(ModListOutputFolder, id), data);
             return id;
         }
 
-        internal string IncludeFile(string data)
+        internal override string IncludeFile(string data)
         {
             var id = Guid.NewGuid().ToString();
             File.WriteAllText(Path.Combine(ModListOutputFolder, id), data);
             return id;
         }
 
-        public bool Compile()
+        public override bool Compile()
         {
             VirtualFileSystem.Clean();
             Info("Looking for other profiles");
@@ -283,6 +287,7 @@ namespace Wabbajack.Lib
                 GameType = GameRegistry.Games.Values.First(f => f.MO2Name == MO2Ini.General.gameName).Game,
                 WabbajackVersion = WabbajackVersion,
                 Archives = SelectedArchives,
+                ModManager = ModManager.MO2,
                 Directives = InstallDirectives,
                 Name = ModListName ?? MO2Profile,
                 Author = ModListAuthor ?? "",
@@ -533,7 +538,7 @@ namespace Wabbajack.Lib
         }
 
 
-        public static Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
+        public override Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
         {
             Utils.Status($"Compiling {source.Path}");
             foreach (var step in stack)
@@ -545,7 +550,7 @@ namespace Wabbajack.Lib
             throw new InvalidDataException("Data fell out of the compilation stack");
         }
 
-        public IEnumerable<ICompilationStep> GetStack()
+        public override IEnumerable<ICompilationStep> GetStack()
         {
             var user_config = Path.Combine(MO2ProfileDir, "compilation_stack.yml");
             if (File.Exists(user_config))
@@ -566,7 +571,7 @@ namespace Wabbajack.Lib
         ///     result included into the pack
         /// </summary>
         /// <returns></returns>
-        public IEnumerable<ICompilationStep> MakeStack()
+        public override IEnumerable<ICompilationStep> MakeStack()
         {
             Utils.Log("Generating compilation stack");
             return new List<ICompilationStep>

Wabbajack.Lib/VortexCompiler.cs

@@ -0,0 +1,308 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using VFS;
+using Wabbajack.Common;
+using Wabbajack.Lib.CompilationSteps;
+
+namespace Wabbajack.Lib
+{
+    public class VortexCompiler : ACompiler
+    {
+        public string GameName { get; }
+
+        public string VortexFolder { get; }
+        public string StagingFolder { get; }
+        public string DownloadsFolder { get; }
+
+        public bool IgnoreMissingFiles { get; set; }
+
+        public VortexCompiler(string gameName, string gamePath)
+        {
+            _vortexCompiler = this;
+            _mo2Compiler = null;
+            ModManager = ModManager.Vortex;
+
+            // TODO: only for testing
+            IgnoreMissingFiles = true;
+
+            GamePath = gamePath;
+            GameName = gameName;
+
+            // currently only works if staging and downloads folder is in the standard directory
+            // aka %APPDATADA%\Vortex\
+            VortexFolder = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Vortex");
+            StagingFolder = Path.Combine(VortexFolder, gameName, "mods");
+            DownloadsFolder = Path.Combine(VortexFolder, "downloads", gameName);
+
+            ModListOutputFolder = "output_folder";
+
+            // TODO: add custom modlist name
+            ModListOutputFile = $"VORTEX_TEST_MODLIST{ExtensionManager.Extension}";
+
+            VFS = VirtualFileSystem.VFS;
+
+            SelectedArchives = new List<Archive>();
+            AllFiles = new List<RawSourceFile>();
+            IndexedArchives = new List<IndexedArchive>();
+            IndexedFiles = new Dictionary<string, IEnumerable<VirtualFile>>();
+        }
+
+        public override void Info(string msg)
+        {
+            Utils.Log(msg);
+        }
+
+        public override void Status(string msg)
+        {
+            WorkQueue.Report(msg, 0);
+        }
+
+        public override void Error(string msg)
+        {
+            Utils.Log(msg);
+            throw new Exception(msg);
+        }
+
+        internal override string IncludeFile(byte[] data)
+        {
+            var id = Guid.NewGuid().ToString();
+            File.WriteAllBytes(Path.Combine(ModListOutputFolder, id), data);
+            return id;
+        }
+
+        internal override string IncludeFile(string data)
+        {
+            var id = Guid.NewGuid().ToString();
+            File.WriteAllText(Path.Combine(ModListOutputFolder, id), data);
+            return id;
+        }
+
+        public override bool Compile()
+        {
+            VirtualFileSystem.Clean();
+            Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at  {DownloadsFolder}.");
+
+            Info($"Indexing {GamePath}");
+            VFS.AddRoot(GamePath);
+
+            Info($"Indexing {DownloadsFolder}");
+            VFS.AddRoot(DownloadsFolder);
+
+            Info("Cleaning output folder");
+            if (Directory.Exists(ModListOutputFolder)) Directory.Delete(ModListOutputFolder, true);
+            Directory.CreateDirectory(ModListOutputFolder);
+
+            IEnumerable<RawSourceFile> game_files = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
+                .Where(p => p.FileExists())
+                .Select(p => new RawSourceFile(VFS.Lookup(p))
+                    { Path = Alphaleonis.Win32.Filesystem.Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) });
+
+            Info("Indexing Archives");
+            IndexedArchives = Directory.EnumerateFiles(DownloadsFolder)
+                .Where(File.Exists)
+                .Select(f => new IndexedArchive
+                {
+                    File = VFS.Lookup(f),
+                    Name = Path.GetFileName(f)
+                })
+                .ToList();
+
+            Info("Indexing Files");
+            IDictionary<VirtualFile, IEnumerable<VirtualFile>> grouped = VFS.GroupedByArchive();
+            IndexedFiles = IndexedArchives.Select(f => grouped.TryGetValue(f.File, out var result) ? result : new List<VirtualFile>())
+                .SelectMany(fs => fs)
+                .Concat(IndexedArchives.Select(f => f.File))
+                .OrderByDescending(f => f.TopLevelArchive.LastModified)
+                .GroupBy(f => f.Hash)
+                .ToDictionary(f => f.Key, f => f.AsEnumerable());
+
+            Info("Searching for mod files");
+            AllFiles = game_files.DistinctBy(f => f.Path).ToList();
+
+            Info($"Found {AllFiles.Count} files to build into mod list");
+
+            Info("Verifying destinations");
+            List<IGrouping<string, RawSourceFile>> dups = AllFiles.GroupBy(f => f.Path)
+                .Where(fs => fs.Count() > 1)
+                .Select(fs =>
+                {
+                    Utils.Log($"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}");
+                    return fs;
+                }).ToList();
+
+            if (dups.Count > 0)
+            {
+                Error($"Found {dups.Count} duplicates, exiting");
+            }
+
+            IEnumerable<ICompilationStep> stack = MakeStack();
+
+            Info("Running Compilation Stack");
+            List<Directive> results = AllFiles.PMap(f => RunStack(stack, f)).ToList();
+
+            IEnumerable<NoMatch> noMatch = results.OfType<NoMatch>().ToList();
+            Info($"No match for {noMatch.Count()} files");
+            foreach (var file in noMatch)
+                Info($"     {file.To}");
+            if (noMatch.Any())
+            {
+                if (IgnoreMissingFiles)
+                {
+                    Info("Continuing even though files were missing at the request of the user.");
+                }
+                else
+                {
+                    Info("Exiting due to no way to compile these files");
+                    return false;
+                }
+            }
+
+            InstallDirectives = results.Where(i => !(i is IgnoredDirectly)).ToList();
+
+            // TODO: nexus stuff
+            /*Info("Getting Nexus api_key, please click authorize if a browser window appears");
+            if (IndexedArchives.Any(a => a.IniData?.General?.gameName != null))
+            {
+                var nexusClient = new NexusApiClient();
+                if (!nexusClient.IsPremium) Error($"User {nexusClient.Username} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue");
+
+            }
+            */
+
+            GatherArchives();
+
+            ModList = new ModList
+            {
+                Archives = SelectedArchives,
+                ModManager = ModManager.Vortex,
+                Directives = InstallDirectives
+            };
+            
+            ExportModList();
+
+            Info("Done Building ModList");
+            return true;
+        }
+
+        private void ExportModList()
+        {
+            Utils.Log($"Exporting ModList to: {ModListOutputFolder}");
+
+            // using JSON for better debugging
+            ModList.ToJSON(Path.Combine(ModListOutputFolder, "modlist.json"));
+            //ModList.ToCERAS(Path.Combine(ModListOutputFolder, "modlist"), ref CerasConfig.Config);
+
+            if(File.Exists(ModListOutputFile))
+                File.Delete(ModListOutputFile);
+
+            using (var fs = new FileStream(ModListOutputFile, FileMode.Create))
+            {
+                using (var za = new ZipArchive(fs, ZipArchiveMode.Create))
+                {
+                    Directory.EnumerateFiles(ModListOutputFolder, "*.*")
+                        .DoProgress("Compressing Modlist",
+                            f =>
+                            {
+                                var ze = za.CreateEntry(Path.GetFileName(f));
+                                using (var os = ze.Open())
+                                using (var ins = File.OpenRead(f))
+                                {
+                                    ins.CopyTo(os);
+                                }
+                            });
+                }
+            }
+            Utils.Log("Removing ModList staging folder");
+            Directory.Delete(ModListOutputFolder, true);
+        }
+
+        private void GatherArchives()
+        {
+            Info("Building a list of archives based on the files required");
+
+            var shas = InstallDirectives.OfType<FromArchive>()
+                .Select(a => a.ArchiveHashPath[0])
+                .Distinct();
+
+            var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified)
+                .GroupBy(f => f.File.Hash)
+                .ToDictionary(f => f.Key, f => f.First());
+
+            SelectedArchives = shas.PMap(sha => ResolveArchive(sha, archives));
+        }
+
+        // TODO: this whole thing
+        private Archive ResolveArchive(string sha, IDictionary<string, IndexedArchive> archives)
+        {
+            if (archives.TryGetValue(sha, out var found))
+            {
+                var result = new Archive();
+
+                result.Name = found.Name;
+                result.Hash = found.File.Hash;
+                result.Size = found.File.Size;
+
+                return result;
+            }
+
+            Error($"No match found for Archive sha: {sha} this shouldn't happen");
+            return null;
+        }
+
+        public override Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
+        {
+            Utils.Status($"Compiling {source.Path}");
+            foreach (var step in stack)
+            {
+                var result = step.Run(source);
+                if (result != null) return result;
+            }
+
+            throw new InvalidDataException("Data fell out of the compilation stack");
+
+        }
+
+        public override IEnumerable<ICompilationStep> GetStack()
+        {
+            var userConfig = Path.Combine(VortexFolder, "compilation_stack.yml");
+            if (File.Exists(userConfig))
+                return Serialization.Deserialize(File.ReadAllText(userConfig), this);
+
+            IEnumerable<ICompilationStep> stack = MakeStack();
+
+            File.WriteAllText(Path.Combine(VortexFolder, "_current_compilation_stack.yml"),
+                Serialization.Serialize(stack));
+
+            return stack;
+        }
+
+        public override IEnumerable<ICompilationStep> MakeStack()
+        {
+            Utils.Log("Generating compilation stack");
+            return new List<ICompilationStep>
+            {
+                //new IncludePropertyFiles(this),
+
+                new IgnoreGameFiles(this), 
+
+                new IgnoreStartsWith(this, Path.Combine(Consts.GameFolderFilesDir, "Data")),
+                new IgnoreStartsWith(this, Path.Combine(Consts.GameFolderFilesDir, "Papyrus Compiler")),
+                new IgnoreStartsWith(this, Path.Combine(Consts.GameFolderFilesDir, "Skyrim")),
+                new IgnoreRegex(this, Consts.GameFolderFilesDir + "\\\\.*\\.bsa"),
+
+                new DirectMatch(this),
+
+                new IgnoreGameFiles(this),
+
+                new IgnoreWabbajackInstallCruft(this),
+
+                new DropAll(this)
+            };
+        }
+    }
+}

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -77,6 +77,7 @@
     <Reference Include="WindowsBase" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="ACompiler.cs" />
     <Compile Include="CerasConfig.cs" />
     <Compile Include="CompilationSteps\ACompilationStep.cs" />
     <Compile Include="CompilationSteps\DeconstructBSAs.cs" />
@@ -131,6 +132,7 @@
     <Compile Include="Validation\DTOs.cs" />
     <Compile Include="Validation\ValidateModlist.cs" />
     <Compile Include="ViewModel.cs" />
+    <Compile Include="VortexCompiler.cs" />
     <Compile Include="WebAutomation\WebAutomation.cs" />
     <Compile Include="WebAutomation\WebAutomationWindow.xaml.cs">
       <DependentUpon>WebAutomationWindow.xaml</DependentUpon>