Merge pull request #286 from erri120/vortex-fixes-6

Vortex Fixes 6

Wabbajack.Lib/ReportBuilder.cs

@@ -64,8 +64,12 @@ namespace Wabbajack.Lib
                         .Do(NoWrapText);
             }
 
+            var archiveCount = lst.Archives.Count + lst.Directives.Count(d => d is SteamMeta);
+            var totalSize = lst.Archives.Sum(a => a.Size);
+            totalSize += lst.Directives.Where(d => d is SteamMeta).Cast<SteamMeta>().Sum(s => s.Size);
+
             Text(
-                $"#### Download Summary ({lst.Archives.Count} archives - {lst.Archives.Sum(a => a.Size).ToFileSizeString()})");
+                $"#### Download Summary ({archiveCount} archives - {totalSize.ToFileSizeString()})");
             foreach (var archive in SortArchives(lst.Archives))
             {
                 var hash = archive.Hash.FromBase64().ToHex();
@@ -75,11 +79,14 @@ namespace Wabbajack.Lib
             }
             lst.Directives.Where(d => d is SteamMeta).Do(f =>
             {
-                if (f is SteamMeta s)
+                if (!(f is SteamMeta s))
                 {
-                    var link = $"https://steamcommunity.com/sharedfiles/filedetails/?id={s.ItemID}";
+                    return;
-                    NoWrapText($"* Steam Workshop Item: [{s.ItemID}]({link}) | Size: {s.Size}");
                 }
+
+                var link = $"https://steamcommunity.com/sharedfiles/filedetails/?id={s.ItemID}";
+                var size = ((long)s.Size).ToFileSizeString();
+                NoWrapText($"* Steam Workshop Item: [{s.ItemID}]({link}) | Size: {size}");
             });
 
             Text("\n\n");

Wabbajack.Lib/VortexCompiler.cs

@@ -6,13 +6,14 @@ using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Tasks;
 using System.Threading;
-using DynamicData;
 using Microsoft.WindowsAPICodePack.Shell;
 using Newtonsoft.Json;
 using Wabbajack.Common;
 using Wabbajack.Lib.CompilationSteps;
 using Wabbajack.Lib.NexusApi;
+using Wabbajack.Lib.Validation;
 using File = Alphaleonis.Win32.Filesystem.File;
+using Game = Wabbajack.Common.Game;
 
 namespace Wabbajack.Lib
 {
@@ -37,7 +38,7 @@ namespace Wabbajack.Lib
 
         public override ModManager ModManager => ModManager.Vortex;
         public override string GamePath { get; }
-        public override string ModListOutputFolder { get; }
+        public override string ModListOutputFolder => "output_folder";
         public override string ModListOutputFile { get; }
 
         public const string StagingMarkerName = "__vortex_staging_folder";
@@ -56,9 +57,18 @@ namespace Wabbajack.Lib
             VortexFolder = vortexFolder;
             DownloadsFolder = downloadsFolder;
             StagingFolder = stagingFolder;
-            ModListOutputFolder = "output_folder";
             ModListOutputFile = outputFile;
 
+            if (string.IsNullOrEmpty(ModListName))
+            {
+                ModListName = $"Vortex ModList for {Game.ToString()}";
+                ModListOutputFile = $"{ModListName}{ExtensionManager.Extension}";
+            }
+
+            GameName = Game.MetaData().NexusName;
+
+            ActiveArchives = new List<string>();
+
             // there can be max one game after filtering
             SteamHandler.Instance.Games.Where(g => g.Game != null && g.Game == game).Do(g =>
             {
@@ -67,52 +77,51 @@ namespace Wabbajack.Lib
                 SteamHandler.Instance.LoadWorkshopItems(_steamGame);
                 _hasSteamWorkshopItems = _steamGame.WorkshopItems.Count > 0;
             });
-
-            ActiveArchives = new List<string>();
         }
         
         protected override async Task<bool> _Begin(CancellationToken cancel)
         {
             if (cancel.IsCancellationRequested) return false;
 
-            ConfigureProcessor(10);
+            Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}.");
-            if (string.IsNullOrEmpty(ModListName))
-                ModListName = $"Vortex ModList for {Game.ToString()}";
 
-            Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at  {DownloadsFolder}.");
+            ConfigureProcessor(12);
+            UpdateTracker.Reset();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Parsing deployment file");
             ParseDeploymentFile();
 
             if (cancel.IsCancellationRequested) return false;
-            Info("Starting pre-compilation steps");
+            UpdateTracker.NextStep("Creating metas for archives");
             await CreateMetaFiles();
 
             if (cancel.IsCancellationRequested) return false;
-            Info($"Indexing {StagingFolder}");
+            await VFS.IntegrateFromFile(_vfsCacheName);
-            await VFS.AddRoot(StagingFolder);
 
-            Info($"Indexing {GamePath}");
+            var roots = new List<string> {StagingFolder, GamePath, DownloadsFolder};
-            await VFS.AddRoot(GamePath);
+            AddExternalFolder(ref roots);
-
-            Info($"Indexing {DownloadsFolder}");
-            await VFS.AddRoot(DownloadsFolder);
 
             if (cancel.IsCancellationRequested) return false;
-            await AddExternalFolder();
+            UpdateTracker.NextStep("Indexing folders");
+            await VFS.AddRoots(roots);
+            await VFS.WriteToFile(_vfsCacheName);
 
             if (cancel.IsCancellationRequested) return false;
-            Info("Cleaning output folder");
+            UpdateTracker.NextStep("Cleaning output folder");
-            if (Directory.Exists(ModListOutputFolder)) Utils.DeleteDirectory(ModListOutputFolder);
+            if (Directory.Exists(ModListOutputFolder))
+                Utils.DeleteDirectory(ModListOutputFolder);
+
             Directory.CreateDirectory(ModListOutputFolder);
 
+            UpdateTracker.NextStep("Finding Install Files");
             var vortexStagingFiles = Directory.EnumerateFiles(StagingFolder, "*", SearchOption.AllDirectories)
                 .Where(p => p.FileExists() && p != StagingMarkerName)
                 .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
                     {Path = p.RelativeTo(StagingFolder)});
             
             var vortexDownloads = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.AllDirectories)
-                .Where(p => p.FileExists())
+                .Where(p => p.FileExists() && p != DownloadMarkerName)
                 .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
                     {Path = p.RelativeTo(DownloadsFolder)});
 
@@ -139,7 +148,6 @@ namespace Wabbajack.Lib
                 .GroupBy(f => f.Hash)
                 .ToDictionary(f => f.Key, f => f.AsEnumerable());
 
-            Info("Searching for mod files");
             AllFiles = vortexStagingFiles.Concat(vortexDownloads)
                 .Concat(gameFiles)
                 .DistinctBy(f => f.Path)
@@ -148,7 +156,7 @@ namespace Wabbajack.Lib
             Info($"Found {AllFiles.Count} files to build into mod list");
 
             if (cancel.IsCancellationRequested) return false;
-            Info("Verifying destinations");
+            UpdateTracker.NextStep("Verifying destinations");
             var duplicates = AllFiles.GroupBy(f => f.Path)
                 .Where(fs => fs.Count() > 1)
                 .Select(fs =>
@@ -228,17 +236,16 @@ namespace Wabbajack.Lib
 
             InstallDirectives = results.Where(i => !(i is IgnoredDirectly)).ToList();
 
-            // TODO: nexus stuff
+            Info("Getting Nexus api_key, please click authorize if a browser window appears");
-            /*Info("Getting Nexus api_key, please click authorize if a browser window appears");
             if (IndexedArchives.Any(a => a.IniData?.General?.gameName != null))
             {
-                var nexusClient = new NexusApiClient();
+                var nexusClient = await NexusApiClient.Get();
-                if (!nexusClient.IsPremium) Error($"User {nexusClient.Username} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue");
+                if (!await nexusClient.IsPremium()) Error($"User {await nexusClient.Username()} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue");
-
             }
-            */
+            
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Gathering Archives");
             await GatherArchives();
 
             ModList = new ModList
@@ -255,15 +262,45 @@ namespace Wabbajack.Lib
                 GameType = Game
             };
             
+            UpdateTracker.NextStep("Running Validation");
+            await ValidateModlist.RunValidation(Queue, ModList);
+
+            UpdateTracker.NextStep("Generating Report");
             GenerateReport();
+
+            UpdateTracker.NextStep("Exporting ModList");
             ExportModList();
 
-            Info("Done Building ModList");
+            ResetMembers();
 
             ShowReport();
+
+            UpdateTracker.NextStep("Done Building ModList");
+
             return true;
         }
 
+        /// <summary>
+        ///     Clear references to lists that hold a lot of data.
+        /// </summary>
+        private void ResetMembers()
+        {
+            AllFiles = null;
+            InstallDirectives = null;
+            SelectedArchives = null;
+        }
+
+        private void AddExternalFolder(ref List<string> roots)
+        {
+            var currentGame = Game.MetaData();
+            if (currentGame.AdditionalFolders == null || currentGame.AdditionalFolders.Count == 0) return;
+            foreach (var path in currentGame.AdditionalFolders.Select(f => f.Replace("%documents%", KnownFolders.Documents.Path)))
+            {
+                if (!Directory.Exists(path)) return;
+                roots.Add(path);
+            }
+        }
+
         private void ParseDeploymentFile()
         {
             Info("Searching for vortex.deployment.json...");
@@ -280,10 +317,10 @@ namespace Wabbajack.Lib
 
             if (string.IsNullOrEmpty(deploymentFile))
             {
-                Info("vortex.deployment.json not found!");
+                Error("vortex.deployment.json not found!");
                 return;
             }
-            Info("vortex.deployment.json found at "+deploymentFile);
+            Info($"vortex.deployment.json found at {deploymentFile}");
 
             Info("Parsing vortex.deployment.json...");
             try
@@ -306,100 +343,82 @@ namespace Wabbajack.Lib
             });
         }
 
-        /// <summary>
-        /// Some have mods outside their game folder located
-        /// </summary>
-        private async Task AddExternalFolder()
-        {
-            var currentGame = Game.MetaData();
-            if (currentGame.AdditionalFolders == null || currentGame.AdditionalFolders.Count == 0) return;
-            foreach (var f in currentGame.AdditionalFolders)
-            {
-                var path = f.Replace("%documents%", KnownFolders.Documents.Path);
-                if (!Directory.Exists(path)) return;
-                Info($"Indexing {path}");
-                await VFS.AddRoot(path);
-            }
-        }
-
         private async Task CreateMetaFiles()
         {
             Utils.Log("Getting Nexus api_key, please click authorize if a browser window appears");
             var nexusClient = await NexusApiClient.Get();
 
-            await Task.WhenAll(
+            var archives = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly).Where(f =>
-                Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly)
+                File.Exists(f) && Path.GetExtension(f) != ".meta" && Path.GetExtension(f) != ".xxHash" &&
-                .Where(File.Exists)
+                !File.Exists($"{f}.meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)));
-                .Select(async f =>
+
+            await archives.PMap(Queue, async f =>
+            {
+                Info($"Creating meta file for {Path.GetFileName(f)}");
+                var metaString = "[General]\n" +
+                                 "repository=Nexus\n" +
+                                 $"gameName={GameName}\n";
+                string hash;
+                using (var md5 = MD5.Create())
+                using (var stream = File.OpenRead(f))
                 {
-                    if (Path.GetExtension(f) != ".meta" && Path.GetExtension(f) != ".xxHash" && !File.Exists($"{f}.meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)))
+                    Utils.Log($"Calculating hash for {Path.GetFileName(f)}");
-                    {
+                    var cH = md5.ComputeHash(stream);
-                        Utils.Log($"Trying to create meta file for {Path.GetFileName(f)}");
+                    hash = BitConverter.ToString(cH).Replace("-", "").ToLowerInvariant();
-                        var metaString = "[General]\n" +
+                    Utils.Log($"Hash is {hash}");
-                                         "repository=Nexus\n" +
+                }
-                                         "installed=true\n" +
-                                         "uninstalled=false\n" +
-                                         "paused=false\n" +
-                                         "removed=false\n" +
-                                         $"gameName={GameName}\n";
-                        string hash;
-                        using(var md5 = MD5.Create())
-                        using (var stream = File.OpenRead(f))
-                        {
-                            Utils.Log($"Calculating hash for {Path.GetFileName(f)}");
-                            var cH = md5.ComputeHash(stream);
-                            hash = BitConverter.ToString(cH).Replace("-", "").ToLowerInvariant();
-                            Utils.Log($"Hash is {hash}");
-                        }
 
-                        var md5Response = await nexusClient.GetModInfoFromMD5(Game, hash);
+                var md5Response = await nexusClient.GetModInfoFromMD5(Game, hash);
-                        if (md5Response.Count >= 1)
+                if (md5Response.Count >= 1)
-                        {
+                {
-                            var modInfo = md5Response[0].mod;
+                    var modInfo = md5Response[0].mod;
-                            metaString += $"modID={modInfo.mod_id}\n" +
+                    metaString += $"modID={modInfo.mod_id}\n" +
-                                          $"modName={modInfo.name}\nfileID={md5Response[0].file_details.file_id}";
+                                  $"modName={modInfo.name}\n" +
-                            File.WriteAllText(f+".meta",metaString, Encoding.UTF8);
+                                  $"fileID={md5Response[0].file_details.file_id}\n" +
-                        }
+                                  $"version={md5Response[0].file_details.version}\n" +
-                        else
+                                  $"hash={hash}\n";
-                        {
+                    File.WriteAllText(f + ".meta", metaString, Encoding.UTF8);
-                            Error("Error while getting information from NexusMods via MD5 hash!");
+                }
-                        }
+                else
-                    }
+                {
-                    else
+                    Error("Error while getting information from NexusMods via MD5 hash!");
+                }
+            });
+
+            var otherFiles = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly).Where(f =>
+                Path.GetExtension(f) == ".meta" && !ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)));
+
+            await otherFiles.PMap(Queue, async f =>
+            {
+                Info($"File {f} is not in ActiveArchives");
+                var lines = File.ReadAllLines(f);
+                if (lines.Length == 0 || !lines.Any(line => lines.Contains("directURL=")))
+                {
+                    if (lines.Length == 0)
+                        return;
+
+                    lines.Do(line =>
                     {
-                        if (Path.GetExtension(f) != ".meta" ||
+                        var tag = "";
-                            ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)))
+                        if (line.Contains("tag="))
+                            tag = line.Substring("tag=".Length);
+
+                        if (tag != Consts.WABBAJACK_VORTEX_MANUAL)
                             return;
 
-                        Utils.Log($"File {f} is not in ActiveArchives");
+                        Info($"File {f} contains the {Consts.WABBAJACK_VORTEX_MANUAL} tag, adding to ActiveArchives");
-                        var lines = File.ReadAllLines(f);
+                        ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
+                    });
+                }
+                else
+                {
+                    Info($"File {f} appears to not be from the Nexus, adding to ActiveArchives");
+                    ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
+                }
+            });
 
-                        if (lines.Length == 0)
+            Info("Checking for Steam Workshop Items...");
-                            return;
+            if (!_isSteamGame || _steamGame == null || !_hasSteamWorkshopItems)
-
-                        lines.Do(line =>
-                        {
-                            var tag = "";
-                            if (line.Contains("tag="))
-                                tag = line.Substring("tag=".Length);
-
-                            if (tag != Consts.WABBAJACK_VORTEX_MANUAL)
-                                return;
-
-                            Utils.Log($"File {f} contains the {Consts.WABBAJACK_VORTEX_MANUAL} tag, adding to ActiveArchives");
-                            ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
-                        });
-
-                        if (lines.Any(line => line.Contains("directURL=")) && !ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)))
-                        {
-                            Utils.Log($"File {f} appears to not come from the Nexus, adding to ActiveArchives");
-                            ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
-                        }
-                    }
-                }));
-
-            Utils.Log($"Checking for Steam Workshop Items...");
-            if (!_isSteamGame || _steamGame == null || _steamGame.WorkshopItems.Count <= 0)
                 return;
 
             _steamGame.WorkshopItems.Do(item =>
@@ -414,7 +433,6 @@ namespace Wabbajack.Lib
                 Utils.Log($"Creating meta file for {item.ItemID}");
                 var metaString = "[General]\n" +
                                  "repository=Steam\n" +
-                                 "installed=true\n" +
                                  $"gameName={GameName}\n" +
                                  $"steamID={_steamGame.AppId}\n" +
                                  $"itemID={item.ItemID}\n" +
@@ -439,20 +457,23 @@ namespace Wabbajack.Lib
 
             var stack = MakeStack();
 
+            var compilationSteps = stack.ToList();
             File.WriteAllText(Path.Combine(s, "_current_compilation_stack.yml"),
-                Serialization.Serialize(stack));
+                Serialization.Serialize(compilationSteps));
 
-            return stack;
+            return compilationSteps;
         }
 
         public override IEnumerable<ICompilationStep> MakeStack()
         {
-            Utils.Log("Generating compilation stack");
+            Info("Generating compilation stack");
             return new List<ICompilationStep>
             {
                 new IncludePropertyFiles(this),
+
                 new IncludeSteamWorkshopItems(this, _steamGame),
                 _hasSteamWorkshopItems ? new IncludeRegex(this, "^steamWorkshopItem_\\d*\\.meta$") : null,
+                
                 new IgnoreDisabledVortexMods(this),
                 new IncludeVortexDeployment(this),
                 new IgnoreVortex(this),

Wabbajack.Lib/VortexInstaller.cs

@@ -49,12 +49,15 @@ namespace Wabbajack.Lib
             Directory.CreateDirectory(DownloadFolder);
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Hashing Archives");
             await HashArchives();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Downloading Missing Archives");
             await DownloadArchives();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Hashing Remaining Archives");
             await HashArchives();
 
             if (cancel.IsCancellationRequested) return false;
@@ -69,26 +72,33 @@ namespace Wabbajack.Lib
                     Error("Cannot continue, was unable to download one or more archives");
             }
 
+            if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Priming VFS");
             await PrimeVFS();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Building Folder Structure");
             BuildFolderStructure();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Installing Archives");
             await InstallArchives();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Installing Included files");
             await InstallIncludedFiles();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Installing Manual files");
             await InstallManualGameFiles();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Installing SteamWorkshopItems");
             await InstallSteamWorkshopItems();
 
             //InstallIncludedDownloadMetas();
             var metric2 = Metrics.Send("finish_install", ModList.Name);
-            Info("Installation complete! You may exit the program.");
+            UpdateTracker.NextStep("Installation complete! You may exit the program.");
             return true;
         }