Merge pull request #264 from wabbajack-tools/check-archives-before-compile

Check archive status before compilation to reduce the chance of grabb…

Wabbajack.Lib/ACompiler.cs

@@ -207,35 +207,41 @@ namespace Wabbajack.Lib
         {
             if (archives.TryGetValue(sha, out var found))
             {
-                if (found.IniData == null)
+                return ResolveArchive(found);
-                    Error($"No download metadata found for {found.Name}, please use MO2 to query info or add a .meta file and try again.");
-
-                var result = new Archive
-                {
-                    State = (AbstractDownloadState) DownloadDispatcher.ResolveArchive(found.IniData)
-                };
-
-                if (result.State == null)
-                    Error($"{found.Name} could not be handled by any of the downloaders");
-
-                result.Name = found.Name;
-                result.Hash = found.File.Hash;
-                result.Meta = found.Meta;
-                result.Size = found.File.Size;
-
-                Info($"Checking link for {found.Name}");
-
-                if (result.State != null && !result.State.Verify())
-                    Error(
-                        $"Unable to resolve link for {found.Name}. If this is hosted on the Nexus the file may have been removed.");
-
-                return result;
             }
 
             Error($"No match found for Archive sha: {sha} this shouldn't happen");
             return null;
         }
 
+        public Archive ResolveArchive(IndexedArchive archive)
+        {
+            if (archive.IniData == null)
+                Error(
+                    $"No download metadata found for {archive.Name}, please use MO2 to query info or add a .meta file and try again.");
+
+            var result = new Archive
+            {
+                State = (AbstractDownloadState) DownloadDispatcher.ResolveArchive(archive.IniData)
+            };
+
+            if (result.State == null)
+                Error($"{archive.Name} could not be handled by any of the downloaders");
+
+            result.Name = archive.Name;
+            result.Hash = archive.File.Hash;
+            result.Meta = archive.Meta;
+            result.Size = archive.File.Size;
+
+            Info($"Checking link for {archive.Name}");
+
+            if (result.State != null && !result.State.Verify())
+                Error(
+                    $"Unable to resolve link for {archive.Name}. If this is hosted on the Nexus the file may have been removed.");
+
+            return result;
+        }
+
         public Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
         {
             Utils.Status($"Compiling {source.Path}");

Wabbajack.Lib/MO2Compiler.cs

@@ -79,7 +79,7 @@ namespace Wabbajack.Lib
 
         protected override bool _Begin()
         {
-            ConfigureProcessor(18);
+            ConfigureProcessor(19);
             UpdateTracker.Reset();
             UpdateTracker.NextStep("Gathering information");
             Info("Looking for other profiles");
@@ -130,20 +130,7 @@ namespace Wabbajack.Lib
             VFS.AddRoot(MO2DownloadsFolder);
             VFS.WriteToFile(_vfsCacheName);
 
-            UpdateTracker.NextStep("Finding Install Files");
+            UpdateTracker.NextStep("Pre-validating Archives");
-            Directory.CreateDirectory(ModListOutputFolder);
-
-            var mo2Files = Directory.EnumerateFiles(MO2Folder, "*", SearchOption.AllDirectories)
-                .Where(p => p.FileExists())
-                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p]) { Path = p.RelativeTo(MO2Folder) });
-
-            var gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
-                .Where(p => p.FileExists())
-                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
-                { Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) });
-
-
-
 
             IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder)
                 .Where(f => File.Exists(f + ".meta"))
@@ -156,6 +143,21 @@ namespace Wabbajack.Lib
                 })
                 .ToList();
 
+            CleanInvalidArchives();
+
+            UpdateTracker.NextStep("Finding Install Files");
+            Directory.CreateDirectory(ModListOutputFolder);
+
+            var mo2Files = Directory.EnumerateFiles(MO2Folder, "*", SearchOption.AllDirectories)
+                .Where(p => p.FileExists())
+                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p]) { Path = p.RelativeTo(MO2Folder) });
+
+            var gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
+                .Where(p => p.FileExists())
+                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
+                { Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) });
+
+            
             ModMetas = Directory.EnumerateDirectories(Path.Combine(MO2Folder, "mods"))
                 .Keep(f =>
                 {
@@ -289,6 +291,31 @@ namespace Wabbajack.Lib
 
             return true;
         }
+
+        private void CleanInvalidArchives()
+        {
+            var remove = IndexedArchives.PMap(Queue, a =>
+            {
+                try
+                {
+                    ResolveArchive(a);
+                    return null;
+                }
+                catch
+                {
+                    return a;
+                }
+            }).Where(a => a != null).ToHashSet();
+
+            if (remove.Count == 0)
+                return;
+
+            Utils.Log(
+                $"Removing {remove.Count} archives from the compilation state, this is probably not an issue but reference this if you have compilation failures");
+            remove.Do(r => Utils.Log($"Resolution failed for: {r.File}"));
+            IndexedArchives.RemoveAll(a => remove.Contains(a));
+        }
+
         private void InferMetas()
         {
             var to_find = Directory.EnumerateFiles(MO2DownloadsFolder)