WIP on the compiler, can compile most of SMEFT

Wabbajack.Compiler/ACompiler.cs

@@ -547,7 +547,7 @@ namespace Wabbajack.Compiler
             if (!await _dispatcher.Verify(result, token.Token))
             {
                 _logger.LogWarning(
-                    "Unable to resolve link for {archive}. If this is hosted on the Nexus the file may have been removed.", archive);
+                    "Unable to resolve link for {Archive}. If this is hosted on the Nexus the file may have been removed.", archive.State!.PrimaryKeyString);
             }
 
             result.Meta = "[General]\n" + string.Join("\n", _dispatcher.MetaIni(result));

Wabbajack.Compiler/CompilationSteps/IgnoreDisabledMods.cs

@@ -16,16 +16,17 @@ namespace Wabbajack.Compiler.CompilationSteps
         public IgnoreDisabledMods(ACompiler compiler) : base(compiler)
         {
             _mo2Compiler = (MO2Compiler) compiler;
-            var alwaysEnabled = _mo2Compiler.ModInis.Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_ENABLE)).Select(f => f.Key).Distinct();
+            //var alwaysEnabled = _mo2Compiler.ModInis.Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_ENABLE)).Select(f => f.Key).Distinct();
-            var alwaysDisabled = _mo2Compiler.ModInis
+            // TODO: Re-enable this
-                .Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_DISABLE)).Select(f => f.Key).Distinct();
+            //var alwaysDisabled = _mo2Compiler.ModInis
+             //   .Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_DISABLE)).Select(f => f.Key).Distinct();
 
             _allEnabledMods = _mo2Compiler._settings.SelectedProfiles
                 .SelectMany(p => _mo2Compiler._settings.Source.Combine("profiles", p, "modlist.txt").ReadAllLines())
                 .Where(line => line.StartsWith("+") || line.EndsWith("_separator"))
                 .Select(line => line[1..].ToRelativePath().RelativeTo(_mo2Compiler.MO2ModsFolder))
-                .Concat(alwaysEnabled)
+                //.Concat(alwaysEnabled)
-                .Except(alwaysDisabled)
+                //.Except(alwaysDisabled)
                 .ToList();
         }
 

Wabbajack.Compiler/Consts.cs

@@ -35,7 +35,7 @@ namespace Wabbajack.Compiler
         public static readonly HashSet<Extension> SupportedBSAs = new[] {".bsa", ".ba2"}
             .Select(s => new Extension(s)).ToHashSet();
         
-        public static HashSet<Extension> ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml"}.Select(s => new Extension(s)).ToHashSet();
+        public static HashSet<Extension> ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml", ".yaml"}.Select(s => new Extension(s)).ToHashSet();
         public static HashSet<Extension> ESPFileExtensions = new []{ ".esp", ".esm", ".esl"}.Select(s => new Extension(s)).ToHashSet();
         public static HashSet<Extension> AssetFileExtensions = new[] {".dds", ".tga", ".nif", ".psc", ".pex"}.Select(s => new Extension(s)).ToHashSet();
         

Wabbajack.Compiler/MO2Compiler.cs

@@ -275,6 +275,10 @@ namespace Wabbajack.Compiler
                 new IncludeRegex(this, ".*\\.txt"),
                 new IgnorePathContains(this,@"\Edit Scripts\Export\"),
                 new IgnoreExtension(this, new Extension(".CACHE")),
+                
+                // Misc
+                new IncludeRegex(this, "modlist-image\\.png"),
+
                 new DropAll(this)
             };
 

Wabbajack.Installer/AInstaller.cs

@@ -256,15 +256,18 @@ namespace Wabbajack.Installer
 
             await Task.WhenAll(dispatchers.Select(d => d.Prepare()));
 
+            _logger.LogInformation("Downloading validation data");
             var validationData = await _wjClient.LoadDownloadAllowList();
 
+            _logger.LogInformation("Validating Archives");
             foreach (var archive in missing.Where(archive =>
                 !_downloadDispatcher.Downloader(archive).IsAllowed(validationData, archive.State)))
             {
                 _logger.LogCritical("File {primaryKeyString} failed validation", archive.State.PrimaryKeyString);
                 return;
             }
-
+            
+            _logger.LogInformation("Downloading missing archives");
             await DownloadMissingArchives(missing, token);
         }
 

Wabbajack.Services.OSIntegrated/ServiceExtensions.cs

@@ -59,6 +59,7 @@ namespace Wabbajack.Services.OSIntegrated
             service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount});
             service.AddAllSingleton<IResource, IResource<DownloadDispatcher>>(s => new Resource<DownloadDispatcher>(12));
             service.AddAllSingleton<IResource, IResource<HttpClient>>(s => new Resource<HttpClient>(12));
+            service.AddAllSingleton<IResource, IResource<Context>>(s => new Resource<Context>(12));
             service.AddAllSingleton<IResource, IResource<FileExtractor.FileExtractor>>(s =>
                 new Resource<FileExtractor.FileExtractor>(12));
 

Wabbajack.VFS/Context.cs

@@ -10,6 +10,7 @@ using Wabbajack.FileExtractor.ExtractedFiles;
 using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Paths;
 using Wabbajack.Paths.IO;
+using Wabbajack.RateLimiter;
 
 namespace Wabbajack.VFS
 {
@@ -25,10 +26,12 @@ namespace Wabbajack.VFS
         public readonly FileHashCache HashCache;
         public readonly ILogger<Context> Logger;
         public readonly VFSCache VfsCache;
+        public readonly IResource<Context> Limiter;
 
         public Context(ILogger<Context> logger, ParallelOptions parallelOptions, TemporaryFileManager manager, VFSCache vfsCache,
-            FileHashCache hashCache, FileExtractor.FileExtractor extractor)
+            FileHashCache hashCache, IResource<Context> limiter, FileExtractor.FileExtractor extractor)
         {
+            Limiter = limiter;
             Logger = logger;
             _manager = manager;
             Extractor = extractor;
@@ -49,7 +52,7 @@ namespace Wabbajack.VFS
             var filesToIndex = root.EnumerateFiles().Distinct().ToList();
 
             var allFiles = await filesToIndex
-                .PMap(_parallelOptions, async f =>
+                .PMapAll(async f =>
                 {
                     if (byPath.TryGetValue(f, out var found))
                         if (found.LastModified == f.LastModifiedUtc().AsUnixTime() && found.Size == f.Size())

Wabbajack.VFS/FileHashCache.cs

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
 using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Paths;
 using Wabbajack.Paths.IO;
+using Wabbajack.RateLimiter;
 
 namespace Wabbajack.VFS
 {
@@ -113,13 +114,13 @@ namespace Wabbajack.VFS
             WriteHashCache(file, hash);
         }
 
-        public async Task<Hash> FileHashCachedAsync(AbsolutePath file, CancellationToken token)
+        public async Task<Hash> FileHashCachedAsync(AbsolutePath file, CancellationToken token, IJob? job = null)
         {
             if (TryGetHashCache(file, out var foundHash)) return foundHash;
 
             await using var fs = file.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
 
-            var hash = await fs.HashingCopy(Stream.Null, token);
+            var hash = await fs.HashingCopy(Stream.Null, token, job);
             if (hash != default)
                 WriteHashCache(file, hash);
             return hash;

Wabbajack.VFS/VirtualFile.cs

@@ -162,14 +162,20 @@ namespace Wabbajack.VFS
             IPath relPath, CancellationToken token, int depth = 0)
         {
             Hash hash;
-            if (extractedFile is NativeFileStreamFactory)
+            using (var job = await context.Limiter.Begin("Hash file", 0, token))
             {
-                hash = await context.HashCache.FileHashCachedAsync((AbsolutePath)extractedFile.Name, token);
+                if (extractedFile is NativeFileStreamFactory)
-            }
+                {
-            else
+                    var absPath = (AbsolutePath)extractedFile.Name;
-            {
+                    job.Size = absPath.Size();
-                await using var hstream = await extractedFile.GetStream();
+                    hash = await context.HashCache.FileHashCachedAsync(absPath, token, job);
-                hash = await hstream.HashingCopy(Stream.Null, token);
+                }
+                else
+                {
+                    await using var hstream = await extractedFile.GetStream();
+                    job.Size = hstream.Length;
+                    hash = await hstream.HashingCopy(Stream.Null, token, job);
+                }
             }
 
             if (context.VfsCache.TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
@@ -195,7 +201,9 @@ namespace Wabbajack.VFS
             if (TextureExtensions.Contains(relPath.FileName.Extension) && await DDSSig.MatchesAsync(stream) != null)
                 try
                 {
+                    using var job = await context.Limiter.Begin("Perceptual hash", self.Size, token);
                     self.ImageState = await ImageLoader.Load(stream);
+                    await job.Report((int)self.Size, token);
                     stream.Position = 0;
                 }
                 catch (Exception)