More compiler fixes

Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs

@@ -194,7 +194,7 @@ namespace Wabbajack
                 {
                     State = CompilerState.Compiling;
 
-                    var mo2Settings = new MO2CompilerSettings
+                    var mo2Settings = new CompilerSettings
                     {
                         Game = BaseGame,
                         ModListName = ModListName,
@@ -210,19 +210,7 @@ namespace Wabbajack
                         UseGamePaths = true
                     };
 
-                    var compiler = new MO2Compiler(_serviceProvider.GetRequiredService<ILogger<MO2Compiler>>(),
+                    var compiler = MO2Compiler.Create(_serviceProvider, mo2Settings);
-                        _serviceProvider.GetRequiredService<FileExtractor.FileExtractor>(),
-                        _serviceProvider.GetRequiredService<FileHashCache>(),
-                        _serviceProvider.GetRequiredService<Context>(),
-                        _serviceProvider.GetRequiredService<TemporaryFileManager>(),
-                        mo2Settings,
-                        _serviceProvider.GetRequiredService<ParallelOptions>(),
-                        _serviceProvider.GetRequiredService<DownloadDispatcher>(),
-                        _serviceProvider.GetRequiredService<Client>(),
-                        _serviceProvider.GetRequiredService<IGameLocator>(),
-                        _serviceProvider.GetRequiredService<DTOSerializer>(),
-                        _serviceProvider.GetRequiredService<IResource<ACompiler>>(),
-                        _serviceProvider.GetRequiredService<IBinaryPatchCache>());
 
                     await compiler.Begin(CancellationToken.None);
 

Wabbajack.CLI/Program.cs

@@ -73,6 +73,7 @@ internal class Program
                 services.AddSingleton<IVerb, Extract>();
                 services.AddSingleton<IVerb, DumpZipInfo>();
                 services.AddSingleton<IVerb, Install>();
+                services.AddSingleton<IVerb, InstallCompileInstallVerify>();
 
                 services.AddSingleton<IUserInterventionHandler, UserInterventionHandler>();
             }).Build();

Wabbajack.CLI/Verbs/InstallCompileInstallVerify.cs

@@ -0,0 +1,140 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.CommandLine;
+using System.CommandLine.Invocation;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.Compiler;
+using Wabbajack.Downloaders;
+using Wabbajack.Downloaders.GameFile;
+using Wabbajack.DTOs;
+using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Installer;
+using Wabbajack.Networking.WabbajackClientApi;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+using Wabbajack.VFS;
+
+namespace Wabbajack.CLI.Verbs;
+
+public class InstallCompileInstallVerify : IVerb
+{
+    private readonly ILogger<InstallCompileInstallVerify> _logger;
+    private readonly Client _wjClient;
+    private readonly DownloadDispatcher _dispatcher;
+
+    private readonly DTOSerializer _dtos;
+    private readonly IServiceProvider _serviceProvider;
+    private readonly FileHashCache _cache;
+    private readonly GameLocator _gameLocator;
+    private readonly CompilerSettingsInferencer _inferencer;
+
+    public InstallCompileInstallVerify(ILogger<InstallCompileInstallVerify> logger, Client wjClient, DownloadDispatcher dispatcher, DTOSerializer dtos, 
+        FileHashCache cache, GameLocator gameLocator, IServiceProvider serviceProvider, CompilerSettingsInferencer inferencer)
+    {
+        _logger = logger;
+        _wjClient = wjClient;
+        _dispatcher = dispatcher;
+        _dtos = dtos;
+        _serviceProvider = serviceProvider;
+        _cache = cache;
+        _gameLocator = gameLocator;
+        _inferencer = inferencer;
+    }
+    
+    public Command MakeCommand()
+    {
+        var command = new Command("install-compile-install-verify");
+        command.Add(new Option<AbsolutePath>(new[] {"-m", "-machineUrls"}, "Machine url(s) to download"));
+        command.Add(new Option<AbsolutePath>(new[] {"-d", "-downloads"}, "Downloads path"));
+        command.Add(new Option<AbsolutePath>(new[] {"-o", "-outputs"}, "Outputs path"));
+        command.Description = "Installs a modlist, compiles it, installs it again, verifies it";
+        command.Handler = CommandHandler.Create(Run);
+        return command;
+    }
+
+    public async Task<int> Run(AbsolutePath outputs, AbsolutePath downloads, IEnumerable<string> machineUrls, CancellationToken token)
+    {
+        foreach (var machineUrl in machineUrls)
+        {
+            _logger.LogInformation("Installing {MachineUrl}", machineUrl);
+            var wabbajackPath = downloads.Combine(machineUrl.Replace("/", "_@@_"));
+            if (!await DownloadMachineUrl(machineUrl, wabbajackPath, token))
+                throw new Exception("Can't download modlist");
+
+            var installPath = outputs.Combine(machineUrl);
+            
+            var modlist = await StandardInstaller.LoadFromFile(_dtos, wabbajackPath);
+
+            var installer = StandardInstaller.Create(_serviceProvider, new InstallerConfiguration
+            {
+                Downloads = downloads,
+                Install = installPath,
+                ModList = modlist,
+                Game = modlist.GameType,
+                ModlistArchive = wabbajackPath,
+                GameFolder = _gameLocator.GameLocation(modlist.GameType)
+            });
+
+            var result = await installer.Begin(token);
+            if (!result)
+            {
+                _logger.LogInformation("Error installing {MachineUrl}", machineUrl);
+                return 1;
+            }
+
+            _logger.LogInformation("Inferring settings");
+            var inferedSettings = await _inferencer.InferFromRootPath(installPath);
+            if (inferedSettings == null)
+            {
+                _logger.LogInformation("Error inferencing settings for {MachineUrl}", machineUrl);
+                return 2;
+            }
+
+            inferedSettings.UseGamePaths = true;
+            
+
+            var compiler = MO2Compiler.Create(_serviceProvider, inferedSettings);
+            result = await compiler.Begin(token);
+
+            return result ? 0 : 3;
+
+        }
+
+        return 0;
+    }
+    
+    private async Task<bool> DownloadMachineUrl(string machineUrl, AbsolutePath wabbajack, CancellationToken token)
+    {
+        _logger.LogInformation("Downloading {MachineUrl}", machineUrl);
+
+        var lists = await _wjClient.LoadLists();
+        var list = lists.FirstOrDefault(l => l.NamespacedName == machineUrl);
+        if (list == null)
+        {
+            _logger.LogInformation("Couldn't find list {MachineUrl}", machineUrl);
+            return false;
+        }
+        
+        if (wabbajack.FileExists() && await _cache.FileHashCachedAsync(wabbajack, token) == list.DownloadMetadata!.Hash)
+        {
+            _logger.LogInformation("File already exists, using cached file");
+            return true;
+        }
+
+        var state = _dispatcher.Parse(new Uri(list.Links.Download));
+
+        await _dispatcher.Download(new Archive
+        {
+            Name = wabbajack.FileName.ToString(),
+            Hash = list.DownloadMetadata!.Hash,
+            Size = list.DownloadMetadata.Size,
+            State = state!
+        }, wabbajack, token);
+
+        return true;
+    }
+}

Wabbajack.Common/Ext.cs

@@ -21,7 +21,6 @@ public static class Ext
     public static Extension Md = new(".md");
     public static Extension MetaData = new(".metadata");
     public static Extension CompilerSettings = new(".compiler_settings");
-    public static Extension MO2CompilerSettings = new(".mo2_compiler_settings");
     public static Extension Temp = new(".temp");
     public static Extension ModlistMetadataExtension = new(".modlist_metadata");
     public static Extension Txt = new(".txt");

Wabbajack.Compiler.Test/ModListHarness.cs

@@ -66,7 +66,7 @@ public class ModListHarness
         return mod;
     }
 
-    public async Task<ModList?> CompileAndInstall(Action<MO2CompilerSettings>? configureSettings = null)
+    public async Task<ModList?> CompileAndInstall(Action<CompilerSettings>? configureSettings = null)
     {
         var modlist = await Compile(configureSettings);
         await Install();
@@ -74,13 +74,13 @@ public class ModListHarness
         return modlist;
     }
 
-    public async Task<ModList?> Compile(Action<MO2CompilerSettings>? configureSettings = null)
+    public async Task<ModList?> Compile(Action<CompilerSettings>? configureSettings = null)
     {
         configureSettings ??= x => { };
 
         _source.Combine(Consts.MO2Profiles, _profileName).CreateDirectory();
         using var scope = _serviceProvider.CreateScope();
-        var settings = scope.ServiceProvider.GetService<MO2CompilerSettings>()!;
+        var settings = scope.ServiceProvider.GetService<CompilerSettings>()!;
         settings.Downloads = _downloadPath;
         settings.Game = Game.SkyrimSpecialEdition;
         settings.Source = _source;

Wabbajack.Compiler.Test/SanityTests.cs

@@ -56,7 +56,7 @@ public class CompilerSanityTests : IAsyncLifetime
     {
     }
 
-    private async Task CompileAndValidate(int expectedDirectives, Action<MO2CompilerSettings>? configureSettings = null)
+    private async Task CompileAndValidate(int expectedDirectives, Action<CompilerSettings>? configureSettings = null)
     {
         _modlist = await _harness.Compile(configureSettings);
         Assert.NotNull(_modlist);

Wabbajack.Compiler/ACompiler.cs

@@ -275,14 +275,14 @@ public abstract class ACompiler
                 if (resolved == null) return null;
 
                 a.State = resolved.State;
-                return null;
+                return a;
             }
             catch (Exception ex)
             {
-                _logger.LogWarning(ex.ToString(), "While resolving archive {archive}", a.Name);
+                _logger.LogWarning(ex, "While resolving archive {Archive}", a.Name);
-                return a;
+                return null;
             }
-        }).ToHashSet(f => f != null);
+        }).ToHashSet(f => f == null);
 
         if (remove.Count == 0) return;
 
@@ -461,6 +461,7 @@ public abstract class ACompiler
                             _logger.LogInformation("Patching {from} {to}", destFile, match.To);
                             await using var srcStream = await sf.GetStream();
                             await using var destStream = await destsfn.GetStream();
+                            using var _ = await CompilerLimiter.Begin($"Patching {match.To}", 100, token);
                             var patchSize =
                                 await _patchCache.CreatePatch(srcStream, vf.Hash, destStream, destvf.Hash);
                             _logger.LogInformation("Patch size {patchSize} for {to}", patchSize, match.To);

Wabbajack.Compiler/CompilerSettingsInferencer.cs

@@ -23,6 +23,14 @@ public class CompilerSettingsInferencer
         _logger = logger;
 
     }
+
+    public async Task<CompilerSettings?> InferFromRootPath(AbsolutePath rootPath)
+    {
+        var mo2File = rootPath.Combine(Consts.MO2IniName).LoadIniFile();
+        var profile = mo2File["General"]["selected_profile"];
+
+        return await InferModListFromLocation(rootPath.Combine(Consts.MO2Profiles, profile, Consts.ModListTxt));
+    }
     
     public async Task<CompilerSettings?> InferModListFromLocation(AbsolutePath settingsFile)
     {

Wabbajack.Compiler/MO2Compiler.cs

@@ -1,9 +1,11 @@
+using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using IniParser.Model;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Wabbajack.Common;
 using Wabbajack.Compiler.CompilationSteps;
@@ -25,7 +27,7 @@ public class MO2Compiler : ACompiler
 {
     public MO2Compiler(ILogger<MO2Compiler> logger, FileExtractor.FileExtractor extractor, FileHashCache hashCache,
         Context vfs,
-        TemporaryFileManager manager, MO2CompilerSettings settings, ParallelOptions parallelOptions,
+        TemporaryFileManager manager, CompilerSettings settings, ParallelOptions parallelOptions,
         DownloadDispatcher dispatcher,
         Client wjClient, IGameLocator locator, DTOSerializer dtos, IResource<ACompiler> compilerLimiter,
         IBinaryPatchCache patchCache) :
@@ -35,7 +37,24 @@ public class MO2Compiler : ACompiler
         MaxSteps = 14;
     }
 
-    public MO2CompilerSettings Mo2Settings => (MO2CompilerSettings) Settings;
+    public static MO2Compiler Create(IServiceProvider provider, CompilerSettings mo2Settings)
+    {
+        return new MO2Compiler(provider.GetRequiredService<ILogger<MO2Compiler>>(),
+            provider.GetRequiredService<FileExtractor.FileExtractor>(),
+            provider.GetRequiredService<FileHashCache>(),
+            provider.GetRequiredService<Context>(),
+            provider.GetRequiredService<TemporaryFileManager>(),
+            mo2Settings,
+            provider.GetRequiredService<ParallelOptions>(),
+            provider.GetRequiredService<DownloadDispatcher>(),
+            provider.GetRequiredService<Client>(),
+            provider.GetRequiredService<IGameLocator>(),
+            provider.GetRequiredService<DTOSerializer>(),
+            provider.GetRequiredService<IResource<ACompiler>>(),
+            provider.GetRequiredService<IBinaryPatchCache>());
+    }
+
+    public CompilerSettings Mo2Settings => (CompilerSettings) Settings;
 
     public AbsolutePath MO2ModsFolder => Settings.Source.Combine(Consts.MO2ModFolderName);
 

Wabbajack.Compiler/MO2CompilerSettings.cs

@@ -1,9 +0,0 @@
-using System;
-using Wabbajack.Paths;
-
-namespace Wabbajack.Compiler;
-
-public class MO2CompilerSettings : CompilerSettings
-{
-
-}

Wabbajack.Compiler/PatchCache/BinaryPatchCache.cs

@@ -6,6 +6,7 @@ using Wabbajack.Compiler.PatchCache;
 using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Paths;
 using Wabbajack.Paths.IO;
+using Wabbajack.RateLimiter;
 
 namespace Wabbajack.Compiler;
 
@@ -38,7 +39,7 @@ public class BinaryPatchCache : IBinaryPatchCache
         cmd.ExecuteNonQuery();
     }
 
-    public async Task<CacheEntry> CreatePatch(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash)
+    public async Task<CacheEntry> CreatePatch(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, IJob? job)
     {
         await using var rcmd = new SQLiteCommand(_conn);
         rcmd.CommandText = "SELECT PatchSize FROM PatchCache WHERE FromHash = @fromHash AND ToHash = @toHash";
@@ -57,7 +58,7 @@ public class BinaryPatchCache : IBinaryPatchCache
 
         await using var sigStream = new MemoryStream();
         await using var patchStream = new MemoryStream();
-        OctoDiff.Create(srcStream, destStream, sigStream, patchStream);
+        OctoDiff.Create(srcStream, destStream, sigStream, patchStream, job);
 
         cmd.Parameters.AddWithValue("@patchSize", patchStream.Length);
         cmd.Parameters.AddWithValue("@patch", patchStream.ToArray());

Wabbajack.Compiler/PatchCache/IBinaryPatchCache.cs

@@ -2,12 +2,13 @@ using System.IO;
 using System.Threading.Tasks;
 using Wabbajack.Compiler.PatchCache;
 using Wabbajack.Hashing.xxHash64;
+using Wabbajack.RateLimiter;
 
 namespace Wabbajack.Compiler;
 
 public interface IBinaryPatchCache
 {
-    public Task<CacheEntry> CreatePatch(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash);
+    public Task<CacheEntry> CreatePatch(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, IJob? job = null);
 
     public Task<CacheEntry?> GetPatch(Hash hashA, Hash hashB);
     public Task<byte[]> GetData(CacheEntry entry);

Wabbajack.Compiler/PatchCache/OctoDiff.cs

@@ -1,6 +1,8 @@
 using System.IO;
+using System.Threading;
 using Octodiff.Core;
 using Octodiff.Diagnostics;
+using Wabbajack.RateLimiter;
 
 namespace Wabbajack.Compiler.PatchCache;
 
@@ -33,11 +35,29 @@ public class OctoDiff
         sigStream.Position = 0;
     }
 
-    public static void Create(Stream oldData, Stream newData, Stream signature, Stream output)
+    public static void Create(Stream oldData, Stream newData, Stream signature, Stream output, IJob? job)
     {
         CreateSignature(oldData, signature);
-        var db = new DeltaBuilder {ProgressReporter = new NullProgressReporter()};
+        var db = new DeltaBuilder {ProgressReporter = new JobProgressReporter(job, 0)};
-        db.BuildDelta(newData, new SignatureReader(signature, new NullProgressReporter()),
+        db.BuildDelta(newData, new SignatureReader(signature, new JobProgressReporter(job, 100)),
             new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
     }
+
+    private class JobProgressReporter : IProgressReporter
+    {
+        private readonly IJob _job;
+        private readonly int _offset;
+
+        public JobProgressReporter(IJob job, int offset)
+        {
+            _offset = offset;
+            _job = job;
+        }
+        public void ReportProgress(string operation, long currentPosition, long total)
+        {
+            var percent = Percent.FactoryPutInRange(currentPosition, total);
+            var toReport = (long) (percent.Value * 100) - (_job.Current - _offset);
+            _job.ReportNoWait((int) toReport);
+        }
+    }
 }

Wabbajack.Installer/AInstaller.cs

@@ -399,8 +399,12 @@ public abstract class AInstaller<T>
             .Concat(_gameLocator.GameLocation(_configuration.Game).EnumerateFiles())
             .ToList();
 
-        var hashDict = allFiles.GroupBy(f => f.Size()).ToDictionary(g => g.Key);
+        _logger.LogInformation("Getting archive sizes");
+        var hashDict = (await allFiles.PMapAll(_limiter, async x => (x, x.Size())).ToList())
+            .GroupBy(f => f.Item2)
+            .ToDictionary(g => g.Key, g => g.Select(v => v.x));
 
+        _logger.LogInformation("Linking archives to downloads");
         var toHash = ModList.Archives.Where(a => hashDict.ContainsKey(a.Size))
             .SelectMany(a => hashDict[a.Size]).ToList();
 

Wabbajack.RateLimiter/IJob.cs

@@ -10,4 +10,5 @@ public interface IJob
     public long Current { get; }
     public string Description { get; }
     public ValueTask Report(int processedSize, CancellationToken token);
+    public void ReportNoWait(int processedSize);
 }

Wabbajack.Services.OSIntegrated/ServiceExtensions.cs

@@ -152,7 +152,7 @@ public static class ServiceExtensions
         // Installer/Compiler Configuration
         service.AddScoped<InstallerConfiguration>();
         service.AddScoped<StandardInstaller>();
-        service.AddScoped<MO2CompilerSettings>();
+        service.AddScoped<CompilerSettings>();
         service.AddScoped<MO2Compiler>();
         service.AddSingleton<CompilerSettingsInferencer>();