Compiler works, needs a lot of refinement

Wabbajack.App/App.axaml.cs

@@ -6,6 +6,7 @@ using Avalonia.Markup.Xaml;
 using CefNet;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
 using ReactiveUI;
 using Splat;
 using Wabbajack.App.Controls;
@@ -36,6 +37,10 @@ namespace Wabbajack.App
         public override void OnFrameworkInitializationCompleted()
         {
             var host = Host.CreateDefaultBuilder(Array.Empty<string>())
+                .ConfigureLogging(c =>
+                {
+                    c.ClearProviders();
+                })
                 .ConfigureServices((host, services) =>
                 {
                     services.AddAppServices();

Wabbajack.App/Controls/LogViewModel.cs

@@ -21,7 +21,7 @@ public class LogViewModel : ViewModelBase, IActivatableViewModel
     public LogViewModel(LoggerProvider provider)
     {
         _messages = new SourceCache<LoggerProvider.ILogMessage, long>(m => m.MessageId);
-        _messages.LimitSizeTo(100);
+        //_messages.LimitSizeTo(100);
         
         Activator = new ViewModelActivator();
         _provider = provider;

Wabbajack.App/Models/SettingsManager.cs

@@ -46,9 +46,9 @@ public class SettingsManager
         {
             if (path.FileExists())
             {
-                await using (var s = path.Open(FileMode.Create, FileAccess.Write))
+                await using (var s = path.Open(FileMode.Open))
                 {
-                    await JsonSerializer.DeserializeAsync<T>(s, _dtos.Options);
+                    return (await JsonSerializer.DeserializeAsync<T>(s, _dtos.Options))!;
                 }
             }
         }

Wabbajack.App/Screens/CompilationView.axaml.cs

@@ -1,3 +1,5 @@
+using Avalonia.Controls.Mixins;
+using ReactiveUI;
 using Wabbajack.App.ViewModels;
 using Wabbajack.App.Views;
 
@@ -8,5 +10,17 @@ public partial class CompilationView : ScreenBase<CompilationViewModel>
     public CompilationView()
     {
         InitializeComponent();
+
+        this.WhenActivated(disposables =>
+        {
+            this.OneWayBind(ViewModel, vm => vm.StatusText, view => view.StatusText.Text)
+                .DisposeWith(disposables);
+
+            this.OneWayBind(ViewModel, vm => vm.StepsProgress, view => view.StepsProgress.Value, p => p.Value * 1000)
+                .DisposeWith(disposables);
+
+            this.OneWayBind(ViewModel, vm => vm.StepProgress, view => view.StepProgress.Value, p => p.Value * 10000)
+                .DisposeWith(disposables);
+        });
     }
 }

Wabbajack.App/Screens/CompilationViewModel.cs

@@ -1,13 +1,16 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
+using Avalonia.Threading;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using ReactiveUI;
+using ReactiveUI.Fody.Helpers;
 using Wabbajack.App.Messages;
 using Wabbajack.App.ViewModels;
 using Wabbajack.Common;
 using Wabbajack.Compiler;
+using Wabbajack.RateLimiter;
 
 namespace Wabbajack.App.Screens;
 
@@ -17,6 +20,11 @@ public class CompilationViewModel : ViewModelBase, IReceiverMarker, IReceiver<St
     private ACompiler _compiler;
     private readonly ILogger<CompilationViewModel> _logger;
     
+    [Reactive] public string StatusText { get; set; } = "";
+    [Reactive] public Percent StepsProgress { get; set; } = Percent.Zero;
+    [Reactive] public Percent StepProgress { get; set; } = Percent.Zero;
+
+
     public CompilationViewModel(ILogger<CompilationViewModel> logger, IServiceProvider provider)
     {
         _logger = logger;
@@ -32,7 +40,15 @@ public class CompilationViewModel : ViewModelBase, IReceiverMarker, IReceiver<St
             var compiler = _provider.GetService<MO2Compiler>()!;
             compiler.Settings = mo2;
             _compiler = compiler;
-
+            _compiler.OnStatusUpdate += (sender, update) =>
+            {
+                Dispatcher.UIThread.InvokeAsync(() =>
+                {
+                    StatusText = update.StatusText;
+                    StepsProgress = update.StepsProgress;
+                    StepProgress = update.StepProgress;
+                });
+            };
         }
         Compile().FireAndForget();
     }
@@ -46,6 +62,7 @@ public class CompilationViewModel : ViewModelBase, IReceiverMarker, IReceiver<St
         catch (Exception ex)
         {
             _logger.LogError(ex, "During Compilation: {Message}", ex.Message);
+            StatusText = $"ERRORED: {ex.Message}";
         }
     }
 }

Wabbajack.App/Screens/CompilerConfigurationViewModel.cs

@@ -3,15 +3,11 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Reactive;
-using System.Reactive.Disposables;
-using System.Reactive.Linq;
 using System.Text.Json;
 using System.Threading.Tasks;
 using Avalonia.Controls.Mixins;
-using DynamicData;
 using ReactiveUI;
 using ReactiveUI.Fody.Helpers;
-using Wabbajack.App.Extensions;
 using Wabbajack.App.Messages;
 using Wabbajack.App.Models;
 using Wabbajack.App.ViewModels;
@@ -82,18 +78,29 @@ public class CompilerConfigurationViewModel : ViewModelBase, IReceiverMarker
 
         OutputFolder = KnownFolders.EntryPoint;
         
-        this.WhenActivated((CompositeDisposable disposables) =>
+        this.WhenActivated(disposables =>
         {
             LoadLastCompilation().FireAndForget();
+            this.WhenAnyValue(v => v.SettingsFile)
+                .Subscribe( location =>
+                {
+                    LoadNewSettingsFile(location).FireAndForget();
+                })
+                .DisposeWith(disposables);
         });
         
     }
 
+    private async Task LoadNewSettingsFile(AbsolutePath location)
+    {
+        if (location == default) return;
+        if (location.FileExists()) await LoadSettings(location);
+    }
+
     private async Task LoadLastCompilation()
     {
         var location = await _settingsManager.Load<AbsolutePath>("last_compilation");
-        if (location == default) return;
+        SettingsFile = location;
-        if (location.FileExists()) await LoadSettings(location);
     }
 
     private async Task BeginCompilation()

Wabbajack.Compiler/ACompiler.cs

@@ -21,6 +21,7 @@ using Wabbajack.Installer;
 using Wabbajack.Networking.WabbajackClientApi;
 using Wabbajack.Paths;
 using Wabbajack.Paths.IO;
+using Wabbajack.RateLimiter;
 using Wabbajack.VFS;
 
 namespace Wabbajack.Compiler
@@ -50,6 +51,16 @@ namespace Wabbajack.Compiler
         private readonly DTOSerializer _dtos;
         public readonly IBinaryPatchCache _patchCache;
         
+        private long _maxStepProgress = 0;
+        private int _currentStep = 0;
+        private string _statusText;
+        private long _currentStepProgress;
+        private readonly Stopwatch _updateStopWatch = new();
+
+        protected long MaxSteps { get; set; }
+
+        public event EventHandler<StatusUpdate> OnStatusUpdate; 
+
         public ACompiler(ILogger logger, FileExtractor.FileExtractor extractor, FileHashCache hashCache, Context vfs, TemporaryFileManager manager, CompilerSettings settings,
             ParallelOptions parallelOptions, DownloadDispatcher dispatcher, Client wjClient, IGameLocator locator, DTOSerializer dtos,
             IBinaryPatchCache patchCache)
@@ -70,9 +81,42 @@ namespace Wabbajack.Compiler
             _patchOptions = new();
             _sourceFileLinks = new();
             _patchCache = patchCache;
+            _updateStopWatch = new();
 
         }
         
+        public void NextStep(string statusText, long maxStepProgress = 1)
+        {
+            _updateStopWatch.Restart();
+            _maxStepProgress = maxStepProgress;
+            _currentStep += 1;
+            _statusText = statusText;
+            _logger.LogInformation("Compiler Step: {Step}", statusText);
+
+            if (OnStatusUpdate != null)
+            {
+                OnStatusUpdate(this, new StatusUpdate($"[{_currentStep}/{MaxSteps}] " + statusText, Percent.FactoryPutInRange(_currentStep, MaxSteps),
+                    Percent.Zero));
+            }
+        }
+
+        public void UpdateProgress(long stepProgress)
+        {
+            Interlocked.Add(ref _currentStepProgress, stepProgress);
+
+            lock (_updateStopWatch)
+            {
+                if (_updateStopWatch.ElapsedMilliseconds < 100) return;
+                _updateStopWatch.Restart();
+            }
+
+            if (OnStatusUpdate != null)
+            {
+                OnStatusUpdate(this, new StatusUpdate(_statusText, Percent.FactoryPutInRange(_currentStep, MaxSteps),
+                    Percent.FactoryPutInRange(_currentStepProgress, _maxStepProgress)));
+            }
+        }
+
         public abstract Task<bool> Begin(CancellationToken token);
 
         public CompilerSettings Settings
@@ -151,8 +195,10 @@ namespace Wabbajack.Compiler
         public async Task<bool> GatherMetaData()
         {
             _logger.LogInformation("Getting meta data for {count} archives", SelectedArchives.Count);
+            NextStep("Gathering Metadata", SelectedArchives.Count);
             await SelectedArchives.PDo(_parallelOptions, async a =>
             {
+                UpdateProgress(1);
                 await _dispatcher.FillInMetadata(a);
             });
 
@@ -162,6 +208,7 @@ namespace Wabbajack.Compiler
 
         protected async Task IndexGameFileHashes()
         {
+            NextStep("Indexing Game Files", 1);
             if (_settings.UseGamePaths)
             {
                 //taking the games in Settings.IncludedGames + currently compiling game so you can eg
@@ -215,6 +262,7 @@ namespace Wabbajack.Compiler
 
         protected async Task CleanInvalidArchivesAndFillState()
         {
+            NextStep("Cleaning Invalid Archives", 1);
             var remove = await IndexedArchives.PMap(_parallelOptions, async a =>
             {
                 try
@@ -248,6 +296,7 @@ namespace Wabbajack.Compiler
 
         protected async Task InferMetas(CancellationToken token)
         {
+
             async Task<bool> HasInvalidMeta(AbsolutePath filename)
             {
                 var metaName = filename.WithExtension(Ext.Meta);
@@ -275,6 +324,7 @@ namespace Wabbajack.Compiler
                 .Where(f => f.FileExists())
                 .ToList();
 
+            NextStep("InferMetas", toFind.Count);
             if (toFind.Count == 0)
             {
                 return;
@@ -284,6 +334,7 @@ namespace Wabbajack.Compiler
 
             await toFind.PDoAll(async f =>
             {
+                UpdateProgress(1);
                 var vf = _vfs.Index.ByRootPath[f];
 
                 var archives = await _wjClient.GetArchivesForHash(vf.Hash);
@@ -315,6 +366,7 @@ namespace Wabbajack.Compiler
 
         protected async Task ExportModList(CancellationToken token)
         {
+            NextStep("Exporting Modlist");
             _logger.LogInformation("Exporting ModList to {location}", _settings.OutputFile);
 
             // Modify readme and ModList image to relative paths if they exist
@@ -382,8 +434,6 @@ namespace Wabbajack.Compiler
         /// </summary>
         protected async Task BuildPatches(CancellationToken token)
         {
-            _logger.LogInformation("Gathering patch files");
-
             var toBuild = InstallDirectives.OfType<PatchedFromArchive>()
                 .Where(p => _patchOptions.GetValueOrDefault(p, Array.Empty<VirtualFile>()).Length > 0)
                 .SelectMany(p => _patchOptions[p].Select(c => new PatchedFromArchive
@@ -395,6 +445,7 @@ namespace Wabbajack.Compiler
                 }))
                 .ToArray();
 
+            NextStep("Generating Patches", toBuild.Length);
             if (toBuild.Length == 0)
             {
                 return;
@@ -406,6 +457,7 @@ namespace Wabbajack.Compiler
             await _vfs.Extract( indexed.Keys.ToHashSet(),
                 async (vf, sf) =>
                 {
+                    UpdateProgress(1);
                     // For each, extract the destination
                     var matches = indexed[vf];
                     foreach (var match in matches)
@@ -484,6 +536,7 @@ namespace Wabbajack.Compiler
 
         public async Task GenerateManifest()
         {
+            NextStep("Generating Manifest");
             var manifest = new Manifest(ModList);
             await using var of = _settings.OutputFile.Open(FileMode.Create, FileAccess.Write);
             await _dtos.Serialize(manifest, of);
@@ -491,6 +544,7 @@ namespace Wabbajack.Compiler
 
         public async Task GatherArchives()
         {
+            NextStep("Gathering Archives");
             _logger.LogInformation("Building a list of archives based on the files required");
 
             var hashes = InstallDirectives.OfType<FromArchive>()
@@ -502,7 +556,11 @@ namespace Wabbajack.Compiler
                 .ToDictionary(f => f.Key, f => f.First());
 
             SelectedArchives.Clear();
-            SelectedArchives.AddRange(await hashes.PMap(_parallelOptions, hash => ResolveArchive(hash, archives)).ToList());
+            SelectedArchives.AddRange(await hashes.PMap(_parallelOptions, hash =>
+            {
+                UpdateProgress(1);
+                return ResolveArchive(hash, archives);
+            }).ToList());
         }
 
         public async Task<Archive> ResolveArchive(Hash hash, IDictionary<Hash, IndexedArchive> archives)
@@ -596,11 +654,13 @@ namespace Wabbajack.Compiler
                 .GroupBy(f => _sourceFileLinks[f].File)
                 .ToDictionary(k => k.Key);
 
+            NextStep("Inlining Files");
             if (grouped.Count == 0) return;
             await _vfs.Extract(grouped.Keys.ToHashSet(), async (vf, sfn) =>
             {
+                UpdateProgress(1);
                 await using var stream = await sfn.GetStream();
-                var id = await IncludeFile(stream);
+                var id = await IncludeFile(stream, token);
                 foreach (var file in grouped[vf])
                 {
                     file.SourceDataID = id;

Wabbajack.Compiler/Consts.cs

@@ -35,7 +35,7 @@ namespace Wabbajack.Compiler
         public static readonly HashSet<Extension> SupportedBSAs = new[] {".bsa", ".ba2"}
             .Select(s => new Extension(s)).ToHashSet();
         
-        public static HashSet<Extension> ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml", ".yaml"}.Select(s => new Extension(s)).ToHashSet();
+        public static HashSet<Extension> ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml", ".yaml", ".compiler_settings", ".mo2_compiler_settings"}.Select(s => new Extension(s)).ToHashSet();
         public static HashSet<Extension> ESPFileExtensions = new []{ ".esp", ".esm", ".esl"}.Select(s => new Extension(s)).ToHashSet();
         public static HashSet<Extension> AssetFileExtensions = new[] {".dds", ".tga", ".nif", ".psc", ".pex"}.Select(s => new Extension(s)).ToHashSet();
         

Wabbajack.Compiler/MO2Compiler.cs

@@ -29,6 +29,7 @@ namespace Wabbajack.Compiler
             Client wjClient, IGameLocator locator, DTOSerializer dtos, IBinaryPatchCache patchCache) : 
             base(logger, extractor, hashCache, vfs, manager, settings, parallelOptions, dispatcher, wjClient, locator, dtos, patchCache)
         {
+            MaxSteps = 14;
         }
         
         public AbsolutePath MO2ModsFolder => Settings.Source.Combine(Consts.MO2ModFolderName);
@@ -52,11 +53,13 @@ namespace Wabbajack.Compiler
             var roots = new List<AbsolutePath> {Settings.Source, Settings.Downloads};
             roots.AddRange(Settings.OtherGames.Append(Settings.Game).Select(g => _locator.GameLocation(g)));
             
-            await _vfs.AddRoots(roots, token);
+            NextStep("Add Roots", 1);
+            await _vfs.AddRoots(roots, token); // Step 1
             
-            await InferMetas(token);
+            await InferMetas(token); // Step 2
 
-            await _vfs.AddRoot(Settings.Downloads, token);
+            NextStep("Add Download Roots", 1);
+            await _vfs.AddRoot(Settings.Downloads, token); // Step 3
 
             // Find all Downloads
             IndexedArchives = await Settings.Downloads.EnumerateFiles()
@@ -69,7 +72,6 @@ namespace Wabbajack.Compiler
                         Meta = await f.WithExtension(Ext.Meta).ReadAllTextAsync()
                     }).ToList();
             
-
             await IndexGameFileHashes();
 
             IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();
@@ -117,10 +119,15 @@ namespace Wabbajack.Compiler
 
             var stack = MakeStack();
 
-            var results = await AllFiles.PMap(_parallelOptions, f => RunStack(stack, f)).ToList();
+            NextStep("Running Compilation Stack", AllFiles.Count);
+            var results = await AllFiles.PMap(_parallelOptions, f =>
+            {
+                UpdateProgress(1);
+                return RunStack(stack, f);
+            }).ToList();
 
+            NextStep("Updating Extra files");
             // Add the extra files that were generated by the stack
-
             results = results.Concat(ExtraFiles).ToList();
 
             var noMatch = results.OfType<NoMatch>().ToArray();
@@ -176,9 +183,11 @@ namespace Wabbajack.Compiler
 
         private async Task RunValidation(ModList modList)
         {
+            NextStep("Validating Archives", modList.Archives.Length);
             var allowList = await _wjClient.LoadDownloadAllowList();
             foreach (var archive in modList.Archives)
             {
+                UpdateProgress(1);
                 if (!_dispatcher.IsAllowed(archive, allowList))
                 {
                     _logger.LogCritical("Archive {name}, {primaryKeyString} is not allowed", archive.Name,

Wabbajack.Services.OSIntegrated/ServiceExtensions.cs

@@ -3,8 +3,6 @@ using System.Collections.Generic;
 using System.Net.Http;
 using System.Threading.Tasks;
 using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using Wabbajack.Common;
 using Wabbajack.Compiler;
 using Wabbajack.Downloaders;
 using Wabbajack.Downloaders.GameFile;