diff --git a/Wabbajack.App/App.axaml.cs b/Wabbajack.App/App.axaml.cs index 0e6e08be..e39aa84d 100644 --- a/Wabbajack.App/App.axaml.cs +++ b/Wabbajack.App/App.axaml.cs @@ -6,6 +6,7 @@ using Avalonia.Markup.Xaml; using CefNet; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using ReactiveUI; using Splat; using Wabbajack.App.Controls; @@ -36,6 +37,10 @@ namespace Wabbajack.App public override void OnFrameworkInitializationCompleted() { var host = Host.CreateDefaultBuilder(Array.Empty()) + .ConfigureLogging(c => + { + c.ClearProviders(); + }) .ConfigureServices((host, services) => { services.AddAppServices(); diff --git a/Wabbajack.App/Controls/LogViewModel.cs b/Wabbajack.App/Controls/LogViewModel.cs index c9c65f88..f0ea35a1 100644 --- a/Wabbajack.App/Controls/LogViewModel.cs +++ b/Wabbajack.App/Controls/LogViewModel.cs @@ -21,7 +21,7 @@ public class LogViewModel : ViewModelBase, IActivatableViewModel public LogViewModel(LoggerProvider provider) { _messages = new SourceCache(m => m.MessageId); - _messages.LimitSizeTo(100); + //_messages.LimitSizeTo(100); Activator = new ViewModelActivator(); _provider = provider; diff --git a/Wabbajack.App/Models/SettingsManager.cs b/Wabbajack.App/Models/SettingsManager.cs index ac9b0701..14050872 100644 --- a/Wabbajack.App/Models/SettingsManager.cs +++ b/Wabbajack.App/Models/SettingsManager.cs @@ -46,9 +46,9 @@ public class SettingsManager { if (path.FileExists()) { - await using (var s = path.Open(FileMode.Create, FileAccess.Write)) + await using (var s = path.Open(FileMode.Open)) { - await JsonSerializer.DeserializeAsync(s, _dtos.Options); + return (await JsonSerializer.DeserializeAsync(s, _dtos.Options))!; } } } diff --git a/Wabbajack.App/Screens/CompilationView.axaml.cs b/Wabbajack.App/Screens/CompilationView.axaml.cs index af315a11..382f3615 100644 --- a/Wabbajack.App/Screens/CompilationView.axaml.cs +++ b/Wabbajack.App/Screens/CompilationView.axaml.cs @@ -1,3 +1,5 @@ +using Avalonia.Controls.Mixins; +using ReactiveUI; using Wabbajack.App.ViewModels; using Wabbajack.App.Views; @@ -8,5 +10,17 @@ public partial class CompilationView : ScreenBase public CompilationView() { InitializeComponent(); + + this.WhenActivated(disposables => + { + this.OneWayBind(ViewModel, vm => vm.StatusText, view => view.StatusText.Text) + .DisposeWith(disposables); + + this.OneWayBind(ViewModel, vm => vm.StepsProgress, view => view.StepsProgress.Value, p => p.Value * 1000) + .DisposeWith(disposables); + + this.OneWayBind(ViewModel, vm => vm.StepProgress, view => view.StepProgress.Value, p => p.Value * 10000) + .DisposeWith(disposables); + }); } } diff --git a/Wabbajack.App/Screens/CompilationViewModel.cs b/Wabbajack.App/Screens/CompilationViewModel.cs index 7ab18f12..76c59dac 100644 --- a/Wabbajack.App/Screens/CompilationViewModel.cs +++ b/Wabbajack.App/Screens/CompilationViewModel.cs @@ -1,13 +1,16 @@ using System; using System.Threading; using System.Threading.Tasks; +using Avalonia.Threading; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using ReactiveUI; +using ReactiveUI.Fody.Helpers; using Wabbajack.App.Messages; using Wabbajack.App.ViewModels; using Wabbajack.Common; using Wabbajack.Compiler; +using Wabbajack.RateLimiter; namespace Wabbajack.App.Screens; @@ -16,6 +19,11 @@ public class CompilationViewModel : ViewModelBase, IReceiverMarker, IReceiver _logger; + + [Reactive] public string StatusText { get; set; } = ""; + [Reactive] public Percent StepsProgress { get; set; } = Percent.Zero; + [Reactive] public Percent StepProgress { get; set; } = Percent.Zero; + public CompilationViewModel(ILogger logger, IServiceProvider provider) { @@ -32,7 +40,15 @@ public class CompilationViewModel : ViewModelBase, IReceiverMarker, IReceiver()!; compiler.Settings = mo2; _compiler = compiler; - + _compiler.OnStatusUpdate += (sender, update) => + { + Dispatcher.UIThread.InvokeAsync(() => + { + StatusText = update.StatusText; + StepsProgress = update.StepsProgress; + StepProgress = update.StepProgress; + }); + }; } Compile().FireAndForget(); } @@ -46,6 +62,7 @@ public class CompilationViewModel : ViewModelBase, IReceiverMarker, IReceiver + this.WhenActivated(disposables => { LoadLastCompilation().FireAndForget(); + this.WhenAnyValue(v => v.SettingsFile) + .Subscribe( location => + { + LoadNewSettingsFile(location).FireAndForget(); + }) + .DisposeWith(disposables); }); } + private async Task LoadNewSettingsFile(AbsolutePath location) + { + if (location == default) return; + if (location.FileExists()) await LoadSettings(location); + } + private async Task LoadLastCompilation() { var location = await _settingsManager.Load("last_compilation"); - if (location == default) return; - if (location.FileExists()) await LoadSettings(location); + SettingsFile = location; } private async Task BeginCompilation() diff --git a/Wabbajack.Compiler/ACompiler.cs b/Wabbajack.Compiler/ACompiler.cs index b35973ac..1ad94075 100644 --- a/Wabbajack.Compiler/ACompiler.cs +++ b/Wabbajack.Compiler/ACompiler.cs @@ -21,6 +21,7 @@ using Wabbajack.Installer; using Wabbajack.Networking.WabbajackClientApi; using Wabbajack.Paths; using Wabbajack.Paths.IO; +using Wabbajack.RateLimiter; using Wabbajack.VFS; namespace Wabbajack.Compiler @@ -49,6 +50,16 @@ namespace Wabbajack.Compiler public readonly IGameLocator _locator; private readonly DTOSerializer _dtos; public readonly IBinaryPatchCache _patchCache; + + private long _maxStepProgress = 0; + private int _currentStep = 0; + private string _statusText; + private long _currentStepProgress; + private readonly Stopwatch _updateStopWatch = new(); + + protected long MaxSteps { get; set; } + + public event EventHandler OnStatusUpdate; public ACompiler(ILogger logger, FileExtractor.FileExtractor extractor, FileHashCache hashCache, Context vfs, TemporaryFileManager manager, CompilerSettings settings, ParallelOptions parallelOptions, DownloadDispatcher dispatcher, Client wjClient, IGameLocator locator, DTOSerializer dtos, @@ -70,8 +81,41 @@ namespace Wabbajack.Compiler _patchOptions = new(); _sourceFileLinks = new(); _patchCache = patchCache; + _updateStopWatch = new(); } + + public void NextStep(string statusText, long maxStepProgress = 1) + { + _updateStopWatch.Restart(); + _maxStepProgress = maxStepProgress; + _currentStep += 1; + _statusText = statusText; + _logger.LogInformation("Compiler Step: {Step}", statusText); + + if (OnStatusUpdate != null) + { + OnStatusUpdate(this, new StatusUpdate($"[{_currentStep}/{MaxSteps}] " + statusText, Percent.FactoryPutInRange(_currentStep, MaxSteps), + Percent.Zero)); + } + } + + public void UpdateProgress(long stepProgress) + { + Interlocked.Add(ref _currentStepProgress, stepProgress); + + lock (_updateStopWatch) + { + if (_updateStopWatch.ElapsedMilliseconds < 100) return; + _updateStopWatch.Restart(); + } + + if (OnStatusUpdate != null) + { + OnStatusUpdate(this, new StatusUpdate(_statusText, Percent.FactoryPutInRange(_currentStep, MaxSteps), + Percent.FactoryPutInRange(_currentStepProgress, _maxStepProgress))); + } + } public abstract Task Begin(CancellationToken token); @@ -151,8 +195,10 @@ namespace Wabbajack.Compiler public async Task GatherMetaData() { _logger.LogInformation("Getting meta data for {count} archives", SelectedArchives.Count); + NextStep("Gathering Metadata", SelectedArchives.Count); await SelectedArchives.PDo(_parallelOptions, async a => { + UpdateProgress(1); await _dispatcher.FillInMetadata(a); }); @@ -162,6 +208,7 @@ namespace Wabbajack.Compiler protected async Task IndexGameFileHashes() { + NextStep("Indexing Game Files", 1); if (_settings.UseGamePaths) { //taking the games in Settings.IncludedGames + currently compiling game so you can eg @@ -215,6 +262,7 @@ namespace Wabbajack.Compiler protected async Task CleanInvalidArchivesAndFillState() { + NextStep("Cleaning Invalid Archives", 1); var remove = await IndexedArchives.PMap(_parallelOptions, async a => { try @@ -248,6 +296,7 @@ namespace Wabbajack.Compiler protected async Task InferMetas(CancellationToken token) { + async Task HasInvalidMeta(AbsolutePath filename) { var metaName = filename.WithExtension(Ext.Meta); @@ -275,6 +324,7 @@ namespace Wabbajack.Compiler .Where(f => f.FileExists()) .ToList(); + NextStep("InferMetas", toFind.Count); if (toFind.Count == 0) { return; @@ -284,6 +334,7 @@ namespace Wabbajack.Compiler await toFind.PDoAll(async f => { + UpdateProgress(1); var vf = _vfs.Index.ByRootPath[f]; var archives = await _wjClient.GetArchivesForHash(vf.Hash); @@ -315,6 +366,7 @@ namespace Wabbajack.Compiler protected async Task ExportModList(CancellationToken token) { + NextStep("Exporting Modlist"); _logger.LogInformation("Exporting ModList to {location}", _settings.OutputFile); // Modify readme and ModList image to relative paths if they exist @@ -382,8 +434,6 @@ namespace Wabbajack.Compiler /// protected async Task BuildPatches(CancellationToken token) { - _logger.LogInformation("Gathering patch files"); - var toBuild = InstallDirectives.OfType() .Where(p => _patchOptions.GetValueOrDefault(p, Array.Empty()).Length > 0) .SelectMany(p => _patchOptions[p].Select(c => new PatchedFromArchive @@ -395,6 +445,7 @@ namespace Wabbajack.Compiler })) .ToArray(); + NextStep("Generating Patches", toBuild.Length); if (toBuild.Length == 0) { return; @@ -406,6 +457,7 @@ namespace Wabbajack.Compiler await _vfs.Extract( indexed.Keys.ToHashSet(), async (vf, sf) => { + UpdateProgress(1); // For each, extract the destination var matches = indexed[vf]; foreach (var match in matches) @@ -484,6 +536,7 @@ namespace Wabbajack.Compiler public async Task GenerateManifest() { + NextStep("Generating Manifest"); var manifest = new Manifest(ModList); await using var of = _settings.OutputFile.Open(FileMode.Create, FileAccess.Write); await _dtos.Serialize(manifest, of); @@ -491,6 +544,7 @@ namespace Wabbajack.Compiler public async Task GatherArchives() { + NextStep("Gathering Archives"); _logger.LogInformation("Building a list of archives based on the files required"); var hashes = InstallDirectives.OfType() @@ -502,7 +556,11 @@ namespace Wabbajack.Compiler .ToDictionary(f => f.Key, f => f.First()); SelectedArchives.Clear(); - SelectedArchives.AddRange(await hashes.PMap(_parallelOptions, hash => ResolveArchive(hash, archives)).ToList()); + SelectedArchives.AddRange(await hashes.PMap(_parallelOptions, hash => + { + UpdateProgress(1); + return ResolveArchive(hash, archives); + }).ToList()); } public async Task ResolveArchive(Hash hash, IDictionary archives) @@ -596,11 +654,13 @@ namespace Wabbajack.Compiler .GroupBy(f => _sourceFileLinks[f].File) .ToDictionary(k => k.Key); + NextStep("Inlining Files"); if (grouped.Count == 0) return; await _vfs.Extract(grouped.Keys.ToHashSet(), async (vf, sfn) => { + UpdateProgress(1); await using var stream = await sfn.GetStream(); - var id = await IncludeFile(stream); + var id = await IncludeFile(stream, token); foreach (var file in grouped[vf]) { file.SourceDataID = id; diff --git a/Wabbajack.Compiler/Consts.cs b/Wabbajack.Compiler/Consts.cs index abb18133..52f6d6b2 100644 --- a/Wabbajack.Compiler/Consts.cs +++ b/Wabbajack.Compiler/Consts.cs @@ -35,7 +35,7 @@ namespace Wabbajack.Compiler public static readonly HashSet SupportedBSAs = new[] {".bsa", ".ba2"} .Select(s => new Extension(s)).ToHashSet(); - public static HashSet ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml", ".yaml"}.Select(s => new Extension(s)).ToHashSet(); + public static HashSet ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml", ".yaml", ".compiler_settings", ".mo2_compiler_settings"}.Select(s => new Extension(s)).ToHashSet(); public static HashSet ESPFileExtensions = new []{ ".esp", ".esm", ".esl"}.Select(s => new Extension(s)).ToHashSet(); public static HashSet AssetFileExtensions = new[] {".dds", ".tga", ".nif", ".psc", ".pex"}.Select(s => new Extension(s)).ToHashSet(); diff --git a/Wabbajack.Compiler/MO2Compiler.cs b/Wabbajack.Compiler/MO2Compiler.cs index 0cdeff4a..6637dbdb 100644 --- a/Wabbajack.Compiler/MO2Compiler.cs +++ b/Wabbajack.Compiler/MO2Compiler.cs @@ -29,6 +29,7 @@ namespace Wabbajack.Compiler Client wjClient, IGameLocator locator, DTOSerializer dtos, IBinaryPatchCache patchCache) : base(logger, extractor, hashCache, vfs, manager, settings, parallelOptions, dispatcher, wjClient, locator, dtos, patchCache) { + MaxSteps = 14; } public AbsolutePath MO2ModsFolder => Settings.Source.Combine(Consts.MO2ModFolderName); @@ -52,11 +53,13 @@ namespace Wabbajack.Compiler var roots = new List {Settings.Source, Settings.Downloads}; roots.AddRange(Settings.OtherGames.Append(Settings.Game).Select(g => _locator.GameLocation(g))); - await _vfs.AddRoots(roots, token); + NextStep("Add Roots", 1); + await _vfs.AddRoots(roots, token); // Step 1 - await InferMetas(token); + await InferMetas(token); // Step 2 - await _vfs.AddRoot(Settings.Downloads, token); + NextStep("Add Download Roots", 1); + await _vfs.AddRoot(Settings.Downloads, token); // Step 3 // Find all Downloads IndexedArchives = await Settings.Downloads.EnumerateFiles() @@ -68,8 +71,7 @@ namespace Wabbajack.Compiler IniData = f.WithExtension(Ext.Meta).LoadIniFile(), Meta = await f.WithExtension(Ext.Meta).ReadAllTextAsync() }).ToList(); - - + await IndexGameFileHashes(); IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList(); @@ -117,10 +119,15 @@ namespace Wabbajack.Compiler var stack = MakeStack(); - var results = await AllFiles.PMap(_parallelOptions, f => RunStack(stack, f)).ToList(); + NextStep("Running Compilation Stack", AllFiles.Count); + var results = await AllFiles.PMap(_parallelOptions, f => + { + UpdateProgress(1); + return RunStack(stack, f); + }).ToList(); + NextStep("Updating Extra files"); // Add the extra files that were generated by the stack - results = results.Concat(ExtraFiles).ToList(); var noMatch = results.OfType().ToArray(); @@ -176,9 +183,11 @@ namespace Wabbajack.Compiler private async Task RunValidation(ModList modList) { + NextStep("Validating Archives", modList.Archives.Length); var allowList = await _wjClient.LoadDownloadAllowList(); foreach (var archive in modList.Archives) { + UpdateProgress(1); if (!_dispatcher.IsAllowed(archive, allowList)) { _logger.LogCritical("Archive {name}, {primaryKeyString} is not allowed", archive.Name, diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index c20b57d3..04c4d591 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -3,8 +3,6 @@ using System.Collections.Generic; using System.Net.Http; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Wabbajack.Common; using Wabbajack.Compiler; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile;