diff --git a/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs b/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs index 3ce9b8f9..116c832e 100644 --- a/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs +++ b/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs @@ -194,7 +194,7 @@ namespace Wabbajack { State = CompilerState.Compiling; - var mo2Settings = new MO2CompilerSettings + var mo2Settings = new CompilerSettings { Game = BaseGame, ModListName = ModListName, @@ -210,19 +210,7 @@ namespace Wabbajack UseGamePaths = true }; - var compiler = new MO2Compiler(_serviceProvider.GetRequiredService>(), - _serviceProvider.GetRequiredService(), - _serviceProvider.GetRequiredService(), - _serviceProvider.GetRequiredService(), - _serviceProvider.GetRequiredService(), - mo2Settings, - _serviceProvider.GetRequiredService(), - _serviceProvider.GetRequiredService(), - _serviceProvider.GetRequiredService(), - _serviceProvider.GetRequiredService(), - _serviceProvider.GetRequiredService(), - _serviceProvider.GetRequiredService>(), - _serviceProvider.GetRequiredService()); + var compiler = MO2Compiler.Create(_serviceProvider, mo2Settings); await compiler.Begin(CancellationToken.None); diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index 10d0458b..507fbb3c 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -73,6 +73,7 @@ internal class Program services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); }).Build(); diff --git a/Wabbajack.CLI/Verbs/InstallCompileInstallVerify.cs b/Wabbajack.CLI/Verbs/InstallCompileInstallVerify.cs new file mode 100644 index 00000000..96b2b9f6 --- /dev/null +++ b/Wabbajack.CLI/Verbs/InstallCompileInstallVerify.cs @@ -0,0 +1,140 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.Compiler; +using Wabbajack.Downloaders; +using Wabbajack.Downloaders.GameFile; +using Wabbajack.DTOs; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Installer; +using Wabbajack.Networking.WabbajackClientApi; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; +using Wabbajack.VFS; + +namespace Wabbajack.CLI.Verbs; + +public class InstallCompileInstallVerify : IVerb +{ + private readonly ILogger _logger; + private readonly Client _wjClient; + private readonly DownloadDispatcher _dispatcher; + + private readonly DTOSerializer _dtos; + private readonly IServiceProvider _serviceProvider; + private readonly FileHashCache _cache; + private readonly GameLocator _gameLocator; + private readonly CompilerSettingsInferencer _inferencer; + + public InstallCompileInstallVerify(ILogger logger, Client wjClient, DownloadDispatcher dispatcher, DTOSerializer dtos, + FileHashCache cache, GameLocator gameLocator, IServiceProvider serviceProvider, CompilerSettingsInferencer inferencer) + { + _logger = logger; + _wjClient = wjClient; + _dispatcher = dispatcher; + _dtos = dtos; + _serviceProvider = serviceProvider; + _cache = cache; + _gameLocator = gameLocator; + _inferencer = inferencer; + } + + public Command MakeCommand() + { + var command = new Command("install-compile-install-verify"); + command.Add(new Option(new[] {"-m", "-machineUrls"}, "Machine url(s) to download")); + command.Add(new Option(new[] {"-d", "-downloads"}, "Downloads path")); + command.Add(new Option(new[] {"-o", "-outputs"}, "Outputs path")); + command.Description = "Installs a modlist, compiles it, installs it again, verifies it"; + command.Handler = CommandHandler.Create(Run); + return command; + } + + public async Task Run(AbsolutePath outputs, AbsolutePath downloads, IEnumerable machineUrls, CancellationToken token) + { + foreach (var machineUrl in machineUrls) + { + _logger.LogInformation("Installing {MachineUrl}", machineUrl); + var wabbajackPath = downloads.Combine(machineUrl.Replace("/", "_@@_")); + if (!await DownloadMachineUrl(machineUrl, wabbajackPath, token)) + throw new Exception("Can't download modlist"); + + var installPath = outputs.Combine(machineUrl); + + var modlist = await StandardInstaller.LoadFromFile(_dtos, wabbajackPath); + + var installer = StandardInstaller.Create(_serviceProvider, new InstallerConfiguration + { + Downloads = downloads, + Install = installPath, + ModList = modlist, + Game = modlist.GameType, + ModlistArchive = wabbajackPath, + GameFolder = _gameLocator.GameLocation(modlist.GameType) + }); + + var result = await installer.Begin(token); + if (!result) + { + _logger.LogInformation("Error installing {MachineUrl}", machineUrl); + return 1; + } + + _logger.LogInformation("Inferring settings"); + var inferedSettings = await _inferencer.InferFromRootPath(installPath); + if (inferedSettings == null) + { + _logger.LogInformation("Error inferencing settings for {MachineUrl}", machineUrl); + return 2; + } + + inferedSettings.UseGamePaths = true; + + + var compiler = MO2Compiler.Create(_serviceProvider, inferedSettings); + result = await compiler.Begin(token); + + return result ? 0 : 3; + + } + + return 0; + } + + private async Task DownloadMachineUrl(string machineUrl, AbsolutePath wabbajack, CancellationToken token) + { + _logger.LogInformation("Downloading {MachineUrl}", machineUrl); + + var lists = await _wjClient.LoadLists(); + var list = lists.FirstOrDefault(l => l.NamespacedName == machineUrl); + if (list == null) + { + _logger.LogInformation("Couldn't find list {MachineUrl}", machineUrl); + return false; + } + + if (wabbajack.FileExists() && await _cache.FileHashCachedAsync(wabbajack, token) == list.DownloadMetadata!.Hash) + { + _logger.LogInformation("File already exists, using cached file"); + return true; + } + + var state = _dispatcher.Parse(new Uri(list.Links.Download)); + + await _dispatcher.Download(new Archive + { + Name = wabbajack.FileName.ToString(), + Hash = list.DownloadMetadata!.Hash, + Size = list.DownloadMetadata.Size, + State = state! + }, wabbajack, token); + + return true; + } +} \ No newline at end of file diff --git a/Wabbajack.Common/Ext.cs b/Wabbajack.Common/Ext.cs index 29e02477..528ab80c 100644 --- a/Wabbajack.Common/Ext.cs +++ b/Wabbajack.Common/Ext.cs @@ -21,7 +21,6 @@ public static class Ext public static Extension Md = new(".md"); public static Extension MetaData = new(".metadata"); public static Extension CompilerSettings = new(".compiler_settings"); - public static Extension MO2CompilerSettings = new(".mo2_compiler_settings"); public static Extension Temp = new(".temp"); public static Extension ModlistMetadataExtension = new(".modlist_metadata"); public static Extension Txt = new(".txt"); diff --git a/Wabbajack.Compiler.Test/ModListHarness.cs b/Wabbajack.Compiler.Test/ModListHarness.cs index 9ebd59fc..c50f2ce1 100644 --- a/Wabbajack.Compiler.Test/ModListHarness.cs +++ b/Wabbajack.Compiler.Test/ModListHarness.cs @@ -66,7 +66,7 @@ public class ModListHarness return mod; } - public async Task CompileAndInstall(Action? configureSettings = null) + public async Task CompileAndInstall(Action? configureSettings = null) { var modlist = await Compile(configureSettings); await Install(); @@ -74,13 +74,13 @@ public class ModListHarness return modlist; } - public async Task Compile(Action? configureSettings = null) + public async Task Compile(Action? configureSettings = null) { configureSettings ??= x => { }; _source.Combine(Consts.MO2Profiles, _profileName).CreateDirectory(); using var scope = _serviceProvider.CreateScope(); - var settings = scope.ServiceProvider.GetService()!; + var settings = scope.ServiceProvider.GetService()!; settings.Downloads = _downloadPath; settings.Game = Game.SkyrimSpecialEdition; settings.Source = _source; diff --git a/Wabbajack.Compiler.Test/SanityTests.cs b/Wabbajack.Compiler.Test/SanityTests.cs index 027d7ef3..6c445160 100644 --- a/Wabbajack.Compiler.Test/SanityTests.cs +++ b/Wabbajack.Compiler.Test/SanityTests.cs @@ -56,7 +56,7 @@ public class CompilerSanityTests : IAsyncLifetime { } - private async Task CompileAndValidate(int expectedDirectives, Action? configureSettings = null) + private async Task CompileAndValidate(int expectedDirectives, Action? configureSettings = null) { _modlist = await _harness.Compile(configureSettings); Assert.NotNull(_modlist); diff --git a/Wabbajack.Compiler/ACompiler.cs b/Wabbajack.Compiler/ACompiler.cs index 4348349e..05dcf274 100644 --- a/Wabbajack.Compiler/ACompiler.cs +++ b/Wabbajack.Compiler/ACompiler.cs @@ -275,14 +275,14 @@ public abstract class ACompiler if (resolved == null) return null; a.State = resolved.State; - return null; + return a; } catch (Exception ex) { - _logger.LogWarning(ex.ToString(), "While resolving archive {archive}", a.Name); - return a; + _logger.LogWarning(ex, "While resolving archive {Archive}", a.Name); + return null; } - }).ToHashSet(f => f != null); + }).ToHashSet(f => f == null); if (remove.Count == 0) return; @@ -461,6 +461,7 @@ public abstract class ACompiler _logger.LogInformation("Patching {from} {to}", destFile, match.To); await using var srcStream = await sf.GetStream(); await using var destStream = await destsfn.GetStream(); + using var _ = await CompilerLimiter.Begin($"Patching {match.To}", 100, token); var patchSize = await _patchCache.CreatePatch(srcStream, vf.Hash, destStream, destvf.Hash); _logger.LogInformation("Patch size {patchSize} for {to}", patchSize, match.To); diff --git a/Wabbajack.Compiler/CompilerSettingsInferencer.cs b/Wabbajack.Compiler/CompilerSettingsInferencer.cs index 61508a9f..ecdcbcef 100644 --- a/Wabbajack.Compiler/CompilerSettingsInferencer.cs +++ b/Wabbajack.Compiler/CompilerSettingsInferencer.cs @@ -23,6 +23,14 @@ public class CompilerSettingsInferencer _logger = logger; } + + public async Task InferFromRootPath(AbsolutePath rootPath) + { + var mo2File = rootPath.Combine(Consts.MO2IniName).LoadIniFile(); + var profile = mo2File["General"]["selected_profile"]; + + return await InferModListFromLocation(rootPath.Combine(Consts.MO2Profiles, profile, Consts.ModListTxt)); + } public async Task InferModListFromLocation(AbsolutePath settingsFile) { diff --git a/Wabbajack.Compiler/MO2Compiler.cs b/Wabbajack.Compiler/MO2Compiler.cs index 3de780ed..4492744b 100644 --- a/Wabbajack.Compiler/MO2Compiler.cs +++ b/Wabbajack.Compiler/MO2Compiler.cs @@ -1,9 +1,11 @@ +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using IniParser.Model; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Compiler.CompilationSteps; @@ -25,7 +27,7 @@ public class MO2Compiler : ACompiler { public MO2Compiler(ILogger logger, FileExtractor.FileExtractor extractor, FileHashCache hashCache, Context vfs, - TemporaryFileManager manager, MO2CompilerSettings settings, ParallelOptions parallelOptions, + TemporaryFileManager manager, CompilerSettings settings, ParallelOptions parallelOptions, DownloadDispatcher dispatcher, Client wjClient, IGameLocator locator, DTOSerializer dtos, IResource compilerLimiter, IBinaryPatchCache patchCache) : @@ -35,7 +37,24 @@ public class MO2Compiler : ACompiler MaxSteps = 14; } - public MO2CompilerSettings Mo2Settings => (MO2CompilerSettings) Settings; + public static MO2Compiler Create(IServiceProvider provider, CompilerSettings mo2Settings) + { + return new MO2Compiler(provider.GetRequiredService>(), + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService(), + mo2Settings, + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService>(), + provider.GetRequiredService()); + } + + public CompilerSettings Mo2Settings => (CompilerSettings) Settings; public AbsolutePath MO2ModsFolder => Settings.Source.Combine(Consts.MO2ModFolderName); diff --git a/Wabbajack.Compiler/MO2CompilerSettings.cs b/Wabbajack.Compiler/MO2CompilerSettings.cs deleted file mode 100644 index 8efc2ccb..00000000 --- a/Wabbajack.Compiler/MO2CompilerSettings.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System; -using Wabbajack.Paths; - -namespace Wabbajack.Compiler; - -public class MO2CompilerSettings : CompilerSettings -{ - -} \ No newline at end of file diff --git a/Wabbajack.Compiler/PatchCache/BinaryPatchCache.cs b/Wabbajack.Compiler/PatchCache/BinaryPatchCache.cs index 59f61d7c..dd141137 100644 --- a/Wabbajack.Compiler/PatchCache/BinaryPatchCache.cs +++ b/Wabbajack.Compiler/PatchCache/BinaryPatchCache.cs @@ -6,6 +6,7 @@ using Wabbajack.Compiler.PatchCache; using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; using Wabbajack.Paths.IO; +using Wabbajack.RateLimiter; namespace Wabbajack.Compiler; @@ -38,7 +39,7 @@ public class BinaryPatchCache : IBinaryPatchCache cmd.ExecuteNonQuery(); } - public async Task CreatePatch(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash) + public async Task CreatePatch(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, IJob? job) { await using var rcmd = new SQLiteCommand(_conn); rcmd.CommandText = "SELECT PatchSize FROM PatchCache WHERE FromHash = @fromHash AND ToHash = @toHash"; @@ -57,7 +58,7 @@ public class BinaryPatchCache : IBinaryPatchCache await using var sigStream = new MemoryStream(); await using var patchStream = new MemoryStream(); - OctoDiff.Create(srcStream, destStream, sigStream, patchStream); + OctoDiff.Create(srcStream, destStream, sigStream, patchStream, job); cmd.Parameters.AddWithValue("@patchSize", patchStream.Length); cmd.Parameters.AddWithValue("@patch", patchStream.ToArray()); diff --git a/Wabbajack.Compiler/PatchCache/IBinaryPatchCache.cs b/Wabbajack.Compiler/PatchCache/IBinaryPatchCache.cs index 538764d3..3ecf0e34 100644 --- a/Wabbajack.Compiler/PatchCache/IBinaryPatchCache.cs +++ b/Wabbajack.Compiler/PatchCache/IBinaryPatchCache.cs @@ -2,12 +2,13 @@ using System.IO; using System.Threading.Tasks; using Wabbajack.Compiler.PatchCache; using Wabbajack.Hashing.xxHash64; +using Wabbajack.RateLimiter; namespace Wabbajack.Compiler; public interface IBinaryPatchCache { - public Task CreatePatch(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash); + public Task CreatePatch(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, IJob? job = null); public Task GetPatch(Hash hashA, Hash hashB); public Task GetData(CacheEntry entry); diff --git a/Wabbajack.Compiler/PatchCache/OctoDiff.cs b/Wabbajack.Compiler/PatchCache/OctoDiff.cs index 5c211e4e..1a4209dd 100644 --- a/Wabbajack.Compiler/PatchCache/OctoDiff.cs +++ b/Wabbajack.Compiler/PatchCache/OctoDiff.cs @@ -1,6 +1,8 @@ using System.IO; +using System.Threading; using Octodiff.Core; using Octodiff.Diagnostics; +using Wabbajack.RateLimiter; namespace Wabbajack.Compiler.PatchCache; @@ -33,11 +35,29 @@ public class OctoDiff sigStream.Position = 0; } - public static void Create(Stream oldData, Stream newData, Stream signature, Stream output) + public static void Create(Stream oldData, Stream newData, Stream signature, Stream output, IJob? job) { CreateSignature(oldData, signature); - var db = new DeltaBuilder {ProgressReporter = new NullProgressReporter()}; - db.BuildDelta(newData, new SignatureReader(signature, new NullProgressReporter()), + var db = new DeltaBuilder {ProgressReporter = new JobProgressReporter(job, 0)}; + db.BuildDelta(newData, new SignatureReader(signature, new JobProgressReporter(job, 100)), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output))); } + + private class JobProgressReporter : IProgressReporter + { + private readonly IJob _job; + private readonly int _offset; + + public JobProgressReporter(IJob job, int offset) + { + _offset = offset; + _job = job; + } + public void ReportProgress(string operation, long currentPosition, long total) + { + var percent = Percent.FactoryPutInRange(currentPosition, total); + var toReport = (long) (percent.Value * 100) - (_job.Current - _offset); + _job.ReportNoWait((int) toReport); + } + } } \ No newline at end of file diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index 9be8a2d5..914861f8 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -399,8 +399,12 @@ public abstract class AInstaller .Concat(_gameLocator.GameLocation(_configuration.Game).EnumerateFiles()) .ToList(); - var hashDict = allFiles.GroupBy(f => f.Size()).ToDictionary(g => g.Key); + _logger.LogInformation("Getting archive sizes"); + var hashDict = (await allFiles.PMapAll(_limiter, async x => (x, x.Size())).ToList()) + .GroupBy(f => f.Item2) + .ToDictionary(g => g.Key, g => g.Select(v => v.x)); + _logger.LogInformation("Linking archives to downloads"); var toHash = ModList.Archives.Where(a => hashDict.ContainsKey(a.Size)) .SelectMany(a => hashDict[a.Size]).ToList(); diff --git a/Wabbajack.RateLimiter/IJob.cs b/Wabbajack.RateLimiter/IJob.cs index a87ac195..7065e6a8 100644 --- a/Wabbajack.RateLimiter/IJob.cs +++ b/Wabbajack.RateLimiter/IJob.cs @@ -10,4 +10,5 @@ public interface IJob public long Current { get; } public string Description { get; } public ValueTask Report(int processedSize, CancellationToken token); + public void ReportNoWait(int processedSize); } \ No newline at end of file diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index 149de66f..e9dccba1 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -152,7 +152,7 @@ public static class ServiceExtensions // Installer/Compiler Configuration service.AddScoped(); service.AddScoped(); - service.AddScoped(); + service.AddScoped(); service.AddScoped(); service.AddSingleton();