diff --git a/Wabbajack.Compiler/ACompiler.cs b/Wabbajack.Compiler/ACompiler.cs index c5d642a1..73568fec 100644 --- a/Wabbajack.Compiler/ACompiler.cs +++ b/Wabbajack.Compiler/ACompiler.cs @@ -547,7 +547,7 @@ namespace Wabbajack.Compiler if (!await _dispatcher.Verify(result, token.Token)) { _logger.LogWarning( - "Unable to resolve link for {archive}. If this is hosted on the Nexus the file may have been removed.", archive); + "Unable to resolve link for {Archive}. If this is hosted on the Nexus the file may have been removed.", archive.State!.PrimaryKeyString); } result.Meta = "[General]\n" + string.Join("\n", _dispatcher.MetaIni(result)); diff --git a/Wabbajack.Compiler/CompilationSteps/IgnoreDisabledMods.cs b/Wabbajack.Compiler/CompilationSteps/IgnoreDisabledMods.cs index 958d71bf..604d2373 100644 --- a/Wabbajack.Compiler/CompilationSteps/IgnoreDisabledMods.cs +++ b/Wabbajack.Compiler/CompilationSteps/IgnoreDisabledMods.cs @@ -16,16 +16,17 @@ namespace Wabbajack.Compiler.CompilationSteps public IgnoreDisabledMods(ACompiler compiler) : base(compiler) { _mo2Compiler = (MO2Compiler) compiler; - var alwaysEnabled = _mo2Compiler.ModInis.Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_ENABLE)).Select(f => f.Key).Distinct(); - var alwaysDisabled = _mo2Compiler.ModInis - .Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_DISABLE)).Select(f => f.Key).Distinct(); + //var alwaysEnabled = _mo2Compiler.ModInis.Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_ENABLE)).Select(f => f.Key).Distinct(); + // TODO: Re-enable this + //var alwaysDisabled = _mo2Compiler.ModInis + // .Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_DISABLE)).Select(f => f.Key).Distinct(); _allEnabledMods = _mo2Compiler._settings.SelectedProfiles .SelectMany(p => _mo2Compiler._settings.Source.Combine("profiles", p, "modlist.txt").ReadAllLines()) .Where(line => line.StartsWith("+") || line.EndsWith("_separator")) .Select(line => line[1..].ToRelativePath().RelativeTo(_mo2Compiler.MO2ModsFolder)) - .Concat(alwaysEnabled) - .Except(alwaysDisabled) + //.Concat(alwaysEnabled) + //.Except(alwaysDisabled) .ToList(); } diff --git a/Wabbajack.Compiler/Consts.cs b/Wabbajack.Compiler/Consts.cs index a0149d15..abb18133 100644 --- a/Wabbajack.Compiler/Consts.cs +++ b/Wabbajack.Compiler/Consts.cs @@ -35,7 +35,7 @@ namespace Wabbajack.Compiler public static readonly HashSet SupportedBSAs = new[] {".bsa", ".ba2"} .Select(s => new Extension(s)).ToHashSet(); - public static HashSet ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml"}.Select(s => new Extension(s)).ToHashSet(); + public static HashSet ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml", ".yaml"}.Select(s => new Extension(s)).ToHashSet(); public static HashSet ESPFileExtensions = new []{ ".esp", ".esm", ".esl"}.Select(s => new Extension(s)).ToHashSet(); public static HashSet AssetFileExtensions = new[] {".dds", ".tga", ".nif", ".psc", ".pex"}.Select(s => new Extension(s)).ToHashSet(); diff --git a/Wabbajack.Compiler/MO2Compiler.cs b/Wabbajack.Compiler/MO2Compiler.cs index e5a7a5ae..a86be9d1 100644 --- a/Wabbajack.Compiler/MO2Compiler.cs +++ b/Wabbajack.Compiler/MO2Compiler.cs @@ -275,6 +275,10 @@ namespace Wabbajack.Compiler new IncludeRegex(this, ".*\\.txt"), new IgnorePathContains(this,@"\Edit Scripts\Export\"), new IgnoreExtension(this, new Extension(".CACHE")), + + // Misc + new IncludeRegex(this, "modlist-image\\.png"), + new DropAll(this) }; diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index a8406456..eda6e3d4 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -256,15 +256,18 @@ namespace Wabbajack.Installer await Task.WhenAll(dispatchers.Select(d => d.Prepare())); + _logger.LogInformation("Downloading validation data"); var validationData = await _wjClient.LoadDownloadAllowList(); + _logger.LogInformation("Validating Archives"); foreach (var archive in missing.Where(archive => !_downloadDispatcher.Downloader(archive).IsAllowed(validationData, archive.State))) { _logger.LogCritical("File {primaryKeyString} failed validation", archive.State.PrimaryKeyString); return; } - + + _logger.LogInformation("Downloading missing archives"); await DownloadMissingArchives(missing, token); } diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index 998d815a..f12a87da 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -59,6 +59,7 @@ namespace Wabbajack.Services.OSIntegrated service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount}); service.AddAllSingleton>(s => new Resource(12)); service.AddAllSingleton>(s => new Resource(12)); + service.AddAllSingleton>(s => new Resource(12)); service.AddAllSingleton>(s => new Resource(12)); diff --git a/Wabbajack.VFS/Context.cs b/Wabbajack.VFS/Context.cs index 818d4845..5f56d273 100644 --- a/Wabbajack.VFS/Context.cs +++ b/Wabbajack.VFS/Context.cs @@ -10,6 +10,7 @@ using Wabbajack.FileExtractor.ExtractedFiles; using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; using Wabbajack.Paths.IO; +using Wabbajack.RateLimiter; namespace Wabbajack.VFS { @@ -25,10 +26,12 @@ namespace Wabbajack.VFS public readonly FileHashCache HashCache; public readonly ILogger Logger; public readonly VFSCache VfsCache; + public readonly IResource Limiter; public Context(ILogger logger, ParallelOptions parallelOptions, TemporaryFileManager manager, VFSCache vfsCache, - FileHashCache hashCache, FileExtractor.FileExtractor extractor) + FileHashCache hashCache, IResource limiter, FileExtractor.FileExtractor extractor) { + Limiter = limiter; Logger = logger; _manager = manager; Extractor = extractor; @@ -49,7 +52,7 @@ namespace Wabbajack.VFS var filesToIndex = root.EnumerateFiles().Distinct().ToList(); var allFiles = await filesToIndex - .PMap(_parallelOptions, async f => + .PMapAll(async f => { if (byPath.TryGetValue(f, out var found)) if (found.LastModified == f.LastModifiedUtc().AsUnixTime() && found.Size == f.Size()) diff --git a/Wabbajack.VFS/FileHashCache.cs b/Wabbajack.VFS/FileHashCache.cs index caa0af49..11edff19 100644 --- a/Wabbajack.VFS/FileHashCache.cs +++ b/Wabbajack.VFS/FileHashCache.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; using Wabbajack.Paths.IO; +using Wabbajack.RateLimiter; namespace Wabbajack.VFS { @@ -113,13 +114,13 @@ namespace Wabbajack.VFS WriteHashCache(file, hash); } - public async Task FileHashCachedAsync(AbsolutePath file, CancellationToken token) + public async Task FileHashCachedAsync(AbsolutePath file, CancellationToken token, IJob? job = null) { if (TryGetHashCache(file, out var foundHash)) return foundHash; await using var fs = file.Open(FileMode.Open, FileAccess.Read, FileShare.Read); - var hash = await fs.HashingCopy(Stream.Null, token); + var hash = await fs.HashingCopy(Stream.Null, token, job); if (hash != default) WriteHashCache(file, hash); return hash; diff --git a/Wabbajack.VFS/VirtualFile.cs b/Wabbajack.VFS/VirtualFile.cs index d6ff3b9e..fa065445 100644 --- a/Wabbajack.VFS/VirtualFile.cs +++ b/Wabbajack.VFS/VirtualFile.cs @@ -162,14 +162,20 @@ namespace Wabbajack.VFS IPath relPath, CancellationToken token, int depth = 0) { Hash hash; - if (extractedFile is NativeFileStreamFactory) + using (var job = await context.Limiter.Begin("Hash file", 0, token)) { - hash = await context.HashCache.FileHashCachedAsync((AbsolutePath)extractedFile.Name, token); - } - else - { - await using var hstream = await extractedFile.GetStream(); - hash = await hstream.HashingCopy(Stream.Null, token); + if (extractedFile is NativeFileStreamFactory) + { + var absPath = (AbsolutePath)extractedFile.Name; + job.Size = absPath.Size(); + hash = await context.HashCache.FileHashCachedAsync(absPath, token, job); + } + else + { + await using var hstream = await extractedFile.GetStream(); + job.Size = hstream.Length; + hash = await hstream.HashingCopy(Stream.Null, token, job); + } } if (context.VfsCache.TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself)) @@ -195,7 +201,9 @@ namespace Wabbajack.VFS if (TextureExtensions.Contains(relPath.FileName.Extension) && await DDSSig.MatchesAsync(stream) != null) try { + using var job = await context.Limiter.Begin("Perceptual hash", self.Size, token); self.ImageState = await ImageLoader.Load(stream); + await job.Report((int)self.Size, token); stream.Position = 0; } catch (Exception)