diff --git a/Wabbajack.CLI/Verbs/DownloadAll.cs b/Wabbajack.CLI/Verbs/DownloadAll.cs index 461928c7..8a33d3f5 100644 --- a/Wabbajack.CLI/Verbs/DownloadAll.cs +++ b/Wabbajack.CLI/Verbs/DownloadAll.cs @@ -62,12 +62,15 @@ public class DownloadAll : IVerb }) .ToHashSet(); - var archives = (await (await _wjClient.LoadLists()) + var lists = await _wjClient.LoadLists(); + + var archives = (await (await _wjClient.GetListStatuses()) + .Where(l => !l.HasFailures) .PMapAll(_limiter, async m => { try { - return await StandardInstaller.Load(_dtos, _dispatcher, m, token); + return await StandardInstaller.Load(_dtos, _dispatcher, lists.First(l => l.NamespacedName == m.MachineURL), token); } catch (Exception ex) { @@ -94,8 +97,7 @@ public class DownloadAll : IVerb var outputFile = output.Combine(file.Name); if (outputFile.FileExists()) { - outputFile = output.Combine(outputFile.FileName.WithoutExtension() + "_" + file.Hash.ToHex() + - outputFile.WithExtension(outputFile.Extension)); + outputFile = output.Combine((outputFile.FileName.WithoutExtension() + "_" + file.Hash.ToHex()).ToRelativePath().WithExtension(outputFile.Extension)); } _logger.LogInformation("Downloading {File}", file.Name); diff --git a/Wabbajack.CLI/Verbs/ValidateLists.cs b/Wabbajack.CLI/Verbs/ValidateLists.cs index 4ba07fb4..80b8177e 100644 --- a/Wabbajack.CLI/Verbs/ValidateLists.cs +++ b/Wabbajack.CLI/Verbs/ValidateLists.cs @@ -217,10 +217,21 @@ public class ValidateLists : IVerb validatedList.Status = archives.Any(a => a.Status == ArchiveStatus.InValid) ? ListStatus.Failed : ListStatus.Available; - - var (smallImage, largeImage) = await ProcessModlistImage(reports, modList, token); - validatedList.SmallImage = new Uri($"https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/reports/{smallImage.ToString().Replace("\\", "/")}"); - validatedList.LargeImage = new Uri($"https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/reports/{largeImage.ToString().Replace("\\", "/")}"); + + try + { + var (smallImage, largeImage) = await ProcessModlistImage(reports, modList, token); + validatedList.SmallImage = + new Uri( + $"https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/reports/{smallImage.ToString().Replace("\\", "/")}"); + validatedList.LargeImage = + new Uri( + $"https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/reports/{largeImage.ToString().Replace("\\", "/")}"); + } + catch (Exception ex) + { + _logger.LogError(ex, "While processing modlist images for {MachineURL}", modList.NamespacedName); + } return validatedList; }).ToArray(); @@ -603,7 +614,7 @@ public class ValidateLists : IVerb case Nexus: return (ArchiveStatus.Valid, archive); case VectorPlexus: - return (ArchiveStatus.Valid, archive); + return (ArchiveStatus.InValid, archive); } if (archive.State is Http http && http.Url.Host.EndsWith("github.com")) diff --git a/Wabbajack.Common/AsyncParallelExtensions.cs b/Wabbajack.Common/AsyncParallelExtensions.cs index 10d96c27..418f60a7 100644 --- a/Wabbajack.Common/AsyncParallelExtensions.cs +++ b/Wabbajack.Common/AsyncParallelExtensions.cs @@ -18,8 +18,11 @@ public static class AsyncParallelExtensions public static async Task PDoAll(this IEnumerable coll, IResource limiter, Func mapFn) { - using var job = await limiter.Begin("", 0, CancellationToken.None); - var tasks = coll.Select(mapFn).ToList(); + var tasks = coll.Select(async x => + { + using var job = await limiter.Begin("", 0, CancellationToken.None); + await mapFn(x); + }).ToList(); await Task.WhenAll(tasks); } diff --git a/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs b/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs index 473727e9..021fe2b4 100644 --- a/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs +++ b/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs @@ -107,6 +107,7 @@ public class DownloadDispatcher { try { + a = await MaybeProxy(a, token); var downloader = Downloader(a); using var job = await _limiter.Begin($"Verifying {a.State.PrimaryKeyString}", -1, token); return await downloader.Verify(a, job, token); diff --git a/Wabbajack.Downloaders.WabbajackCDN/WabbajackCDNDownloader.cs b/Wabbajack.Downloaders.WabbajackCDN/WabbajackCDNDownloader.cs index e51250fe..66fa10fd 100644 --- a/Wabbajack.Downloaders.WabbajackCDN/WabbajackCDNDownloader.cs +++ b/Wabbajack.Downloaders.WabbajackCDN/WabbajackCDNDownloader.cs @@ -160,6 +160,9 @@ public class WabbajackCDNDownloader : ADownloader, IUrlDownloader, { var state = archive.State as WabbajackCDN; var definition = await GetDefinition(state!, token); + if (definition == null) + throw new Exception("Could not get CDN definition"); + return new ChunkedSeekableDownloader(state!, definition!, this); } diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index e2c416a0..e30e3dcb 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -446,6 +446,8 @@ public abstract class AInstaller protected async Task OptimizeModlist(CancellationToken token) { _logger.LogInformation("Optimizing ModList directives"); + UnoptimizedArchives = ModList.Archives; + UnoptimizedDirectives = ModList.Directives; var indexed = ModList.Directives.ToDictionary(d => d.To); @@ -564,9 +566,7 @@ public abstract class AInstaller .GroupBy(d => d.ArchiveHashPath.Hash) .Select(d => d.Key) .ToHashSet(); - - UnoptimizedArchives = ModList.Archives; - UnoptimizedDirectives = ModList.Directives; + ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToArray(); ModList.Directives = indexed.Values.ToArray(); }