From 81c8c1c3a525c92713204df7bcd16747fbbaf293 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 13 Sep 2021 21:22:07 -0600 Subject: [PATCH] Update the app to work with the new Github based action system --- Wabbajack.Common/Consts.cs | 4 +- .../DTOs/ModListValidation/ArchiveStatus.cs | 14 +++++ .../DTOs/ModListValidation/ListStatus.cs | 12 +++++ .../ModListValidation/ValidatedArchive.cs | 17 +++++++ .../ModListValidation/ValidatedModList.cs | 16 ++++++ .../Downloaders/DownloadDispatcher.cs | 51 ++++++++----------- Wabbajack.Server/Controllers/NexusCache.cs | 2 - Wabbajack.Server/Startup.cs | 20 ++++---- 8 files changed, 94 insertions(+), 42 deletions(-) create mode 100644 Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ArchiveStatus.cs create mode 100644 Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ListStatus.cs create mode 100644 Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ValidatedArchive.cs create mode 100644 Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ValidatedModList.cs diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 1c2caf38..0a18737b 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -93,7 +93,9 @@ namespace Wabbajack.Common public static string ModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/modlists.json"; public static string UtilityModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/utility_modlists.json"; public static string UnlistedModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/unlisted_modlists.json"; - public static string ModlistSummaryURL = "https://build.wabbajack.org/lists/status.json"; + public static string ModlistSummaryURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/reports/modListSummary.json"; + + public static string UpgradedFilesURL = ""; public static string UserAgent { get diff --git a/Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ArchiveStatus.cs b/Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ArchiveStatus.cs new file mode 100644 index 00000000..d7d43cda --- /dev/null +++ b/Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ArchiveStatus.cs @@ -0,0 +1,14 @@ +using System.Text.Json.Serialization; + +namespace Wabbajack.Lib.Downloaders.DTOs.ModListValidation +{ + [JsonConverter(typeof(JsonStringEnumConverter))] + public enum ArchiveStatus + { + Valid, + InValid, + Updating, + Updated, + Mirrored + } +} diff --git a/Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ListStatus.cs b/Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ListStatus.cs new file mode 100644 index 00000000..7e843135 --- /dev/null +++ b/Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ListStatus.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace Wabbajack.Lib.Downloaders.DTOs.ModListValidation +{ + [JsonConverter(typeof(JsonStringEnumConverter))] + public enum ListStatus : int + { + Available, + Failed, + ForcedDown + } +} diff --git a/Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ValidatedArchive.cs b/Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ValidatedArchive.cs new file mode 100644 index 00000000..8fd4753b --- /dev/null +++ b/Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ValidatedArchive.cs @@ -0,0 +1,17 @@ +using System; +using System.Text.Json.Serialization; + +namespace Wabbajack.Lib.Downloaders.DTOs.ModListValidation +{ + public class ValidatedArchive + { + public ArchiveStatus Status { get; set; } + public Archive Original { get; set; } = new(new HTTPDownloader.State("http://foo")); + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Archive? PatchedFrom { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Uri? PatchUrl { get; set; } + } +} diff --git a/Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ValidatedModList.cs b/Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ValidatedModList.cs new file mode 100644 index 00000000..6b02b9ba --- /dev/null +++ b/Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ValidatedModList.cs @@ -0,0 +1,16 @@ +using System; +using Wabbajack.Common; + +namespace Wabbajack.Lib.Downloaders.DTOs.ModListValidation +{ + public class ValidatedModList + { + public string MachineURL { get; set; } = ""; + + public string Name { get; set; } = ""; + public Version? Version { get; set; } + public Hash ModListHash { get; set; } = default; + public ValidatedArchive[] Archives { get; set; } = Array.Empty(); + public ListStatus Status { get; set; } + } +} diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index 1caa015f..a0f38a13 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -6,6 +6,7 @@ using System.Net.Http; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; +using Wabbajack.Lib.Downloaders.DTOs.ModListValidation; using Wabbajack.Lib.Downloaders.UrlDownloaders; namespace Wabbajack.Lib.Downloaders @@ -110,54 +111,46 @@ namespace Wabbajack.Lib.Downloaders if (downloadedHash == archive.Hash || archive.Hash == default) return DownloadResult.Success; } - - if (await DownloadFromMirror(archive, destination)) + var client = new Http.Client(); + Utils.Log($"Loading for alternative to {archive.Hash}"); + var replacementMeta = (await client.GetJsonAsync(Consts.UpgradedFilesURL)) + .FirstOrDefault(a => a.Original.Hash == archive.Hash); + + if (replacementMeta == null) + { + Utils.Log($"No alternative for {archive.Hash} could be found"); + return DownloadResult.Failure; + } + + if (replacementMeta.Status == ArchiveStatus.Mirrored && await DownloadFromMirror(replacementMeta.PatchedFrom!, destination)) { await destination.FileHashCachedAsync(); return DownloadResult.Mirror; } - if (!(archive.State is IUpgradingState)) + if (replacementMeta.Status != ArchiveStatus.Updated || !(archive.State is IUpgradingState)) { Utils.Log($"Download failed for {archive.Name} and no upgrade from this download source is possible"); return DownloadResult.Failure; } - Utils.Log($"Trying to find solution to broken download for {archive.Name}"); + Utils.Log($"Downloading patch for {archive.Name}"); - var result = await FindUpgrade(archive); - if (result == default ) - { - result = await AbstractDownloadState.ServerFindUpgrade(archive); - if (result == default) - { - Utils.Log( - $"No solution for broken download {archive.Name} {archive.State.PrimaryKeyString} could be found"); - return DownloadResult.Failure; - } - } - Utils.Log($"Looking for patch for {archive.Name} ({(long)archive.Hash} {archive.Hash.ToHex()} -> {(long)result.Archive!.Hash} {result.Archive!.Hash.ToHex()})"); - var patchResult = await ClientAPI.GetModUpgrade(archive, result.Archive!); + await using var tempFile = new TempFile(); + await using var newFile = new TempFile(); - Utils.Log($"Downloading patch for {archive.Name} from {patchResult}"); + await Download(replacementMeta.PatchedFrom!, newFile.Path); - var tempFile = new TempFile(); - - if (WabbajackCDNDownloader.DomainRemaps.TryGetValue(patchResult.Host, out var remap)) { - var builder = new UriBuilder(patchResult) {Host = remap}; - patchResult = builder.Uri; + using var response = await client.GetAsync(replacementMeta.PatchUrl!); + await using var strm = await response.Content.ReadAsStreamAsync(); + await tempFile.Path.WriteAllAsync(await response.Content.ReadAsStreamAsync()); } - using var response = await (await ClientAPI.GetClient()).GetAsync(patchResult); - - await tempFile.Path.WriteAllAsync(await response.Content.ReadAsStreamAsync()); - response.Dispose(); - Utils.Log($"Applying patch to {archive.Name}"); - await using(var src = await result.NewFile.Path.OpenShared()) + await using(var src = await newFile.Path.OpenShared()) await using (var final = await destination.Create()) { Utils.ApplyPatch(src, () => tempFile.Path.OpenShared().Result, final); diff --git a/Wabbajack.Server/Controllers/NexusCache.cs b/Wabbajack.Server/Controllers/NexusCache.cs index d493fc9b..9b2200e6 100644 --- a/Wabbajack.Server/Controllers/NexusCache.cs +++ b/Wabbajack.Server/Controllers/NexusCache.cs @@ -98,7 +98,6 @@ namespace Wabbajack.BuildServer.Controllers if (result == null) { var api = await GetClient(); - var permission = HTMLInterface.GetUploadPermissions(game, ModId); try { result = await api.GetModFiles(game, ModId, false); @@ -114,7 +113,6 @@ namespace Wabbajack.BuildServer.Controllers var date = result.files.Select(f => f.uploaded_time).OrderByDescending(o => o).FirstOrDefault(); date = date == default ? DateTime.UtcNow : date; await _sql.AddNexusModFiles(game, ModId, date, result); - await _sql.SetNexusPermission(game, ModId, await permission); method = "NOT_CACHED"; Interlocked.Increment(ref ForwardCount); diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 0134e570..c53b73fa 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -130,19 +130,19 @@ namespace Wabbajack.Server app.UseAuthentication(); app.UseAuthorization(); app.UseNexusPoll(); - app.UseArchiveMaintainer(); - app.UseModListDownloader(); + //app.UseArchiveMaintainer(); + //app.UseModListDownloader(); app.UseResponseCompression(); - app.UseService(); - app.UseService(); - app.UseService(); + //app.UseService(); + //app.UseService(); + //app.UseService(); app.UseService(); - app.UseService(); - app.UseService(); - app.UseService(); - app.UseService(); - app.UseService(); + //app.UseService(); + //app.UseService(); + //app.UseService(); + //app.UseService(); + //app.UseService(); //app.UseService(); app.UseService(); app.UseService();