Merge pull request #1610 from wabbajack-tools/github-validation-usage

Update the app to work with the new Github based action system

Wabbajack.Common/Consts.cs

@@ -93,7 +93,9 @@ namespace Wabbajack.Common
         public static string ModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/modlists.json";
         public static string UtilityModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/utility_modlists.json";
         public static string UnlistedModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/unlisted_modlists.json";
-        public static string ModlistSummaryURL = "https://build.wabbajack.org/lists/status.json";
+        public static string ModlistSummaryURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/reports/modListSummary.json";
+
+        public static string UpgradedFilesURL = "";
         public static string UserAgent
         {
             get

Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ArchiveStatus.cs

@@ -0,0 +1,14 @@
+using System.Text.Json.Serialization;
+
+namespace Wabbajack.Lib.Downloaders.DTOs.ModListValidation
+{
+    [JsonConverter(typeof(JsonStringEnumConverter))]
+    public enum ArchiveStatus
+    {
+        Valid,
+        InValid,
+        Updating,
+        Updated,
+        Mirrored
+    }
+}

Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ListStatus.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace Wabbajack.Lib.Downloaders.DTOs.ModListValidation
+{
+    [JsonConverter(typeof(JsonStringEnumConverter))]
+    public enum ListStatus : int
+    {
+        Available,
+        Failed,
+        ForcedDown
+    }
+}

Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ValidatedArchive.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Text.Json.Serialization;
+
+namespace Wabbajack.Lib.Downloaders.DTOs.ModListValidation
+{
+    public class ValidatedArchive
+    {
+        public ArchiveStatus Status { get; set; }
+        public Archive Original { get; set; } = new(new HTTPDownloader.State("http://foo"));
+
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+        public Archive? PatchedFrom { get; set; }
+        
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+        public Uri? PatchUrl { get; set; }
+    }
+}

Wabbajack.Lib/Downloaders/DTOs/ModListValidation/ValidatedModList.cs

@@ -0,0 +1,16 @@
+using System;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.Downloaders.DTOs.ModListValidation
+{
+    public class ValidatedModList
+    {
+        public string MachineURL { get; set; } = "";
+
+        public string Name { get; set; } = "";
+        public Version? Version { get; set; }
+        public Hash ModListHash { get; set; } = default;
+        public ValidatedArchive[] Archives { get; set; } = Array.Empty<ValidatedArchive>();
+        public ListStatus Status { get; set; }
+    }
+}

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -6,6 +6,7 @@ using System.Net.Http;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
+using Wabbajack.Lib.Downloaders.DTOs.ModListValidation;
 using Wabbajack.Lib.Downloaders.UrlDownloaders;
 
 namespace Wabbajack.Lib.Downloaders
@@ -110,54 +111,46 @@ namespace Wabbajack.Lib.Downloaders
                 if (downloadedHash == archive.Hash || archive.Hash == default) 
                     return DownloadResult.Success;
             }
-
             
-            if (await DownloadFromMirror(archive, destination))
+            var client = new Http.Client();
+            Utils.Log($"Loading for alternative to {archive.Hash}");
+            var replacementMeta = (await client.GetJsonAsync<ValidatedArchive[]>(Consts.UpgradedFilesURL))
+                .FirstOrDefault(a => a.Original.Hash == archive.Hash);
+
+            if (replacementMeta == null)
+            {
+                Utils.Log($"No alternative for {archive.Hash} could be found");
+                return DownloadResult.Failure;
+            }
+            
+            if (replacementMeta.Status == ArchiveStatus.Mirrored && await DownloadFromMirror(replacementMeta.PatchedFrom!, destination))
             {
                 await destination.FileHashCachedAsync();
                 return DownloadResult.Mirror;
             }
 
-            if (!(archive.State is IUpgradingState))
+            if (replacementMeta.Status != ArchiveStatus.Updated || !(archive.State is IUpgradingState))
             {
                 Utils.Log($"Download failed for {archive.Name} and no upgrade from this download source is possible");
                 return DownloadResult.Failure;
             }
 
-            Utils.Log($"Trying to find solution to broken download for {archive.Name}");
+            Utils.Log($"Downloading patch for {archive.Name}");
             
-            var result = await FindUpgrade(archive);
-            if (result == default )
-            {
-                result = await AbstractDownloadState.ServerFindUpgrade(archive);
-                if (result == default)
-                {
-                    Utils.Log(
-                        $"No solution for broken download {archive.Name} {archive.State.PrimaryKeyString} could be found");
-                    return DownloadResult.Failure;
-                }
-            }
 
-            Utils.Log($"Looking for patch for {archive.Name} ({(long)archive.Hash} {archive.Hash.ToHex()} -> {(long)result.Archive!.Hash} {result.Archive!.Hash.ToHex()})");
+            await using var tempFile = new TempFile();
-            var patchResult = await ClientAPI.GetModUpgrade(archive, result.Archive!);
+            await using var newFile = new TempFile();
 
-            Utils.Log($"Downloading patch for {archive.Name} from {patchResult}");
+            await Download(replacementMeta.PatchedFrom!, newFile.Path);
             
-            var tempFile = new TempFile();
-
-            if (WabbajackCDNDownloader.DomainRemaps.TryGetValue(patchResult.Host, out var remap))
             {
-                var builder = new UriBuilder(patchResult) {Host = remap};
+                using var response = await client.GetAsync(replacementMeta.PatchUrl!);
-                patchResult = builder.Uri;
+                await using var strm = await response.Content.ReadAsStreamAsync();
+                await tempFile.Path.WriteAllAsync(await response.Content.ReadAsStreamAsync());
             }
 
-            using var response = await (await ClientAPI.GetClient()).GetAsync(patchResult);
-
-            await tempFile.Path.WriteAllAsync(await response.Content.ReadAsStreamAsync());
-            response.Dispose();
-
             Utils.Log($"Applying patch to {archive.Name}");
-            await using(var src = await result.NewFile.Path.OpenShared())
+            await using(var src = await newFile.Path.OpenShared())
             await using (var final = await destination.Create())
             {
                 Utils.ApplyPatch(src, () => tempFile.Path.OpenShared().Result, final);

Wabbajack.Server/Controllers/NexusCache.cs

@@ -98,7 +98,6 @@ namespace Wabbajack.BuildServer.Controllers
             if (result == null)
             {
                 var api = await GetClient();
-                var permission = HTMLInterface.GetUploadPermissions(game, ModId);
                 try
                 {
                     result = await api.GetModFiles(game, ModId, false);
@@ -114,7 +113,6 @@ namespace Wabbajack.BuildServer.Controllers
                 var date = result.files.Select(f => f.uploaded_time).OrderByDescending(o => o).FirstOrDefault();
                 date = date == default ? DateTime.UtcNow : date;
                 await _sql.AddNexusModFiles(game, ModId, date, result);
-                await _sql.SetNexusPermission(game, ModId, await permission);
 
                 method = "NOT_CACHED";
                 Interlocked.Increment(ref ForwardCount);

Wabbajack.Server/Startup.cs

@@ -130,19 +130,19 @@ namespace Wabbajack.Server
             app.UseAuthentication();
             app.UseAuthorization();
             app.UseNexusPoll();
-            app.UseArchiveMaintainer();
+            //app.UseArchiveMaintainer();
-            app.UseModListDownloader();
+            //app.UseModListDownloader();
             app.UseResponseCompression();
             
-            app.UseService<NonNexusDownloadValidator>();
+            //app.UseService<NonNexusDownloadValidator>();
-            app.UseService<ListValidator>();
+            //app.UseService<ListValidator>();
-            app.UseService<ArchiveDownloader>();
+            //app.UseService<ArchiveDownloader>();
             app.UseService<DiscordWebHook>();
-            app.UseService<NexusKeyMaintainance>();
+            //app.UseService<NexusKeyMaintainance>();
-            app.UseService<PatchBuilder>();
+            //app.UseService<PatchBuilder>();
-            app.UseService<CDNMirrorList>();
+            //app.UseService<CDNMirrorList>();
-            app.UseService<NexusPermissionsUpdater>();
+            //app.UseService<NexusPermissionsUpdater>();
-            app.UseService<MirrorUploader>();
+            //app.UseService<MirrorUploader>();
             //app.UseService<MirrorQueueService>();
             app.UseService<Watchdog>();
             app.UseService<DiscordFrontend>();