Support self-healing lists, and a lot of server fixes

Wabbajack.BuildServer/Controllers/ListValidation.cs

@@ -11,6 +11,7 @@ using MongoDB.Driver;
 using MongoDB.Driver.Linq;
 using Nettle;
 using Wabbajack.BuildServer.Models;
+using Wabbajack.Common;
 using Wabbajack.Lib.ModListRegistry;
 
 namespace Wabbajack.BuildServer.Controllers
@@ -39,7 +40,7 @@ namespace Wabbajack.BuildServer.Controllers
     <description>These are mods that are broken and need updating</description>
     {{ each $.failed }}
     <item>
-       <title>{{$.Archive.Name}}</title>
+       <title>{{$.Archive.Name}} {{$.Archive.Hash}} {{$.Archive.State.PrimaryKeyString}}</title>
        <link>{{$.Archive.Name}}</link>
     </item>
     {{/each}}
@@ -104,6 +105,21 @@ namespace Wabbajack.BuildServer.Controllers
                 Content = response
             };
         }
+        
+        [HttpGet]
+        [Route("status/{Name}.json")]
+        public async Task<ContentResult> HandleGetListJson(string Name)
+        {
+
+            var lst = (await ModListStatus.ByName(Db, Name)).DetailedStatus;
+            lst.Archives.Do(a => a.Archive.Meta = null);
+            return new ContentResult
+            {
+                ContentType = "application/json",
+                StatusCode = (int) HttpStatusCode.OK,
+                Content = lst.ToJSON()
+            };
+        }
 
 
     }

Wabbajack.BuildServer/Controllers/ModlistUpdater.cs

@@ -0,0 +1,135 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Model.Models;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.BuildServer.Models.Jobs;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.NexusApi;
+using AlphaFile = Alphaleonis.Win32.Filesystem.File;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [ApiController]
+    [Route("/listupdater")]
+    public class ModlistUpdater : AControllerBase<ModlistUpdater>
+    {
+        private AppSettings _settings;
+        private SqlService _sql;
+
+        public ModlistUpdater(ILogger<ModlistUpdater> logger, DBContext db, SqlService sql, AppSettings settings) : base(logger, db)
+        {
+            _settings = settings;
+            _sql = sql;
+        }
+
+        [HttpGet]
+        [Route("/alternative/{xxHash}")]
+        public async Task<IActionResult> GetAlternative(string xxHash)
+        {
+            var startingHash = xxHash.FromHex().ToBase64();
+            Utils.Log($"Alternative requested for {startingHash}");
+
+            var state = await Db.DownloadStates.AsQueryable()
+                .Where(s => s.Hash == startingHash)
+                .Where(s => s.State is NexusDownloader.State)
+                .OrderByDescending(s => s.LastValidationTime).FirstOrDefaultAsync();
+
+            if (state == null)
+                return NotFound("Original state not found");
+
+            var nexusState = state.State as NexusDownloader.State;
+
+            Utils.Log($"Found original, looking for alternatives to {startingHash}");
+            var newArchive = await FindAlternatives(nexusState, startingHash);
+            if (newArchive == null)
+            {
+                return NotFound("No alternative available");
+            }
+
+            Utils.Log($"Found {newArchive.State.PrimaryKeyString} as an alternative to {startingHash}");
+            if (newArchive.Hash == null)
+            {
+                Db.Jobs.InsertOne(new Job
+                {
+                    Payload = new IndexJob
+                    {
+                        Archive = newArchive
+                    },
+                    OnSuccess = new Job
+                    {
+                        Payload = new PatchArchive
+                        {
+                            Src = startingHash,
+                            DestPK = newArchive.State.PrimaryKeyString
+                        }
+                    }
+                });
+                return Accepted("Enqueued for Processing");
+            }
+
+            if (!AlphaFile.Exists(PatchArchive.CdnPath(startingHash, newArchive.Hash)))
+            {
+                Db.Jobs.InsertOne(new Job
+                {
+                    Priority = Job.JobPriority.High,
+                    Payload = new PatchArchive
+                    {
+                        Src = startingHash,
+                        DestPK = newArchive.State.PrimaryKeyString
+                    }
+                });
+            }
+            return Ok(newArchive.ToJSON());
+        }
+
+        private async Task<Archive> FindAlternatives(NexusDownloader.State state, string srcHash)
+        {
+            var origSize = AlphaFile.GetSize(_settings.PathForArchive(srcHash));
+            var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
+            var allMods = await api.GetModFiles(GameRegistry.GetByFuzzyName(state.GameName).Game,
+                int.Parse(state.ModID));
+            var archive = allMods.files.Where(m => !string.IsNullOrEmpty(m.category_name))
+                .OrderBy(s => Math.Abs((long)s.size - origSize))
+                .Select(s => new Archive {
+                    Name = s.file_name,
+                    Size = (long)s.size,
+                    State = new NexusDownloader.State 
+                    {
+                    GameName = state.GameName, 
+                    ModID = state.ModID, 
+                    FileID = s.file_id.ToString()
+                }}).FirstOrDefault();
+
+            if (archive == null)
+            {
+                Utils.Log($"No alternative for {srcHash}");
+                return null;
+            }
+            
+            Utils.Log($"Found alternative for {srcHash}");
+            
+            var indexed = await Db.DownloadStates.AsQueryable().Where(s => s.Key == archive.State.PrimaryKeyString).FirstOrDefaultAsync();
+
+            if (indexed == null)
+            {
+                return archive;
+            }
+
+            Utils.Log($"Pre-Indexed alternative {indexed.Hash} found for {srcHash}");
+            archive.Hash = indexed.Hash;
+            return archive;
+
+        }
+
+
+    }
+}

Wabbajack.BuildServer/Extensions.cs

@@ -1,13 +1,18 @@
 using System;
+using System.Collections.Concurrent;
 using System.IO;
 using System.Linq;
 using System.Linq.Expressions;
 using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Builder;
 using MongoDB.Driver;
 using MongoDB.Driver.Linq;
+using Wabbajack.Common;
+using Directory =Alphaleonis.Win32.Filesystem.Directory;
 using File = Alphaleonis.Win32.Filesystem.File;
+using Path = Alphaleonis.Win32.Filesystem.Path;
 
 namespace Wabbajack.BuildServer
 {
@@ -41,5 +46,25 @@ namespace Wabbajack.BuildServer
         {
             return authenticationBuilder.AddScheme<ApiKeyAuthenticationOptions, ApiKeyAuthenticationHandler>(ApiKeyAuthenticationOptions.DefaultScheme, options);
         }
+        
+        private static ConcurrentDictionary<string, string> PathForArchiveHash = new ConcurrentDictionary<string, string>();
+        public static string PathForArchive(this AppSettings settings, string hash)
+        {
+            if (PathForArchiveHash.TryGetValue(hash, out string result))
+                return result;
+            
+            var hexHash = hash.FromBase64().ToHex();
+
+            var ends = "_" + hexHash + "_";
+            var file = Directory.EnumerateFiles(settings.ArchiveDir, DirectoryEnumerationOptions.Files,
+                new DirectoryEnumerationFilters
+                {
+                    InclusionFilter = f => Path.GetFileNameWithoutExtension(f.FileName).EndsWith(ends)
+                }).FirstOrDefault();
+            
+            if (file != null) 
+                PathForArchiveHash.TryAdd(hash, file);
+            return file;
+        }
     }
 }

Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs

@@ -21,7 +21,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue
             typeof(EnqueueRecentFiles),
             typeof(UploadToCDN),
             typeof(IndexDynDOLOD),
-            typeof(ReindexArchives)
+            typeof(ReindexArchives),
+            typeof(PatchArchive)
         };
         public static Dictionary<Type, string> TypeToName { get; set; }
         public static Dictionary<string, Type> NameToType { get; set; }

Wabbajack.BuildServer/Models/JobQueue/Job.cs

@@ -28,6 +28,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue
         public JobResult Result { get; set; }
         public bool RequiresNexus { get; set; } = true;
         public AJobPayload Payload { get; set; }
+        
+        public Job OnSuccess { get; set; }
 
         public static async Task<String> Enqueue(DBContext db, Job job)
         {
@@ -52,6 +54,11 @@ namespace Wabbajack.BuildServer.Models.JobQueue
 
         public static async Task<Job> Finish(DBContext db, Job job, JobResult jobResult)
         {
+            if (jobResult.ResultType == JobResultType.Success && job.OnSuccess != null)
+            {
+                await db.Jobs.InsertOneAsync(job.OnSuccess);
+            }
+
             var filter = new BsonDocument
             {
                 {"_id", job.Id},

Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs

@@ -2,12 +2,15 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
 using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.BuildServer.Models.JobQueue;
 using Wabbajack.Common;
 using Wabbajack.Lib;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.ModListRegistry;
+using Wabbajack.Lib.NexusApi;
 using Wabbajack.Lib.Validation;
 using File = Alphaleonis.Win32.Filesystem.File;
 
@@ -43,10 +46,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
             return JobResult.Success();
         }
         
-         private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
+         private async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
         {
-            var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
-            
             var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + Consts.ModListExtension);
 
             if (list.NeedsDownload(modlist_path))
@@ -76,17 +77,9 @@ namespace Wabbajack.BuildServer.Models.Jobs
             var validated = (await installer.Archives
                     .PMap(queue, async archive =>
                     {
-                        bool is_failed;
+                        var isValid = await IsValid(db, whitelists, archive);
-                        try
-                        {
-                            is_failed = !(await archive.State.Verify(archive)) || !archive.State.IsWhitelisted(whitelists.ServerWhitelist);
-                        }
-                        catch (Exception)
-                        {
-                            is_failed = false;
-                        }
 
-                        return new DetailedStatusItem {IsFailing = is_failed, Archive = archive};
+                        return new DetailedStatusItem {IsFailing = !isValid, Archive = archive};
                     }))
                 .ToList();
 
@@ -105,6 +98,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
                 Summary = new ModlistSummary
                 {
                     Name = status.Name,
+                    MachineURL = list.Links?.MachineURL ?? status.Name,
                     Checked = status.Checked,
                     Failed = status.Archives.Count(a => a.IsFailing),
                     Passed = status.Archives.Count(a => !a.IsFailing),
@@ -119,5 +113,79 @@ namespace Wabbajack.BuildServer.Models.Jobs
                 $"Done updating {dto.Summary.Name}");
 
         }
+
+         private async Task<bool> IsValid(DBContext db, ValidateModlist whitelists, Archive archive)
+         {
+             try
+             {
+                 if (!archive.State.IsWhitelisted(whitelists.ServerWhitelist)) return false;
+
+                 try
+                 {
+                     if (archive.State is NexusDownloader.State state)
+                     {
+                         if (await ValidateNexusFast(db, state)) return true;
+
+                     }
+                     else if (archive.State is HTTPDownloader.State hstate &&
+                              hstate.Url.StartsWith("https://wabbajack"))
+                     {
+                         return true;
+                     }
+                     else
+                     {
+                         if (await archive.State.Verify(archive)) return true;
+                     }
+                 }
+                 catch (Exception)
+                 {
+                     // ignored
+                 }
+
+                 var result = await ClientAPI.GetModUpgrade(archive.Hash);
+                 if (result != null) return true;
+
+             }
+             catch (Exception)
+             {
+                 return true;
+             }
+
+             return false;
+         }
+
+         private async Task<bool> ValidateNexusFast(DBContext db, NexusDownloader.State state)
+         {
+             try
+             {
+                 var gameMeta = GameRegistry.GetByFuzzyName(state.GameName);
+                 if (gameMeta == null)
+                     return false;
+
+                 var game = gameMeta.Game;
+                 if (!int.TryParse(state.ModID, out var modID))
+                     return false;
+
+                 var modFiles = (await db.NexusModFiles.AsQueryable().Where(g => g.Game == gameMeta.NexusName && g.ModId == state.ModID).FirstOrDefaultAsync())?.Data;
+
+                 if (modFiles == null)
+                 {
+                     Utils.Log($"No Cache for {state.PrimaryKeyString} falling back to HTTP");
+                     var nexusApi = await NexusApiClient.Get();
+                     modFiles = await nexusApi.GetModFiles(game, modID);
+                 }
+
+                 if (!ulong.TryParse(state.FileID, out var fileID))
+                     return false;
+
+                 var found = modFiles.files
+                     .FirstOrDefault(file => file.file_id == fileID && file.category_name != null);
+                 return found != null;
+             }
+             catch (Exception ex)
+             {
+                 return false;
+             }
+         }
     }
 }

Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs

@@ -69,7 +69,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
             return JobResult.Success();
         }
 
-        private class Progress : IProgress<FluentFTP.FtpProgress>
+        public class Progress : IProgress<FluentFTP.FtpProgress>
         {
             private string _name;
             private DateTime LastUpdate = DateTime.UnixEpoch;

Wabbajack.BuildServer/Models/PatchArchive.cs

@@ -0,0 +1,77 @@
+using System;
+using System.IO;
+using System.Net;
+using System.Threading.Tasks;
+using FluentFTP;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Model.Models;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.BuildServer.Models.Jobs;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using File = Alphaleonis.Win32.Filesystem.File;
+
+namespace Wabbajack.BuildServer.Models
+{
+    public class PatchArchive : AJobPayload
+    {
+        public override string Description => "Create a archive update patch";
+        public string Src { get; set; }
+        public string DestPK { get; set; }
+        public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
+        {
+            var srcPath = settings.PathForArchive(Src);
+            var destHash = (await db.DownloadStates.AsQueryable().Where(s => s.Key == DestPK).FirstOrDefaultAsync()).Hash;
+            var destPath = settings.PathForArchive(destHash);
+
+            Utils.Log($"Creating Patch ({Src} -> {DestPK})");
+            var cdnPath = CdnPath(Src, destHash);
+            
+            if (File.Exists(cdnPath))
+                return JobResult.Success();
+
+            Utils.Log($"Calculating Patch ({Src} -> {DestPK})");
+            await using var fs = File.Create(cdnPath);
+            
+            await using (var srcStream = File.OpenRead(srcPath))
+            await using (var destStream = File.OpenRead(destPath))
+            await using (var sigStream = File.Create(cdnPath + ".octo_sig"))
+            {
+                OctoDiff.Create(destStream, srcStream, sigStream, fs);
+            }
+            fs.Position = 0;
+            
+            Utils.Log($"Uploading Patch ({Src} -> {DestPK})");
+
+            int retries = 0;
+            TOP:
+            using (var client = new FtpClient("storage.bunnycdn.com"))
+            {
+                client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);
+                await client.ConnectAsync();
+                try
+                {
+                    await client.UploadAsync(fs, $"updates/{Src.FromBase64().ToHex()}_{destHash.FromBase64().ToHex()}", progress: new UploadToCDN.Progress(cdnPath));
+                }
+                catch (Exception ex)
+                {
+                    if (retries > 10) throw;
+                    Utils.Log(ex.ToString());
+                    Utils.Log("Retrying FTP Upload");
+                    retries++;
+                    goto TOP;
+                }
+            }
+            
+            return JobResult.Success();
+            
+        }
+
+        public static string CdnPath(string srcHash, string destHash)
+        {
+            return $"updates/{srcHash.FromBase64().ToHex()}_{destHash.FromBase64().ToHex()}";
+        }
+    }
+}

Wabbajack.Common/OctoDiff.cs

@@ -27,6 +27,21 @@ namespace Wabbajack.Common
             sigStream.Position = 0;
             return sigStream;
         }
+        
+        private static void CreateSignature(FileStream oldData, FileStream sigStream)
+        {
+            Utils.Status("Creating Patch Signature");
+            var signatureBuilder = new SignatureBuilder();
+            signatureBuilder.Build(oldData, new SignatureWriter(sigStream));
+            sigStream.Position = 0;
+        }
+        
+        public static void Create(FileStream oldData, FileStream newData, FileStream signature, FileStream output)
+        {
+            CreateSignature(oldData, signature);
+            var db = new DeltaBuilder {ProgressReporter = reporter};
+            db.BuildDelta(newData, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
+        }
 
         private class ProgressReporter : IProgressReporter
         {
@@ -44,5 +59,11 @@ namespace Wabbajack.Common
             var deltaApplier = new DeltaApplier();
             deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, reporter), output);
         }
+        
+        public static void Apply(FileStream input, FileStream patchStream, FileStream output)
+        {
+            var deltaApplier = new DeltaApplier();
+            deltaApplier.Apply(input, new BinaryDeltaReader(patchStream, reporter), output);
+        }
     }
 }

Wabbajack.Lib/AInstaller.cs

@@ -281,9 +281,7 @@ namespace Wabbajack.Lib
             {
                 if (destination == null) 
                     destination = Path.Combine(DownloadFolder, archive.Name);
-                await archive.State.Download(archive, destination);
+                await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, destination);
-                destination.FileHashCached();
-
             }
             catch (Exception ex)
             {

Wabbajack.Lib/ClientAPI.cs

@@ -0,0 +1,22 @@
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib
+{
+    public class ClientAPI
+    {
+        public static Common.Http.Client GetClient()
+        {
+            var client = new Common.Http.Client();
+            client.Headers.Add((Consts.MetricsKeyHeader, Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)));
+            return client;
+        }
+
+        public static async Task<Archive> GetModUpgrade(string hash)
+        {
+            using var response = await GetClient()
+                .GetAsync($"https://{Consts.WabbajackCacheHostname}/alternative/{hash.FromBase64().ToHex()}");
+            return !response.IsSuccessStatusCode ? null : (await response.Content.ReadAsStringAsync()).FromJSONString<Archive>();
+        }
+    }
+}

Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs

@@ -125,7 +125,6 @@ namespace Wabbajack.Lib.Downloaders
                 var streamResult = await downloader.AuthedClient.GetAsync(url);
                 if (streamResult.StatusCode != HttpStatusCode.OK)
                 {
-                    streamResult.Dispose();
                     Utils.ErrorThrow(new InvalidOperationException(), $"{downloader.SiteName} servers reported an error for file: {FileID}");
                 }
 

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
 using Wabbajack.Lib.Downloaders.UrlDownloaders;
 
@@ -75,5 +76,76 @@ namespace Wabbajack.Lib.Downloaders
                   .Distinct()
                   .Do(t => Downloaders.First(d => d.GetType() == t).Prepare());
         }
+
+        public static async Task<bool> DownloadWithPossibleUpgrade(Archive archive, string destination)
+        {
+            var success = await Download(archive, destination);
+            if (success)
+            {
+                await destination.FileHashCachedAsync();
+                return true;
+            }
+
+            Utils.Log($"Download failed, looking for upgrade");
+            var upgrade = await ClientAPI.GetModUpgrade(archive.Hash);
+            if (upgrade == null)
+            {
+                Utils.Log($"No upgrade found for {archive.Hash}");
+                return false;
+            }
+
+            Utils.Log($"Upgrading {archive.Hash}");
+            var upgradePath = Path.Combine(Path.GetDirectoryName(destination), "_Upgrade_" + archive.Name);
+            var upgradeResult = await Download(upgrade, upgradePath);
+            if (!upgradeResult) return false;
+
+            var patchName = $"{archive.Hash.FromBase64().ToHex()}_{upgrade.Hash.FromBase64().ToHex()}";
+            var patchPath = Path.Combine(Path.GetDirectoryName(destination), "_Patch_" + patchName);
+
+            var patchState = new Archive
+            {
+                Name = patchName,
+                State = new HTTPDownloader.State
+                {
+                    Url = $"https://wabbajackcdn.b-cdn.net/updates/{patchName}"
+                }
+            };
+
+            var patchResult = await Download(patchState, patchPath);
+            if (!patchResult) return false;
+
+            Utils.Status($"Applying Upgrade to {archive.Hash}");
+            await using (var patchStream = File.OpenRead(patchPath))
+            await using (var srcStream = File.OpenRead(upgradePath))
+            await using (var destStream = File.Create(destination))
+            {
+                OctoDiff.Apply(srcStream, patchStream, destStream);
+            }
+
+            await destination.FileHashCachedAsync();
+
+            return true;
+        }
+
+        private static async Task<bool> Download(Archive archive, string destination)
+        {
+            try
+            {
+                var result =  await archive.State.Download(archive, destination);
+                if (!result) return false;
+
+                if (archive.Hash == null) return true;
+                var hash = await destination.FileHashCachedAsync();
+                if (hash == archive.Hash) return true;
+
+                Utils.Log($"Hashed download is incorrect");
+                return false;
+
+            }
+            catch (Exception ex)
+            {
+                return false;
+            }
+        }
     }
 }

Wabbajack.Lib/ModListRegistry/ModListMetadata.cs

@@ -110,6 +110,10 @@ namespace Wabbajack.Lib.ModListRegistry
     {
         [JsonProperty("name")]
         public string Name { get; set; }
+        
+        [JsonProperty("machineURL")]
+        public string MachineURL { get; set; }
+        
         [JsonProperty("checked")]
         public DateTime Checked { get; set; }
         [JsonProperty("failed")]
@@ -117,9 +121,9 @@ namespace Wabbajack.Lib.ModListRegistry
         [JsonProperty("passed")]
         public int Passed { get; set; }
         [JsonProperty("link")]
-        public string Link => $"/lists/status/{Name}.json";
+        public string Link => $"/lists/status/{MachineURL}.json";
         [JsonProperty("report")]
-        public string Report => $"/lists/status/{Name}.html";
+        public string Report => $"/lists/status/{MachineURL}.html";
         [JsonProperty("has_failures")]
         public bool HasFailures => Failed > 0;
     }

Wabbajack.Test/DownloaderTests.cs

@@ -416,26 +416,6 @@ namespace Wabbajack.Test
             CollectionAssert.AreEqual(File.ReadAllBytes(Path.Combine(Game.SkyrimSpecialEdition.MetaData().GameLocation(), "Data/Update.esm")), File.ReadAllBytes(filename));
             Consts.TestMode = true;
         }
-
-        [TestMethod]
-        public async Task AFKModsDownloadTest()
-        {
-            await DownloadDispatcher.GetInstance<AFKModsDownloader>().Prepare();
-            const string ini = "[General]\n" +
-                               "directURL=https://www.afkmods.com/index.php?/files/file/2120-skyrim-save-system-overhaul/&do=download&r=20112&confirm=1&t=1&csrfKey=840a4a373144097693171a79df77d521";
-
-            var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
-
-            Assert.IsNotNull(state);
-
-            var converted = await state.RoundTripState();
-            Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
-            var filename = Guid.NewGuid().ToString();
-
-            Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
-
-            await converted.Download(new Archive { Name = "AFKMods Test.zip" }, filename);
-        }
         
         [TestMethod]
         public async Task BethesdaNetDownload()
@@ -519,6 +499,28 @@ namespace Wabbajack.Test
             
         }
 
+        [TestMethod]
+        public async Task TestUpgrading()
+        {
+            using var folder = new TempFolder();
+            var dest = Path.Combine(folder.Dir.FullName, "Cori.7z");
+            var archive = new Archive
+            {
+                Name = "Cori.7z",
+                Hash = "gCRVrvzDNH0=",
+                State = new NexusDownloader.State
+                {
+                    GameName = Game.SkyrimSpecialEdition.MetaData().NexusName,
+                    ModID = "24808",
+                    FileID = "123501"
+                }
+            };
+            Assert.IsTrue(await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, dest));
+            Assert.AreEqual("gCRVrvzDNH0=", await dest.FileHashCachedAsync());
+        }
+        
+        
+
         class TestInstaller : AInstaller
         {
             public TestInstaller(string archive, ModList modList, string outputFolder, string downloadFolder, SystemParameters parameters) : base(archive, modList, outputFolder, downloadFolder, parameters)