#### Version - 2.5.3.21 - 6/9/2022

* Fix a bug in the streaming MediaFire downloader
* Improve the reliability of MediaFire, and Manual downloaders
* Improve logging around the Wabbajack CDN

CHANGELOG.md

@@ -1,5 +1,16 @@
 ### Changelog
 
+#### Version - 2.5.3.21 - 6/9/2022
+* Fix a bug in the streaming MediaFire downloader
+* Improve the reliability of MediaFire, and Manual downloaders
+* Improve logging around the Wabbajack CDN
+
+#### Version - 2.5.3.20 - 6/8/2022
+* Improve reliability of MediaFire, Mega and GDrive downloaders
+
+#### Version - 2.5.3.19 - 6/4/2022
+* Fix a potential long standing problem with hash caching
+
 #### Version - 2.5.3.18 - 6/1/2022
 * Downgrade to a working copy of Game Finder
 

Wabbajack.CLI/Wabbajack.CLI.csproj

@@ -6,8 +6,8 @@
         <AssemblyName>wabbajack-cli</AssemblyName>
         <Company>Wabbajack</Company>
         <Platforms>x64</Platforms>
-        <AssemblyVersion>2.5.3.18</AssemblyVersion>
+        <AssemblyVersion>2.5.3.21</AssemblyVersion>
-        <FileVersion>2.5.3.18</FileVersion>
+        <FileVersion>2.5.3.21</FileVersion>
         <Copyright>Copyright © 2019-2022</Copyright>
         <Description>An automated ModList installer</Description>
         <PublishReadyToRun>true</PublishReadyToRun>        

Wabbajack.Launcher/Wabbajack.Launcher.csproj

@@ -4,8 +4,8 @@
         <OutputType>Exe</OutputType>
         <TargetFramework>net5.0-windows</TargetFramework>
         <UseWPF>true</UseWPF>
-        <AssemblyVersion>2.5.3.18</AssemblyVersion>
+        <AssemblyVersion>2.5.3.21</AssemblyVersion>
-        <FileVersion>2.5.3.18</FileVersion>
+        <FileVersion>2.5.3.21</FileVersion>
         <Copyright>Copyright © 2019-2022</Copyright>
         <Description>Wabbajack Application Launcher</Description>
         <PublishReadyToRun>true</PublishReadyToRun>

Wabbajack.Lib/AInstaller.cs

@@ -235,6 +235,20 @@ namespace Wabbajack.Lib
             Utils.Log("Getting upgrades list");
             var upgrades =  (await client.GetJsonAsync<ValidatedArchive[]>(Consts.UpgradedFilesURL));
 
+            var tmp = new List<Archive>();
+            foreach (var miss in missing)
+            {
+                if (miss.State is ManualDownloader.State ms && await DownloadDispatcher.ProxyHas(new Uri(ms.Url)))
+                {
+                    tmp.Add(DownloadDispatcher.MaybeProxy(miss));
+                }
+                else
+                {
+                    tmp.Add(DownloadDispatcher.MaybeProxy(miss));
+                }
+            }
+            missing = tmp;
+            
             if (download)
             {
                 var result = SendDownloadMetrics(missing);

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -3,7 +3,9 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
+using System.Text;
 using System.Threading.Tasks;
+using System.Web;
 using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
 using Wabbajack.Lib.Downloaders.DTOs.ModListValidation;
@@ -105,6 +107,8 @@ namespace Wabbajack.Lib.Downloaders
 
         public static async Task<DownloadResult> DownloadWithPossibleUpgrade(Archive archive, AbsolutePath destination, ValidatedArchive[]? upgrades = null)
         {
+            archive = MaybeProxy(archive);
+            
             bool ShouldTry(Archive archive)
             {
                 return upgrades == null || upgrades.All(a => a.Original.Hash != archive.Hash);
@@ -178,6 +182,45 @@ namespace Wabbajack.Lib.Downloaders
             return DownloadResult.Update;
         }
 
+        public static Archive MaybeProxy(Archive archive)
+        {
+            if (archive.State is (not GoogleDriveDownloader.State 
+                and not MegaDownloader.State 
+                and not MediaFireDownloader.State 
+                and not ModDBDownloader.State
+                and not ManualDownloader.State))
+                return archive;
+
+            var uri = archive.State.GetManifestURL(archive);
+            var hash = archive.Hash != default ? $"&hash={archive.Hash.ToHex()}" : "";
+            Utils.Log($"Downloading via proxy ({Encoding.UTF8.GetBytes(uri!).xxHash().ToHex()}) {uri}");
+            var newUri = $"https://build.wabbajack.org/proxy?name={archive.Name}{hash}&uri={HttpUtility.UrlEncode(uri)}";
+
+            return new Archive(new HTTPDownloader.State(newUri))
+            {
+                Name = archive.Name,
+                Size = archive.Size,
+                Hash = archive.Hash,
+            };
+
+        }
+
+        public static async Task<bool> ProxyHas(Uri uri)
+        {
+            var newUri = $"https://build.wabbajack.org/proxy?uri={HttpUtility.UrlEncode(uri.ToString())}";
+            var msg = new HttpRequestMessage(HttpMethod.Head, newUri);
+            var client = new Http.Client();
+            try
+            {
+                var result = await client.SendAsync(msg);
+                return result.IsSuccessStatusCode;
+            }
+            catch (Exception ex)
+            {
+                return false;
+            }
+        }
+
         public static async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>>? downloadResolver = null)
         {
             downloadResolver ??= async a => default;

Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs

@@ -89,25 +89,35 @@ namespace Wabbajack.Lib.Downloaders
                 using var queue = new WorkQueue();
                 await definition.Parts.PMap(queue, async part =>
                 {
-                    Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(definition.Parts.Length - part.Index, definition.Parts.Length));
+                    try
-                    await using var ostream = mmfile.CreateViewStream(part.Offset, part.Size);
-
-                    if (DomainRemaps.TryGetValue(Url.Host, out var remap))
                     {
-                        var builder = new UriBuilder(Url) {Host = remap};
+                        Utils.Status($"Downloading {a.Name}",
-                        using var response = await GetWithCDNRetry(client, $"{builder}/parts/{part.Index}");
+                            Percent.FactoryPutInRange(definition.Parts.Length - part.Index, definition.Parts.Length));
-                        if (!response.IsSuccessStatusCode)
+                        await using var ostream = mmfile.CreateViewStream(part.Offset, part.Size);
-                            throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown");
-                        await response.Content.CopyToAsync(ostream);
 
+                        if (DomainRemaps.TryGetValue(Url.Host, out var remap))
+                        {
+                            var builder = new UriBuilder(Url) {Host = remap};
+                            using var response = await GetWithCDNRetry(client, $"{builder}/parts/{part.Index}");
+                            if (!response.IsSuccessStatusCode)
+                                throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown");
+                            await response.Content.CopyToAsync(ostream);
+
+                        }
+                        else
+                        {
+                            using var response = await GetWithRetry(client, $"{Url}/parts/{part.Index}");
+                            if (!response.IsSuccessStatusCode)
+                                throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown");
+                            await response.Content.CopyToAsync(ostream);
+
+                        }
                     }
-                    else
+                    catch (Exception ex)
                     {
-                        using var response = await GetWithRetry(client, $"{Url}/parts/{part.Index}");
+                        Utils.LogStraightToFile("CDN ERROR");
-                        if (!response.IsSuccessStatusCode)
+                        Utils.LogStraightToFile(ex.ToString());
-                            throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown");
+                        throw;
-                        await response.Content.CopyToAsync(ostream);
-
                     }
 
                 });

Wabbajack.Test/DownloaderTests.cs

@@ -78,7 +78,7 @@ namespace Wabbajack.Test
             Assert.True(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List<string>{"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k" } }));
             Assert.False(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>{ "blerg" }}));
 
-            await converted.Download(new Archive(state: null!) {Name = "MEGA Test.txt"}, filename.Path);
+            await DownloadDispatcher.DownloadWithPossibleUpgrade(new Archive(state: converted) {Name = "MEGA Test.txt"}, filename.Path);
 
             Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
 
@@ -141,7 +141,7 @@ namespace Wabbajack.Test
             Assert.True(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List<string> { "1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_" } }));
             Assert.False(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List<string>()}));
 
-            await converted.Download(new Archive(state: null!) { Name = "MEGA Test.txt" }, filename.Path);
+            await DownloadDispatcher.DownloadWithPossibleUpgrade(new Archive(converted) { Name = "MEGA Test.txt" }, filename.Path);
 
             Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
 
@@ -225,7 +225,7 @@ namespace Wabbajack.Test
                 {AllowedPrefixes = new List<string> {"http://www.mediafire.com/file/agiqzm1xwebczpx/"}}));
             Assert.False(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List<string>()}));
 
-            await converted.Download(new Archive(state: null!) { Name = "Media Fire Test.zip" }, filename.Path);
+            await DownloadDispatcher.DownloadWithPossibleUpgrade(new Archive(state: converted) { Name = "Media Fire Test.zip" }, filename.Path);
 
             Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
 

Wabbajack/Wabbajack.csproj

@@ -6,8 +6,8 @@
     <UseWPF>true</UseWPF>
     <Platforms>x64</Platforms>
     <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
-      <AssemblyVersion>2.5.3.18</AssemblyVersion>
+      <AssemblyVersion>2.5.3.21</AssemblyVersion>
-      <FileVersion>2.5.3.18</FileVersion>
+      <FileVersion>2.5.3.21</FileVersion>
     <Copyright>Copyright © 2019-2022</Copyright>
     <Description>An automated ModList installer</Description>
     <PublishReadyToRun>true</PublishReadyToRun>