diff --git a/CHANGELOG.md b/CHANGELOG.md index fe40b6bf..e6b43ff3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ ### Changelog +#### Version - 2.5.3.21 - 6/9/2022 +* Fix a bug in the streaming MediaFire downloader +* Improve the reliability of MediaFire, and Manual downloaders +* Improve logging around the Wabbajack CDN + +#### Version - 2.5.3.20 - 6/8/2022 +* Improve reliability of MediaFire, Mega and GDrive downloaders + +#### Version - 2.5.3.19 - 6/4/2022 +* Fix a potential long standing problem with hash caching + #### Version - 2.5.3.18 - 6/1/2022 * Downgrade to a working copy of Game Finder diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 9804cc04..18beec34 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -6,8 +6,8 @@ wabbajack-cli Wabbajack x64 - 2.5.3.18 - 2.5.3.18 + 2.5.3.21 + 2.5.3.21 Copyright © 2019-2022 An automated ModList installer true diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj index 8c80d7b2..aacadd35 100644 --- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj +++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj @@ -4,8 +4,8 @@ Exe net5.0-windows true - 2.5.3.18 - 2.5.3.18 + 2.5.3.21 + 2.5.3.21 Copyright © 2019-2022 Wabbajack Application Launcher true diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 56d33517..3530e2ea 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -234,6 +234,20 @@ namespace Wabbajack.Lib var client = new Http.Client(); Utils.Log("Getting upgrades list"); var upgrades = (await client.GetJsonAsync(Consts.UpgradedFilesURL)); + + var tmp = new List(); + foreach (var miss in missing) + { + if (miss.State is ManualDownloader.State ms && await DownloadDispatcher.ProxyHas(new Uri(ms.Url))) + { + tmp.Add(DownloadDispatcher.MaybeProxy(miss)); + } + else + { + tmp.Add(DownloadDispatcher.MaybeProxy(miss)); + } + } + missing = tmp; if (download) { diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index 17d8af82..9e2875e9 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -3,7 +3,9 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; +using System.Text; using System.Threading.Tasks; +using System.Web; using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; using Wabbajack.Lib.Downloaders.DTOs.ModListValidation; @@ -105,6 +107,8 @@ namespace Wabbajack.Lib.Downloaders public static async Task DownloadWithPossibleUpgrade(Archive archive, AbsolutePath destination, ValidatedArchive[]? upgrades = null) { + archive = MaybeProxy(archive); + bool ShouldTry(Archive archive) { return upgrades == null || upgrades.All(a => a.Original.Hash != archive.Hash); @@ -177,7 +181,46 @@ namespace Wabbajack.Lib.Downloaders return DownloadResult.Update; } - + + public static Archive MaybeProxy(Archive archive) + { + if (archive.State is (not GoogleDriveDownloader.State + and not MegaDownloader.State + and not MediaFireDownloader.State + and not ModDBDownloader.State + and not ManualDownloader.State)) + return archive; + + var uri = archive.State.GetManifestURL(archive); + var hash = archive.Hash != default ? $"&hash={archive.Hash.ToHex()}" : ""; + Utils.Log($"Downloading via proxy ({Encoding.UTF8.GetBytes(uri!).xxHash().ToHex()}) {uri}"); + var newUri = $"https://build.wabbajack.org/proxy?name={archive.Name}{hash}&uri={HttpUtility.UrlEncode(uri)}"; + + return new Archive(new HTTPDownloader.State(newUri)) + { + Name = archive.Name, + Size = archive.Size, + Hash = archive.Hash, + }; + + } + + public static async Task ProxyHas(Uri uri) + { + var newUri = $"https://build.wabbajack.org/proxy?uri={HttpUtility.UrlEncode(uri.ToString())}"; + var msg = new HttpRequestMessage(HttpMethod.Head, newUri); + var client = new Http.Client(); + try + { + var result = await client.SendAsync(msg); + return result.IsSuccessStatusCode; + } + catch (Exception ex) + { + return false; + } + } + public static async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func>? downloadResolver = null) { downloadResolver ??= async a => default; diff --git a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs index d394dddc..844887f0 100644 --- a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs +++ b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs @@ -89,25 +89,35 @@ namespace Wabbajack.Lib.Downloaders using var queue = new WorkQueue(); await definition.Parts.PMap(queue, async part => { - Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(definition.Parts.Length - part.Index, definition.Parts.Length)); - await using var ostream = mmfile.CreateViewStream(part.Offset, part.Size); - - if (DomainRemaps.TryGetValue(Url.Host, out var remap)) + try { - var builder = new UriBuilder(Url) {Host = remap}; - using var response = await GetWithCDNRetry(client, $"{builder}/parts/{part.Index}"); - if (!response.IsSuccessStatusCode) - throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown"); - await response.Content.CopyToAsync(ostream); - + Utils.Status($"Downloading {a.Name}", + Percent.FactoryPutInRange(definition.Parts.Length - part.Index, definition.Parts.Length)); + await using var ostream = mmfile.CreateViewStream(part.Offset, part.Size); + + if (DomainRemaps.TryGetValue(Url.Host, out var remap)) + { + var builder = new UriBuilder(Url) {Host = remap}; + using var response = await GetWithCDNRetry(client, $"{builder}/parts/{part.Index}"); + if (!response.IsSuccessStatusCode) + throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown"); + await response.Content.CopyToAsync(ostream); + + } + else + { + using var response = await GetWithRetry(client, $"{Url}/parts/{part.Index}"); + if (!response.IsSuccessStatusCode) + throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown"); + await response.Content.CopyToAsync(ostream); + + } } - else + catch (Exception ex) { - using var response = await GetWithRetry(client, $"{Url}/parts/{part.Index}"); - if (!response.IsSuccessStatusCode) - throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown"); - await response.Content.CopyToAsync(ostream); - + Utils.LogStraightToFile("CDN ERROR"); + Utils.LogStraightToFile(ex.ToString()); + throw; } }); diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 8ebba271..1037fc2f 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -78,7 +78,7 @@ namespace Wabbajack.Test Assert.True(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List{"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k" } })); Assert.False(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List{ "blerg" }})); - await converted.Download(new Archive(state: null!) {Name = "MEGA Test.txt"}, filename.Path); + await DownloadDispatcher.DownloadWithPossibleUpgrade(new Archive(state: converted) {Name = "MEGA Test.txt"}, filename.Path); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); @@ -141,7 +141,7 @@ namespace Wabbajack.Test Assert.True(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List { "1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_" } })); Assert.False(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List()})); - await converted.Download(new Archive(state: null!) { Name = "MEGA Test.txt" }, filename.Path); + await DownloadDispatcher.DownloadWithPossibleUpgrade(new Archive(converted) { Name = "MEGA Test.txt" }, filename.Path); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); @@ -225,7 +225,7 @@ namespace Wabbajack.Test {AllowedPrefixes = new List {"http://www.mediafire.com/file/agiqzm1xwebczpx/"}})); Assert.False(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List()})); - await converted.Download(new Archive(state: null!) { Name = "Media Fire Test.zip" }, filename.Path); + await DownloadDispatcher.DownloadWithPossibleUpgrade(new Archive(state: converted) { Name = "Media Fire Test.zip" }, filename.Path); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 8114ff55..a662db0e 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -6,8 +6,8 @@ true x64 win10-x64 - 2.5.3.18 - 2.5.3.18 + 2.5.3.21 + 2.5.3.21 Copyright © 2019-2022 An automated ModList installer true