Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

#### Version - 2.5.3.21 - 6/9/2022

Browse Source 
* Fix a bug in the streaming MediaFire downloader
* Improve the reliability of MediaFire, and Manual downloaders
* Improve logging around the Wabbajack CDN
This commit is contained in:
Timothy Baldridge 2022-06-09 14:53:42 -06:00
parent 4e631ddd69
commit 9c6145ae39
8 changed files with 104 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
CHANGELOG.md
View File

@ -1,5 +1,16 @@
### Changelog
#### Version - 2.5.3.21 - 6/9/2022
* Fix a bug in the streaming MediaFire downloader
* Improve the reliability of MediaFire, and Manual downloaders
* Improve logging around the Wabbajack CDN
#### Version - 2.5.3.20 - 6/8/2022
* Improve reliability of MediaFire, Mega and GDrive downloaders
#### Version - 2.5.3.19 - 6/4/2022
* Fix a potential long standing problem with hash caching
#### Version - 2.5.3.18 - 6/1/2022
* Downgrade to a working copy of Game Finder

4
Wabbajack.CLI/Wabbajack.CLI.csproj
View File

@ -6,8 +6,8 @@
<AssemblyName>wabbajack-cli</AssemblyName>
<Company>Wabbajack</Company>
<Platforms>x64</Platforms>
<AssemblyVersion>2.5.3.18</AssemblyVersion>
<FileVersion>2.5.3.18</FileVersion>
<AssemblyVersion>2.5.3.21</AssemblyVersion>
<FileVersion>2.5.3.21</FileVersion>
<Copyright>Copyright © 2019-2022</Copyright>
<Description>An automated ModList installer</Description>
<PublishReadyToRun>true</PublishReadyToRun>

4
Wabbajack.Launcher/Wabbajack.Launcher.csproj
View File

@ -4,8 +4,8 @@
<OutputType>Exe</OutputType>
<TargetFramework>net5.0-windows</TargetFramework>
<UseWPF>true</UseWPF>
<AssemblyVersion>2.5.3.18</AssemblyVersion>
<FileVersion>2.5.3.18</FileVersion>
<AssemblyVersion>2.5.3.21</AssemblyVersion>
<FileVersion>2.5.3.21</FileVersion>
<Copyright>Copyright © 2019-2022</Copyright>
<Description>Wabbajack Application Launcher</Description>
<PublishReadyToRun>true</PublishReadyToRun>

14
Wabbajack.Lib/AInstaller.cs
View File

@ -234,6 +234,20 @@ namespace Wabbajack.Lib
var client = new Http.Client();
Utils.Log("Getting upgrades list");
var upgrades = (await client.GetJsonAsync<ValidatedArchive[]>(Consts.UpgradedFilesURL));
var tmp = new List<Archive>();
foreach (var miss in missing)
{
if (miss.State is ManualDownloader.State ms && await DownloadDispatcher.ProxyHas(new Uri(ms.Url)))
{
tmp.Add(DownloadDispatcher.MaybeProxy(miss));
}
else
{
tmp.Add(DownloadDispatcher.MaybeProxy(miss));
}
}
missing = tmp;
if (download)
{

45
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -3,7 +3,9 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using System.Web;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders.DTOs.ModListValidation;
@ -105,6 +107,8 @@ namespace Wabbajack.Lib.Downloaders
public static async Task<DownloadResult> DownloadWithPossibleUpgrade(Archive archive, AbsolutePath destination, ValidatedArchive[]? upgrades = null)
{
archive = MaybeProxy(archive);
bool ShouldTry(Archive archive)
{
return upgrades == null || upgrades.All(a => a.Original.Hash != archive.Hash);
@ -177,7 +181,46 @@ namespace Wabbajack.Lib.Downloaders
return DownloadResult.Update;
}
public static Archive MaybeProxy(Archive archive)
{
if (archive.State is (not GoogleDriveDownloader.State
and not MegaDownloader.State
and not MediaFireDownloader.State
and not ModDBDownloader.State
and not ManualDownloader.State))
return archive;
var uri = archive.State.GetManifestURL(archive);
var hash = archive.Hash != default ? $"&hash={archive.Hash.ToHex()}" : "";
Utils.Log($"Downloading via proxy ({Encoding.UTF8.GetBytes(uri!).xxHash().ToHex()}) {uri}");
var newUri = $"https://build.wabbajack.org/proxy?name={archive.Name}{hash}&uri={HttpUtility.UrlEncode(uri)}";
return new Archive(new HTTPDownloader.State(newUri))
{
Name = archive.Name,
Size = archive.Size,
Hash = archive.Hash,
};
}
public static async Task<bool> ProxyHas(Uri uri)
{
var newUri = $"https://build.wabbajack.org/proxy?uri={HttpUtility.UrlEncode(uri.ToString())}";
var msg = new HttpRequestMessage(HttpMethod.Head, newUri);
var client = new Http.Client();
try
{
var result = await client.SendAsync(msg);
return result.IsSuccessStatusCode;
}
catch (Exception ex)
{
return false;
}
}
public static async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>>? downloadResolver = null)
{
downloadResolver ??= async a => default;

42
Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs
View File

@ -89,25 +89,35 @@ namespace Wabbajack.Lib.Downloaders
using var queue = new WorkQueue();
await definition.Parts.PMap(queue, async part =>
{
Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(definition.Parts.Length - part.Index, definition.Parts.Length));
await using var ostream = mmfile.CreateViewStream(part.Offset, part.Size);
if (DomainRemaps.TryGetValue(Url.Host, out var remap))
try
{
var builder = new UriBuilder(Url) {Host = remap};
using var response = await GetWithCDNRetry(client, $"{builder}/parts/{part.Index}");
if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown");
await response.Content.CopyToAsync(ostream);
Utils.Status($"Downloading {a.Name}",
Percent.FactoryPutInRange(definition.Parts.Length - part.Index, definition.Parts.Length));
await using var ostream = mmfile.CreateViewStream(part.Offset, part.Size);
if (DomainRemaps.TryGetValue(Url.Host, out var remap))
{
var builder = new UriBuilder(Url) {Host = remap};
using var response = await GetWithCDNRetry(client, $"{builder}/parts/{part.Index}");
if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown");
await response.Content.CopyToAsync(ostream);
}
else
{
using var response = await GetWithRetry(client, $"{Url}/parts/{part.Index}");
if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown");
await response.Content.CopyToAsync(ostream);
}
}
else
catch (Exception ex)
{
using var response = await GetWithRetry(client, $"{Url}/parts/{part.Index}");
if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown");
await response.Content.CopyToAsync(ostream);
Utils.LogStraightToFile("CDN ERROR");
Utils.LogStraightToFile(ex.ToString());
throw;
}
});

6
Wabbajack.Test/DownloaderTests.cs
View File

@ -78,7 +78,7 @@ namespace Wabbajack.Test
Assert.True(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List<string>{"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k" } }));
Assert.False(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>{ "blerg" }}));
await converted.Download(new Archive(state: null!) {Name = "MEGA Test.txt"}, filename.Path);
await DownloadDispatcher.DownloadWithPossibleUpgrade(new Archive(state: converted) {Name = "MEGA Test.txt"}, filename.Path);
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
@ -141,7 +141,7 @@ namespace Wabbajack.Test
Assert.True(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List<string> { "1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_" } }));
Assert.False(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List<string>()}));
await converted.Download(new Archive(state: null!) { Name = "MEGA Test.txt" }, filename.Path);
await DownloadDispatcher.DownloadWithPossibleUpgrade(new Archive(converted) { Name = "MEGA Test.txt" }, filename.Path);
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
@ -225,7 +225,7 @@ namespace Wabbajack.Test
{AllowedPrefixes = new List<string> {"http://www.mediafire.com/file/agiqzm1xwebczpx/"}}));
Assert.False(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List<string>()}));
await converted.Download(new Archive(state: null!) { Name = "Media Fire Test.zip" }, filename.Path);
await DownloadDispatcher.DownloadWithPossibleUpgrade(new Archive(state: converted) { Name = "Media Fire Test.zip" }, filename.Path);
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());

4
Wabbajack/Wabbajack.csproj
View File

@ -6,8 +6,8 @@
<UseWPF>true</UseWPF>
<Platforms>x64</Platforms>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
<AssemblyVersion>2.5.3.18</AssemblyVersion>
<FileVersion>2.5.3.18</FileVersion>
<AssemblyVersion>2.5.3.21</AssemblyVersion>
<FileVersion>2.5.3.21</FileVersion>
<Copyright>Copyright © 2019-2022</Copyright>
<Description>An automated ModList installer</Description>
<PublishReadyToRun>true</PublishReadyToRun>