Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #584 from wabbajack-tools/self-healing-lists

Browse Source 
Self healing lists
This commit is contained in:
Timothy Baldridge 2020-02-27 17:24:11 -07:00 committed by GitHub
commit 13e62124a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 554 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Wabbajack.BuildServer/Controllers/ListValidation.cs
View File

@ -11,6 +11,7 @@ using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.BuildServer.Controllers
@ -39,7 +40,7 @@ namespace Wabbajack.BuildServer.Controllers
<description>These are mods that are broken and need updating</description>
{{ each $.failed }}
<item>
<title>{{$.Archive.Name}}</title>
<title>{{$.Archive.Name}} {{$.Archive.Hash}} {{$.Archive.State.PrimaryKeyString}}</title>
<link>{{$.Archive.Name}}</link>
</item>
{{/each}}
@ -105,6 +106,21 @@ namespace Wabbajack.BuildServer.Controllers
};
}
[HttpGet]
[Route("status/{Name}.json")]
public async Task<ContentResult> HandleGetListJson(string Name)
{
var lst = (await ModListStatus.ByName(Db, Name)).DetailedStatus;
lst.Archives.Do(a => a.Archive.Meta = null);
return new ContentResult
{
ContentType = "application/json",
StatusCode = (int) HttpStatusCode.OK,
Content = lst.ToJSON()
};
}
}
}

135
Wabbajack.BuildServer/Controllers/ModlistUpdater.cs Normal file
View File

@ -0,0 +1,135 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using AlphaFile = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.BuildServer.Controllers
{
[ApiController]
[Route("/listupdater")]
public class ModlistUpdater : AControllerBase<ModlistUpdater>
{
private AppSettings _settings;
private SqlService _sql;
public ModlistUpdater(ILogger<ModlistUpdater> logger, DBContext db, SqlService sql, AppSettings settings) : base(logger, db)
{
_settings = settings;
_sql = sql;
}
[HttpGet]
[Route("/alternative/{xxHash}")]
public async Task<IActionResult> GetAlternative(string xxHash)
{
var startingHash = xxHash.FromHex().ToBase64();
Utils.Log($"Alternative requested for {startingHash}");
var state = await Db.DownloadStates.AsQueryable()
.Where(s => s.Hash == startingHash)
.Where(s => s.State is NexusDownloader.State)
.OrderByDescending(s => s.LastValidationTime).FirstOrDefaultAsync();
if (state == null)
return NotFound("Original state not found");
var nexusState = state.State as NexusDownloader.State;
Utils.Log($"Found original, looking for alternatives to {startingHash}");
var newArchive = await FindAlternatives(nexusState, startingHash);
if (newArchive == null)
{
return NotFound("No alternative available");
}
Utils.Log($"Found {newArchive.State.PrimaryKeyString} as an alternative to {startingHash}");
if (newArchive.Hash == null)
{
Db.Jobs.InsertOne(new Job
{
Payload = new IndexJob
{
Archive = newArchive
},
OnSuccess = new Job
{
Payload = new PatchArchive
{
Src = startingHash,
DestPK = newArchive.State.PrimaryKeyString
}
}
});
return Accepted("Enqueued for Processing");
}
if (!AlphaFile.Exists(PatchArchive.CdnPath(startingHash, newArchive.Hash)))
{
Db.Jobs.InsertOne(new Job
{
Priority = Job.JobPriority.High,
Payload = new PatchArchive
{
Src = startingHash,
DestPK = newArchive.State.PrimaryKeyString
}
});
}
return Ok(newArchive.ToJSON());
}
private async Task<Archive> FindAlternatives(NexusDownloader.State state, string srcHash)
{
var origSize = AlphaFile.GetSize(_settings.PathForArchive(srcHash));
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var allMods = await api.GetModFiles(GameRegistry.GetByFuzzyName(state.GameName).Game,
int.Parse(state.ModID));
var archive = allMods.files.Where(m => !string.IsNullOrEmpty(m.category_name))
.OrderBy(s => Math.Abs((long)s.size - origSize))
.Select(s => new Archive {
Name = s.file_name,
Size = (long)s.size,
State = new NexusDownloader.State
{
GameName = state.GameName,
ModID = state.ModID,
FileID = s.file_id.ToString()
}}).FirstOrDefault();
if (archive == null)
{
Utils.Log($"No alternative for {srcHash}");
return null;
}
Utils.Log($"Found alternative for {srcHash}");
var indexed = await Db.DownloadStates.AsQueryable().Where(s => s.Key == archive.State.PrimaryKeyString).FirstOrDefaultAsync();
if (indexed == null)
{
return archive;
}
Utils.Log($"Pre-Indexed alternative {indexed.Hash} found for {srcHash}");
archive.Hash = indexed.Hash;
return archive;
}
}
}

25
Wabbajack.BuildServer/Extensions.cs
View File

@ -1,13 +1,18 @@
using System;
using System.Collections.Concurrent;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Builder;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.Common;
using Directory =Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.BuildServer
{
@ -41,5 +46,25 @@ namespace Wabbajack.BuildServer
{
return authenticationBuilder.AddScheme<ApiKeyAuthenticationOptions, ApiKeyAuthenticationHandler>(ApiKeyAuthenticationOptions.DefaultScheme, options);
}
private static ConcurrentDictionary<string, string> PathForArchiveHash = new ConcurrentDictionary<string, string>();
public static string PathForArchive(this AppSettings settings, string hash)
{
if (PathForArchiveHash.TryGetValue(hash, out string result))
return result;
var hexHash = hash.FromBase64().ToHex();
var ends = "_" + hexHash + "_";
var file = Directory.EnumerateFiles(settings.ArchiveDir, DirectoryEnumerationOptions.Files,
new DirectoryEnumerationFilters
{
InclusionFilter = f => Path.GetFileNameWithoutExtension(f.FileName).EndsWith(ends)
}).FirstOrDefault();
if (file != null)
PathForArchiveHash.TryAdd(hash, file);
return file;
}
}
}

3
Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs
View File

@ -21,7 +21,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue
typeof(EnqueueRecentFiles),
typeof(UploadToCDN),
typeof(IndexDynDOLOD),
typeof(ReindexArchives)
typeof(ReindexArchives),
typeof(PatchArchive)
};
public static Dictionary<Type, string> TypeToName { get; set; }
public static Dictionary<string, Type> NameToType { get; set; }

7
Wabbajack.BuildServer/Models/JobQueue/Job.cs
View File

@ -29,6 +29,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue
public bool RequiresNexus { get; set; } = true;
public AJobPayload Payload { get; set; }
public Job OnSuccess { get; set; }
public static async Task<String> Enqueue(DBContext db, Job job)
{
await db.Jobs.InsertOneAsync(job);
@ -52,6 +54,11 @@ namespace Wabbajack.BuildServer.Models.JobQueue
public static async Task<Job> Finish(DBContext db, Job job, JobResult jobResult)
{
if (jobResult.ResultType == JobResultType.Success && job.OnSuccess != null)
{
await db.Jobs.InsertOneAsync(job.OnSuccess);
}
var filter = new BsonDocument
{
{"_id", job.Id},

94
Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs
View File

@ -2,12 +2,15 @@
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using File = Alphaleonis.Win32.Filesystem.File;
@ -43,10 +46,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
private async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
{
var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlist_path))
@ -76,17 +77,9 @@ namespace Wabbajack.BuildServer.Models.Jobs
var validated = (await installer.Archives
.PMap(queue, async archive =>
{
bool is_failed;
try
{
is_failed = !(await archive.State.Verify(archive)) || !archive.State.IsWhitelisted(whitelists.ServerWhitelist);
}
catch (Exception)
{
is_failed = false;
}
var isValid = await IsValid(db, whitelists, archive);
return new DetailedStatusItem {IsFailing = is_failed, Archive = archive};
return new DetailedStatusItem {IsFailing = !isValid, Archive = archive};
}))
.ToList();
@ -105,6 +98,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
Summary = new ModlistSummary
{
Name = status.Name,
MachineURL = list.Links?.MachineURL ?? status.Name,
Checked = status.Checked,
Failed = status.Archives.Count(a => a.IsFailing),
Passed = status.Archives.Count(a => !a.IsFailing),
@ -119,5 +113,79 @@ namespace Wabbajack.BuildServer.Models.Jobs
$"Done updating {dto.Summary.Name}");
}
private async Task<bool> IsValid(DBContext db, ValidateModlist whitelists, Archive archive)
{
try
{
if (!archive.State.IsWhitelisted(whitelists.ServerWhitelist)) return false;
try
{
if (archive.State is NexusDownloader.State state)
{
if (await ValidateNexusFast(db, state)) return true;
}
else if (archive.State is HTTPDownloader.State hstate &&
hstate.Url.StartsWith("https://wabbajack"))
{
return true;
}
else
{
if (await archive.State.Verify(archive)) return true;
}
}
catch (Exception)
{
// ignored
}
var result = await ClientAPI.GetModUpgrade(archive.Hash);
if (result != null) return true;
}
catch (Exception)
{
return true;
}
return false;
}
private async Task<bool> ValidateNexusFast(DBContext db, NexusDownloader.State state)
{
try
{
var gameMeta = GameRegistry.GetByFuzzyName(state.GameName);
if (gameMeta == null)
return false;
var game = gameMeta.Game;
if (!int.TryParse(state.ModID, out var modID))
return false;
var modFiles = (await db.NexusModFiles.AsQueryable().Where(g => g.Game == gameMeta.NexusName && g.ModId == state.ModID).FirstOrDefaultAsync())?.Data;
if (modFiles == null)
{
Utils.Log($"No Cache for {state.PrimaryKeyString} falling back to HTTP");
var nexusApi = await NexusApiClient.Get();
modFiles = await nexusApi.GetModFiles(game, modID);
}
if (!ulong.TryParse(state.FileID, out var fileID))
return false;
var found = modFiles.files
.FirstOrDefault(file => file.file_id == fileID && file.category_name != null);
return found != null;
}
catch (Exception ex)
{
return false;
}
}
}
}

2
Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs
View File

@ -69,7 +69,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
private class Progress : IProgress<FluentFTP.FtpProgress>
public class Progress : IProgress<FluentFTP.FtpProgress>
{
private string _name;
private DateTime LastUpdate = DateTime.UnixEpoch;

77
Wabbajack.BuildServer/Models/PatchArchive.cs Normal file
View File

@ -0,0 +1,77 @@
using System;
using System.IO;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using File = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.BuildServer.Models
{
public class PatchArchive : AJobPayload
{
public override string Description => "Create a archive update patch";
public string Src { get; set; }
public string DestPK { get; set; }
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
{
var srcPath = settings.PathForArchive(Src);
var destHash = (await db.DownloadStates.AsQueryable().Where(s => s.Key == DestPK).FirstOrDefaultAsync()).Hash;
var destPath = settings.PathForArchive(destHash);
Utils.Log($"Creating Patch ({Src} -> {DestPK})");
var cdnPath = CdnPath(Src, destHash);
if (File.Exists(cdnPath))
return JobResult.Success();
Utils.Log($"Calculating Patch ({Src} -> {DestPK})");
await using var fs = File.Create(cdnPath);
await using (var srcStream = File.OpenRead(srcPath))
await using (var destStream = File.OpenRead(destPath))
await using (var sigStream = File.Create(cdnPath + ".octo_sig"))
{
OctoDiff.Create(destStream, srcStream, sigStream, fs);
}
fs.Position = 0;
Utils.Log($"Uploading Patch ({Src} -> {DestPK})");
int retries = 0;
TOP:
using (var client = new FtpClient("storage.bunnycdn.com"))
{
client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);
await client.ConnectAsync();
try
{
await client.UploadAsync(fs, $"updates/{Src.FromBase64().ToHex()}_{destHash.FromBase64().ToHex()}", progress: new UploadToCDN.Progress(cdnPath));
}
catch (Exception ex)
{
if (retries > 10) throw;
Utils.Log(ex.ToString());
Utils.Log("Retrying FTP Upload");
retries++;
goto TOP;
}
}
return JobResult.Success();
}
public static string CdnPath(string srcHash, string destHash)
{
return $"updates/{srcHash.FromBase64().ToHex()}_{destHash.FromBase64().ToHex()}";
}
}
}

25
Wabbajack.Common/Http/Client.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
@ -11,7 +12,7 @@ namespace Wabbajack.Common.Http
{
public List<(string, string)> Headers = new List<(string, string)>();
public List<Cookie> Cookies = new List<Cookie>();
public async Task<HttpResponseMessage> GetAsync(string url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseContentRead)
public async Task<HttpResponseMessage> GetAsync(string url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
foreach (var (k, v) in Headers)
@ -19,6 +20,23 @@ namespace Wabbajack.Common.Http
return await SendAsync(request, responseHeadersRead);
}
public async Task<HttpResponseMessage> PostAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead)
{
var request = new HttpRequestMessage(HttpMethod.Post, url) {Content = content};
foreach (var (k, v) in Headers)
request.Headers.Add(k, v);
return await SendAsync(request, responseHeadersRead);
}
public async Task<HttpResponseMessage> PutAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead)
{
var request = new HttpRequestMessage(HttpMethod.Put, url) {Content = content};
foreach (var (k, v) in Headers)
request.Headers.Add(k, v);
return await SendAsync(request, responseHeadersRead);
}
public async Task<string> GetStringAsync(string url)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
@ -32,12 +50,11 @@ namespace Wabbajack.Common.Http
private async Task<string> SendStringAsync(HttpRequestMessage request)
{
var result = await SendAsync(request);
using var result = await SendAsync(request);
return await result.Content.ReadAsStringAsync();
}
public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseContentRead)
public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead)
{
int retries = 0;
TOP:

6
Wabbajack.Common/Http/ClientFactory.cs
View File

@ -15,9 +15,9 @@ namespace Wabbajack.Common.Http
_socketsHandler = new SysHttp.SocketsHttpHandler
{
CookieContainer = Cookies,
MaxConnectionsPerServer = 8,
PooledConnectionLifetime = TimeSpan.FromSeconds(2),
PooledConnectionIdleTimeout = TimeSpan.FromSeconds(2)
MaxConnectionsPerServer = 20,
PooledConnectionLifetime = TimeSpan.FromMilliseconds(100),
PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(100)
};
Client = new SysHttp.HttpClient(_socketsHandler);
}

21
Wabbajack.Common/OctoDiff.cs
View File

@ -28,6 +28,21 @@ namespace Wabbajack.Common
return sigStream;
}
private static void CreateSignature(FileStream oldData, FileStream sigStream)
{
Utils.Status("Creating Patch Signature");
var signatureBuilder = new SignatureBuilder();
signatureBuilder.Build(oldData, new SignatureWriter(sigStream));
sigStream.Position = 0;
}
public static void Create(FileStream oldData, FileStream newData, FileStream signature, FileStream output)
{
CreateSignature(oldData, signature);
var db = new DeltaBuilder {ProgressReporter = reporter};
db.BuildDelta(newData, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
}
private class ProgressReporter : IProgressReporter
{
public void ReportProgress(string operation, long currentPosition, long total)
@ -44,5 +59,11 @@ namespace Wabbajack.Common
var deltaApplier = new DeltaApplier();
deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, reporter), output);
}
public static void Apply(FileStream input, FileStream patchStream, FileStream output)
{
var deltaApplier = new DeltaApplier();
deltaApplier.Apply(input, new BinaryDeltaReader(patchStream, reporter), output);
}
}
}

4
Wabbajack.Lib/AInstaller.cs
View File

@ -281,9 +281,7 @@ namespace Wabbajack.Lib
{
if (destination == null)
destination = Path.Combine(DownloadFolder, archive.Name);
await archive.State.Download(archive, destination);
destination.FileHashCached();
await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, destination);
}
catch (Exception ex)
{

22
Wabbajack.Lib/ClientAPI.cs Normal file
View File

@ -0,0 +1,22 @@
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.Lib
{
public class ClientAPI
{
public static Common.Http.Client GetClient()
{
var client = new Common.Http.Client();
client.Headers.Add((Consts.MetricsKeyHeader, Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)));
return client;
}
public static async Task<Archive> GetModUpgrade(string hash)
{
using var response = await GetClient()
.GetAsync($"https://{Consts.WabbajackCacheHostname}/alternative/{hash.FromBase64().ToHex()}");
return !response.IsSuccessStatusCode ? null : (await response.Content.ReadAsStringAsync()).FromJSONString<Archive>();
}
}
}

7
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -90,8 +90,8 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Download(Archive a, string destination)
{
var stream = await ResolveDownloadStream();
using (var file = File.Open(destination, FileMode.Create))
await using var stream = await ResolveDownloadStream();
await using (var file = File.Open(destination, FileMode.Create))
{
stream.CopyTo(file);
}
@ -125,7 +125,7 @@ namespace Wabbajack.Lib.Downloaders
var streamResult = await downloader.AuthedClient.GetAsync(url);
if (streamResult.StatusCode != HttpStatusCode.OK)
{
Utils.Error(new InvalidOperationException(), $"{downloader.SiteName} servers reported an error for file: {FileID}");
Utils.ErrorThrow(new InvalidOperationException(), $"{downloader.SiteName} servers reported an error for file: {FileID}");
}
var contentType = streamResult.Content.Headers.ContentType;
@ -141,6 +141,7 @@ namespace Wabbajack.Lib.Downloaders
Utils.Status($"Waiting for {secs} at the request of {downloader.SiteName}", Percent.FactoryPutInRange(x, secs));
await Task.Delay(1000);
}
streamResult.Dispose();
Utils.Status("Retrying download");
goto TOP;
}

40
Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs
View File

@ -134,7 +134,7 @@ namespace Wabbajack.Lib.Downloaders
foreach (var chunk in info.depot_list[0].file_list[0].chunk_list.OrderBy(c => c.index))
{
Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(chunk.index, max_chunks));
var got = await client.GetAsync(
using var got = await client.GetAsync(
$"https://content.cdp.bethesda.net/{collected.CDPProductId}/{collected.CDPPropertiesId}/{chunk.sha}");
var data = await got.Content.ReadAsByteArrayAsync();
if (collected.AESKey != null)
@ -196,36 +196,35 @@ namespace Wabbajack.Lib.Downloaders
return true;
}
private async Task<(HttpClient, CDPTree, CollectedBNetInfo)> ResolveDownloadInfo()
private async Task<(Common.Http.Client, CDPTree, CollectedBNetInfo)> ResolveDownloadInfo()
{
var info = new CollectedBNetInfo();
var login_info = Utils.FromEncryptedJson<BethesdaNetData>(DataName);
var client = new HttpClient();
var client = new Common.Http.Client();
client.BaseAddress = new Uri("https://api.bethesda.net");
client.DefaultRequestHeaders.Add("User-Agent", "bnet");
client.Headers.Add(("User-Agent", "bnet"));
foreach (var header in login_info.headers.Where(h => h.Key.ToLower().StartsWith("x-")))
client.DefaultRequestHeaders.Add(header.Key, header.Value);
client.Headers.Add((header.Key, header.Value));
var posted = await client.PostAsync("/beam/accounts/external_login",
var posted = await client.PostAsync("https://api.bethesda.net/beam/accounts/external_login",
new StringContent(login_info.body, Encoding.UTF8, "application/json"));
info.AccessToken = (await posted.Content.ReadAsStringAsync()).FromJSONString<BeamLoginResponse>().access_token;
client.DefaultRequestHeaders.Add("x-cdp-app", "UGC SDK");
client.DefaultRequestHeaders.Add("x-cdp-app-ver", "0.9.11314/debug");
client.DefaultRequestHeaders.Add("x-cdp-lib-ver", "0.9.11314/debug");
client.DefaultRequestHeaders.Add("x-cdp-platform","Win/32");
client.Headers.Add(("x-cdp-app", "UGC SDK"));
client.Headers.Add(("x-cdp-app-ver", "0.9.11314/debug"));
client.Headers.Add(("x-cdp-lib-ver", "0.9.11314/debug"));
client.Headers.Add(("x-cdp-platform","Win/32"));
posted = await client.PostAsync("cdp-user/auth",
posted = await client.PostAsync("https://api.bethesda.net/cdp-user/auth",
new StringContent("{\"access_token\": \"" + info.AccessToken + "\"}", Encoding.UTF8,
"application/json"));
info.CDPToken = (await posted.Content.ReadAsStringAsync()).FromJSONString<CDPLoginResponse>().token;
client.DefaultRequestHeaders.Add("X-Access-Token", info.AccessToken);
var got = await client.GetAsync($"mods/ugc-workshop/content/get?content_id={ContentId}");
client.Headers.Add(("X-Access-Token", info.AccessToken));
var got = await client.GetAsync($"https://api.bethesda.net/mods/ugc-workshop/content/get?content_id={ContentId}");
JObject data = JObject.Parse(await got.Content.ReadAsStringAsync());
var content = data["platform"]["response"]["content"];
@ -233,18 +232,21 @@ namespace Wabbajack.Lib.Downloaders
info.CDPBranchId = (int)content["cdp_branch_id"];
info.CDPProductId = (int)content["cdp_product_id"];
client.DefaultRequestHeaders.Add("Authorization", $"Token {info.CDPToken}");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Headers.Add(("Authorization", $"Token {info.CDPToken}"));
client.Headers.Add(("Accept", "application/json"));
got.Dispose();
got = await client.GetAsync(
$"/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/tree/.json");
$"https://api.bethesda.net/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/tree/.json");
var tree = (await got.Content.ReadAsStringAsync()).FromJSONString<CDPTree>();
got = await client.PostAsync($"/mods/ugc-content/add-subscription", new StringContent($"{{\"content_id\": \"{ContentId}\"}}", Encoding.UTF8, "application/json"));
got.Dispose();
got = await client.PostAsync($"https://api.bethesda.net/mods/ugc-content/add-subscription", new StringContent($"{{\"content_id\": \"{ContentId}\"}}", Encoding.UTF8, "application/json"));
got.Dispose();
got = await client.GetAsync(
$"/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/depots/.json");
$"https://api.bethesda.net/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/depots/.json");
var props_obj = JObject.Parse(await got.Content.ReadAsStringAsync()).Properties().First();
info.CDPPropertiesId = (int)props_obj.Value["properties_id"];

72
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders.UrlDownloaders;
@ -75,5 +76,76 @@ namespace Wabbajack.Lib.Downloaders
.Distinct()
.Do(t => Downloaders.First(d => d.GetType() == t).Prepare());
}
public static async Task<bool> DownloadWithPossibleUpgrade(Archive archive, string destination)
{
var success = await Download(archive, destination);
if (success)
{
await destination.FileHashCachedAsync();
return true;
}
Utils.Log($"Download failed, looking for upgrade");
var upgrade = await ClientAPI.GetModUpgrade(archive.Hash);
if (upgrade == null)
{
Utils.Log($"No upgrade found for {archive.Hash}");
return false;
}
Utils.Log($"Upgrading {archive.Hash}");
var upgradePath = Path.Combine(Path.GetDirectoryName(destination), "_Upgrade_" + archive.Name);
var upgradeResult = await Download(upgrade, upgradePath);
if (!upgradeResult) return false;
var patchName = $"{archive.Hash.FromBase64().ToHex()}_{upgrade.Hash.FromBase64().ToHex()}";
var patchPath = Path.Combine(Path.GetDirectoryName(destination), "_Patch_" + patchName);
var patchState = new Archive
{
Name = patchName,
State = new HTTPDownloader.State
{
Url = $"https://wabbajackcdn.b-cdn.net/updates/{patchName}"
}
};
var patchResult = await Download(patchState, patchPath);
if (!patchResult) return false;
Utils.Status($"Applying Upgrade to {archive.Hash}");
await using (var patchStream = File.OpenRead(patchPath))
await using (var srcStream = File.OpenRead(upgradePath))
await using (var destStream = File.Create(destination))
{
OctoDiff.Apply(srcStream, patchStream, destStream);
}
await destination.FileHashCachedAsync();
return true;
}
private static async Task<bool> Download(Archive archive, string destination)
{
try
{
var result = await archive.State.Download(archive, destination);
if (!result) return false;
if (archive.Hash == null) return true;
var hash = await destination.FileHashCachedAsync();
if (hash == archive.Hash) return true;
Utils.Log($"Hashed download is incorrect");
return false;
}
catch (Exception ex)
{
return false;
}
}
}
}

2
Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs
View File

@ -54,7 +54,7 @@ namespace Wabbajack.Lib.Downloaders
{
var initialURL = $"https://drive.google.com/uc?id={Id}&export=download";
var client = new Common.Http.Client();
var response = await client.GetAsync(initialURL);
using var response = await client.GetAsync(initialURL);
if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase);
var regex = new Regex("(?<=/uc\\?export=download&amp;confirm=).*(?=;id=)");

8
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -102,7 +102,7 @@ namespace Wabbajack.Lib.Downloaders
var bufferSize = 1024 * 32;
Utils.Status($"Starting Download {a?.Name ?? Url}", Percent.Zero);
var response = await client.GetAsync(Url, HttpCompletionOption.ResponseHeadersRead);
var response = await client.GetAsync(Url);
TOP:
if (!response.IsSuccessStatusCode)
@ -166,8 +166,8 @@ TOP:
var msg = new HttpRequestMessage(HttpMethod.Get, Url);
msg.Headers.Range = new RangeHeaderValue(totalRead, null);
response = await client.SendAsync(msg,
HttpCompletionOption.ResponseHeadersRead);
response.Dispose();
response = await client.SendAsync(msg);
goto TOP;
}
throw ex;
@ -185,7 +185,7 @@ TOP:
totalRead += read;
}
}
response.Dispose();
return true;
}
}

2
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -117,8 +117,8 @@ namespace Wabbajack.Lib.Downloaders
{
Utils.ErrorThrow(new UnconvertedError($"Aborting at the request of the user"));
}
_prepared = true;
}
_prepared = true;
}
}
}

7
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -119,11 +119,10 @@ namespace Wabbajack.Lib.FileUploader
return tcs.Task;
}
public static HttpClient GetAuthorizedClient()
public static Common.Http.Client GetAuthorizedClient()
{
var handler = new HttpClientHandler {MaxConnectionsPerServer = Consts.MaxConnectionsPerServer};
var client = new HttpClient(handler);
client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
var client = new Common.Http.Client();
client.Headers.Add(("X-API-KEY", GetAPIKey()));
return client;
}

4
Wabbajack.Lib/MO2Compiler.cs
View File

@ -357,8 +357,8 @@ namespace Wabbajack.Lib
await to_find.PMap(Queue, async f =>
{
var vf = VFS.Index.ByFullPath[f];
var client = new HttpClient();
var response =
var client = new Common.Http.Client();
using var response =
await client.GetAsync(
$"http://build.wabbajack.org/indexed_files/{vf.Hash.FromBase64().ToHex()}/meta.ini");

10
Wabbajack.Lib/ModListRegistry/ModListMetadata.cs
View File

@ -62,7 +62,7 @@ namespace Wabbajack.Lib.ModListRegistry
public static async Task<List<ModlistMetadata>> LoadFromGithub()
{
var client = new HttpClient();
var client = new Common.Http.Client();
Utils.Log("Loading ModLists from GitHub");
var metadataResult = client.GetStringAsync(Consts.ModlistMetadataURL);
var summaryResult = client.GetStringAsync(Consts.ModlistSummaryURL);
@ -110,6 +110,10 @@ namespace Wabbajack.Lib.ModListRegistry
{
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("machineURL")]
public string MachineURL { get; set; }
[JsonProperty("checked")]
public DateTime Checked { get; set; }
[JsonProperty("failed")]
@ -117,9 +121,9 @@ namespace Wabbajack.Lib.ModListRegistry
[JsonProperty("passed")]
public int Passed { get; set; }
[JsonProperty("link")]
public string Link => $"/lists/status/{Name}.json";
public string Link => $"/lists/status/{MachineURL}.json";
[JsonProperty("report")]
public string Report => $"/lists/status/{Name}.html";
public string Report => $"/lists/status/{MachineURL}.html";
[JsonProperty("has_failures")]
public bool HasFailures => Failed > 0;
}

2
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -224,7 +224,7 @@ namespace Wabbajack.Lib.NexusApi
TOP:
try
{
using var response = await HttpClient.GetAsync(url, HttpCompletionOption.ResponseContentRead);
using var response = await HttpClient.GetAsync(url);
UpdateRemaining(response);
if (!response.IsSuccessStatusCode)
{

11
Wabbajack.Lib/Validation/ValidateModlist.cs
View File

@ -38,16 +38,11 @@ namespace Wabbajack.Lib.Validation
public async Task LoadListsFromGithub()
{
var client = new HttpClient();
Utils.Log("Loading Nexus mod permissions");
using (var result = await client.GetStreamAsync(Consts.ModPermissionsURL))
{
AuthorPermissions = result.FromYaml<Dictionary<string, Author>>();
Utils.Log($"Loaded permissions for {AuthorPermissions.Count} authors");
}
var client = new Common.Http.Client();
Utils.Log("Loading server whitelist");
using (var result = await client.GetStreamAsync(Consts.ServerWhitelistURL))
using (var response = await client.GetAsync(Consts.ServerWhitelistURL))
using (var result = await response.Content.ReadAsStreamAsync())
{
ServerWhitelist = result.FromYaml<ServerWhitelist>();
Utils.Log($"Loaded permissions for {ServerWhitelist.AllowedPrefixes.Count} servers and {ServerWhitelist.GoogleIDs.Count} Google Drive files");

42
Wabbajack.Test/DownloaderTests.cs
View File

@ -417,26 +417,6 @@ namespace Wabbajack.Test
Consts.TestMode = true;
}
[TestMethod]
public async Task AFKModsDownloadTest()
{
await DownloadDispatcher.GetInstance<AFKModsDownloader>().Prepare();
const string ini = "[General]\n" +
"directURL=https://www.afkmods.com/index.php?/files/file/2120-skyrim-save-system-overhaul/&do=download&r=20112&confirm=1&t=1&csrfKey=840a4a373144097693171a79df77d521";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.IsNotNull(state);
var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive { Name = "AFKMods Test.zip" }, filename);
}
[TestMethod]
public async Task BethesdaNetDownload()
{
@ -519,6 +499,28 @@ namespace Wabbajack.Test
}
[TestMethod]
public async Task TestUpgrading()
{
using var folder = new TempFolder();
var dest = Path.Combine(folder.Dir.FullName, "Cori.7z");
var archive = new Archive
{
Name = "Cori.7z",
Hash = "gCRVrvzDNH0=",
State = new NexusDownloader.State
{
GameName = Game.SkyrimSpecialEdition.MetaData().NexusName,
ModID = "24808",
FileID = "123501"
}
};
Assert.IsTrue(await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, dest));
Assert.AreEqual("gCRVrvzDNH0=", await dest.FileHashCachedAsync());
}
class TestInstaller : AInstaller
{
public TestInstaller(string archive, ModList modList, string outputFolder, string downloadFolder, SystemParameters parameters) : base(archive, modList, outputFolder, downloadFolder, parameters)