Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add size validation to HTTPDownloader, to catch stealth updates. Check whitelist during list validation.

Browse Source
This commit is contained in:
Timothy Baldridge 2020-01-13 15:55:55 -07:00
parent caa2ec8dd4
commit 14f7f45a77
18 changed files with 62 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.BuildServer/Controllers/ListValidation.cs
View File

@ -93,7 +93,7 @@ namespace Wabbajack.BuildServer.Controllers
var response = HandleGetListTemplate(new
{
lst,
ago = (DateTime.Now - lst.Checked).TotalMinutes,
ago = (DateTime.UtcNow - lst.Checked).TotalMinutes,
failed = lst.Archives.Where(a => a.IsFailing).ToList(),
passed = lst.Archives.Where(a => !a.IsFailing).ToList()
});

11
Wabbajack.BuildServer/JobManager.cs
View File

@ -26,7 +26,6 @@ namespace Wabbajack.BuildServer
public void StartJobRunners()
{
return;
for (var idx = 0; idx < 2; idx++)
{
Task.Run(async () =>
@ -71,14 +70,12 @@ namespace Wabbajack.BuildServer
Utils.LogMessages.Subscribe(msg => Logger.Log(LogLevel.Information, msg.ToString()));
while (true)
{
/*
await KillOrphanedJobs();
await ScheduledJob<GetNexusUpdatesJob>(TimeSpan.FromHours(2), Job.JobPriority.High);
//await ScheduledJob<GetNexusUpdatesJob>(TimeSpan.FromHours(2), Job.JobPriority.High);
await ScheduledJob<UpdateModLists>(TimeSpan.FromMinutes(30), Job.JobPriority.High);
await ScheduledJob<EnqueueAllArchives>(TimeSpan.FromHours(2), Job.JobPriority.Low);
await ScheduledJob<EnqueueAllGameFiles>(TimeSpan.FromHours(24), Job.JobPriority.High);
await ScheduledJob<EnqueueRecentFiles>(TimeSpan.FromHours(6), Job.JobPriority.Low);
*/
//await ScheduledJob<EnqueueAllArchives>(TimeSpan.FromHours(2), Job.JobPriority.Low);
//await ScheduledJob<EnqueueAllGameFiles>(TimeSpan.FromHours(24), Job.JobPriority.High);
//await ScheduledJob<EnqueueRecentFiles>(TimeSpan.FromHours(6), Job.JobPriority.Low);
await Task.Delay(10000);
}
}

21
Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs
View File

@ -7,6 +7,8 @@ using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.Validation;
using File = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.BuildServer.Models.Jobs
{
@ -20,11 +22,15 @@ namespace Wabbajack.BuildServer.Models.Jobs
using (var queue = new WorkQueue())
{
foreach (var list in modlists.Skip(1).Take(1))
var whitelists = new ValidateModlist(queue);
await whitelists.LoadListsFromGithub();
foreach (var list in modlists)
{
try
{
await ValidateList(db, list, queue);
await ValidateList(db, list, queue, whitelists);
}
catch (Exception ex)
{
@ -36,7 +42,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue)
private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
{
var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
@ -65,14 +71,14 @@ namespace Wabbajack.BuildServer.Models.Jobs
DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
var validated = (await installer.Archives
.PMap(queue, async archive =>
{
Utils.Log($"Validating: {archive.Name}");
bool is_failed;
try
{
is_failed = !(await archive.State.Verify());
is_failed = !(await archive.State.Verify(archive)) || !archive.State.IsWhitelisted(whitelists.ServerWhitelist);
}
catch (Exception)
{
@ -105,7 +111,12 @@ namespace Wabbajack.BuildServer.Models.Jobs
DetailedStatus = status,
Metadata = list
};
Utils.Log(
$"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed");
await ModListStatus.Update(db, dto);
Utils.Log(
$"Done updating {dto.Summary.Name}");
}
}
}

2
Wabbajack.BuildServer/Models/ModListStatus.cs
View File

@ -47,7 +47,7 @@ namespace Wabbajack.BuildServer.Models
public class DetailedStatus
{
public string Name { get; set; }
public DateTime Checked { get; set; } = DateTime.Now;
public DateTime Checked { get; set; } = DateTime.UtcNow;
public List<DetailedStatusItem> Archives { get; set; }
public DownloadMetadata DownloadMetaData { get; set; }
public bool HasFailures { get; set; }

2
Wabbajack.Lib/ACompiler.cs
View File

@ -239,7 +239,7 @@ namespace Wabbajack.Lib
result.Meta = archive.Meta;
result.Size = archive.File.Size;
if (result.State != null && !await result.State.Verify())
if (result.State != null && !await result.State.Verify(result))
Error(
$"Unable to resolve link for {archive.Name}. If this is hosted on the Nexus the file may have been removed.");

2
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -75,7 +75,7 @@ namespace Wabbajack.Lib.Downloaders
/// Returns true if this link is still valid
/// </summary>
/// <returns></returns>
public abstract Task<bool> Verify();
public abstract Task<bool> Verify(Archive archive);
public abstract IDownloader GetDownloader();

2
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -129,7 +129,7 @@ namespace Wabbajack.Lib.Downloaders
public int CurrentTime { get; set; }
}
public override async Task<bool> Verify()
public override async Task<bool> Verify(Archive a)
{
var stream = await ResolveDownloadStream();
if (stream == null)

2
Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
View File

@ -72,7 +72,7 @@ namespace Wabbajack.Lib.Downloaders
}
}
public override async Task<bool> Verify()
public override async Task<bool> Verify(Archive a)
{
return File.Exists(SourcePath) && SourcePath.FileHashCached() == Hash;
}

4
Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs
View File

@ -64,10 +64,10 @@ namespace Wabbajack.Lib.Downloaders
return httpState;
}
public override async Task<bool> Verify()
public override async Task<bool> Verify(Archive a)
{
var state = await ToHttpState();
return await state.Verify();
return await state.Verify(a);
}
public override IDownloader GetDownloader()

19
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -119,12 +119,23 @@ namespace Wabbajack.Lib.Downloaders
return false;
}
if (!download)
return true;
var headerVar = a.Size == 0 ? "1" : a.Size.ToString();
long header_content_size = 0;
if (response.Content.Headers.Contains("Content-Length"))
{
headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault();
if (headerVar != null)
long.TryParse(headerVar, out header_content_size);
}
if (!download)
{
if (a.Size != 0 && header_content_size != 0)
return a.Size == header_content_size;
return true;
}
var supportsResume = response.Headers.AcceptRanges.FirstOrDefault(f => f == "bytes") != null;
var contentSize = headerVar != null ? long.Parse(headerVar) : 1;
@ -179,9 +190,9 @@ namespace Wabbajack.Lib.Downloaders
}
}
public override async Task<bool> Verify()
public override async Task<bool> Verify(Archive a)
{
return await DoDownload(new Archive {Name = ""}, "", false);
return await DoDownload(a, "", false);
}
public override IDownloader GetDownloader()

2
Wabbajack.Lib/Downloaders/MEGADownloader.cs
View File

@ -37,7 +37,7 @@ namespace Wabbajack.Lib.Downloaders
client.DownloadFile(fileLink, destination);
}
public override async Task<bool> Verify()
public override async Task<bool> Verify(Archive a)
{
var client = new MegaApiClient();
Utils.Status("Logging into MEGA (as anonymous)");

2
Wabbajack.Lib/Downloaders/ManualDownloader.cs
View File

@ -111,7 +111,7 @@ namespace Wabbajack.Lib.Downloaders
}
}
public override async Task<bool> Verify()
public override async Task<bool> Verify(Archive a)
{
return true;
}

2
Wabbajack.Lib/Downloaders/MediaFireDownloader.cs
View File

@ -37,7 +37,7 @@ namespace Wabbajack.Lib.Downloaders
await result.Download(a, destination);
}
public override async Task<bool> Verify()
public override async Task<bool> Verify(Archive a)
{
return await Resolve() != null;
}

2
Wabbajack.Lib/Downloaders/ModDBDownloader.cs
View File

@ -92,7 +92,7 @@ namespace Wabbajack.Lib.Downloaders
return mirrors.Select(d => d.Link).ToArray();
}
public override async Task<bool> Verify()
public override async Task<bool> Verify(Archive a)
{
await GetDownloadUrls();
return true;

2
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -158,7 +158,7 @@ namespace Wabbajack.Lib.Downloaders
}
public override async Task<bool> Verify()
public override async Task<bool> Verify(Archive a)
{
try
{

2
Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs
View File

@ -79,7 +79,7 @@ namespace Wabbajack.Lib.Downloaders
}
}
public override async Task<bool> Verify()
public override async Task<bool> Verify(Archive a)
{
//TODO: find a way to verify steam workshop items
throw new NotImplementedException();

22
Wabbajack.Test/DownloaderTests.cs
View File

@ -64,7 +64,7 @@ namespace Wabbajack.Test
var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify());
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List<string>{"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k" } }));
@ -94,7 +94,7 @@ namespace Wabbajack.Test
((HTTPDownloader.State)url_state).Url);
var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify());
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?" } }));
@ -124,7 +124,7 @@ namespace Wabbajack.Test
((GoogleDriveDownloader.State)url_state).Id);
var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify());
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List<string> { "1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_" } }));
@ -153,7 +153,7 @@ namespace Wabbajack.Test
((HTTPDownloader.State)url_state).Url);
var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify());
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "http://build.wabbajack.org/" } }));
@ -177,7 +177,7 @@ namespace Wabbajack.Test
Assert.IsNotNull(state);
var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify());
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "http://build.wabbajack.org/" } }));
@ -238,9 +238,9 @@ namespace Wabbajack.Test
var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify());
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
// Exercise the cache code
Assert.IsTrue(await converted.Verify());
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> () }));
@ -273,7 +273,7 @@ namespace Wabbajack.Test
((ModDBDownloader.State)url_state).Url);
var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify());
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
@ -299,7 +299,7 @@ namespace Wabbajack.Test
((HTTPDownloader.State)url_state).Url);
*/
var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify());
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
@ -327,7 +327,7 @@ namespace Wabbajack.Test
((HTTPDownloader.State)url_state).Url);
*/
var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify());
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
@ -354,7 +354,7 @@ namespace Wabbajack.Test
Assert.IsNotNull(state);
var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify());
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));

3
Wabbajack.Test/ModlistMetadataTests.cs
View File

@ -1,6 +1,7 @@
using System.Linq;
using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
@ -25,7 +26,7 @@ namespace Wabbajack.Test
{
var logo_state = DownloadDispatcher.ResolveArchive(modlist.ImageUri);
Assert.IsNotNull(logo_state);
Assert.IsTrue(await logo_state.Verify(), $"{modlist.ImageUri} is not valid");
Assert.IsTrue(await logo_state.Verify(new Archive{Size = 0}), $"{modlist.ImageUri} is not valid");
}
}
}