Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #384 from wabbajack-tools/enqueue-recent-files

Browse Source 
Enqueue recent files
This commit is contained in:
Timothy Baldridge 2020-01-14 15:21:09 -07:00 committed by GitHub
commit 853c6ad29a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 289 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Wabbajack.BuildServer/Controllers/Jobs.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Generic; using System;
using System.Collections.Generic;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
@ -26,5 +27,15 @@ namespace Wabbajack.BuildServer.Controllers
.OrderByDescending(j => j.Priority) .OrderByDescending(j => j.Priority)
.ToListAsync(); .ToListAsync();
} }
[HttpGet]
[Route("enqueue_job/{JobName}")]
public async Task<string> EnqueueJob(string JobName)
{
var jobtype = AJobPayload.NameToType[JobName];
var job = new Job{Priority = Job.JobPriority.High, Payload = (AJobPayload)jobtype.GetConstructor(new Type[0]).Invoke(new object?[0])};
await Db.Jobs.InsertOneAsync(job);
return job.Id;
}
} }
} }

87
Wabbajack.BuildServer/Controllers/ListValidation.cs
View File

@ -1,9 +1,15 @@
using System.Collections.Generic; using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.ServiceModel.Syndication;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.AspNetCore.Http.Extensions;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MongoDB.Driver; using MongoDB.Driver;
using MongoDB.Driver.Linq; using MongoDB.Driver.Linq;
using Nettle;
using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models;
using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.ModListRegistry;
@ -16,12 +22,89 @@ namespace Wabbajack.BuildServer.Controllers
public ListValidation(ILogger<ListValidation> logger, DBContext db) : base(logger, db) public ListValidation(ILogger<ListValidation> logger, DBContext db) : base(logger, db)
{ {
} }
[HttpGet] [HttpGet]
[Route("status.json")] [Route("status.json")]
public async Task<IList<ModlistSummary>> HandleGetLists() public async Task<IList<ModlistSummary>> HandleGetLists()
{ {
return await Db.ModListStatus.AsQueryable().Select(m => m.Summary).ToListAsync(); return await Db.ModListStatus.AsQueryable().Select(m => m.Summary).ToListAsync();
} }
private static readonly Func<object, string> HandleGetRssFeedTemplate = NettleEngine.GetCompiler().Compile(@"
<?xml version=""1.0""?>
<rss version=""2.0"">
<channel>
<title>{{lst.Name}} - Broken Mods</title>
<link>http://build.wabbajack.org/status/{{lst.Name}}.html</link>
<description>These are mods that are broken and need updating</description>
{{ each $.failed }}
<item>
<title>{{$.Archive.Name}}</title>
<link>{{$.Archive.Name}}</link>
</item>
{{/each}}
</channel>
</rss>
");
[HttpGet]
[Route("status/{Name}/broken.rss")]
public async Task<ContentResult> HandleGetRSSFeed(string Name)
{
var lst = (await ModListStatus.ByName(Db, Name)).DetailedStatus;
var response = HandleGetRssFeedTemplate(new
{
lst,
failed = lst.Archives.Where(a => a.IsFailing).ToList(),
passed = lst.Archives.Where(a => !a.IsFailing).ToList()
});
return new ContentResult
{
ContentType = "application/rss+xml",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
<html><body>
<h2>{{lst.Name}} - {{lst.Checked}} - {{ago}}min ago</h2>
<h3>Failed ({{failed.Count}}):</h3>
<ul>
{{each $.failed }}
<li>{{$.Archive.Name}}</li>
{{/each}}
</ul>
<h3>Passed ({{passed.Count}}):</h3>
<ul>
{{each $.passed }}
<li>{{$.Archive.Name}}</li>
{{/each}}
</ul>
</body></html>
");
[HttpGet]
[Route("status/{Name}.html")]
public async Task<ContentResult> HandleGetListHtml(string Name)
{
var lst = (await ModListStatus.ByName(Db, Name)).DetailedStatus;
var response = HandleGetListTemplate(new
{
lst,
ago = (DateTime.UtcNow - lst.Checked).TotalMinutes,
failed = lst.Archives.Where(a => a.IsFailing).ToList(),
passed = lst.Archives.Where(a => !a.IsFailing).ToList()
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
} }
} }

1
Wabbajack.BuildServer/JobManager.cs
View File

@ -75,6 +75,7 @@ namespace Wabbajack.BuildServer
await ScheduledJob<UpdateModLists>(TimeSpan.FromMinutes(30), Job.JobPriority.High); await ScheduledJob<UpdateModLists>(TimeSpan.FromMinutes(30), Job.JobPriority.High);
await ScheduledJob<EnqueueAllArchives>(TimeSpan.FromHours(2), Job.JobPriority.Low); await ScheduledJob<EnqueueAllArchives>(TimeSpan.FromHours(2), Job.JobPriority.Low);
await ScheduledJob<EnqueueAllGameFiles>(TimeSpan.FromHours(24), Job.JobPriority.High); await ScheduledJob<EnqueueAllGameFiles>(TimeSpan.FromHours(24), Job.JobPriority.High);
await ScheduledJob<EnqueueRecentFiles>(TimeSpan.FromHours(6), Job.JobPriority.Low);
await Task.Delay(10000); await Task.Delay(10000);
} }
} }

1
Wabbajack.BuildServer/Models/DBContext.cs
View File

@ -26,6 +26,7 @@ namespace Wabbajack.BuildServer.Models
public IMongoCollection<DownloadState> DownloadStates => Client.GetCollection<DownloadState>(_settings.Collections["DownloadStates"]); public IMongoCollection<DownloadState> DownloadStates => Client.GetCollection<DownloadState>(_settings.Collections["DownloadStates"]);
public IMongoCollection<Metric> Metrics => Client.GetCollection<Metric>(_settings.Collections["Metrics"]); public IMongoCollection<Metric> Metrics => Client.GetCollection<Metric>(_settings.Collections["Metrics"]);
public IMongoCollection<IndexedFile> IndexedFiles => Client.GetCollection<IndexedFile>(_settings.Collections["IndexedFiles"]); public IMongoCollection<IndexedFile> IndexedFiles => Client.GetCollection<IndexedFile>(_settings.Collections["IndexedFiles"]);
public IMongoCollection<NexusCacheData<List<NexusUpdateEntry>>> NexusUpdates => Client.GetCollection<NexusCacheData<List<NexusUpdateEntry>>>(_settings.Collections["NexusUpdates"]);
public IMongoCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles => public IMongoCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles =>
Client.GetCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>>( Client.GetCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>>(

3
Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs
View File

@ -16,7 +16,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue
typeof(GetNexusUpdatesJob), typeof(GetNexusUpdatesJob),
typeof(UpdateModLists), typeof(UpdateModLists),
typeof(EnqueueAllArchives), typeof(EnqueueAllArchives),
typeof(EnqueueAllGameFiles) typeof(EnqueueAllGameFiles),
typeof(EnqueueRecentFiles)
}; };
public static Dictionary<Type, string> TypeToName { get; set; } public static Dictionary<Type, string> TypeToName { get; set; }
public static Dictionary<string, Type> NameToType { get; set; } public static Dictionary<string, Type> NameToType { get; set; }

1
Wabbajack.BuildServer/Models/JobQueue/Job.cs
View File

@ -6,6 +6,7 @@ using System.Threading.Tasks;
using MongoDB.Bson; using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes; using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver; using MongoDB.Driver;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer.Models.JobQueue namespace Wabbajack.BuildServer.Models.JobQueue
{ {

84
Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs Normal file
View File

@ -0,0 +1,84 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using MongoDB.Driver;
using MongoDB.Driver.Core.Authentication;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class EnqueueRecentFiles : AJobPayload
{
public override string Description => "Enqueue the past days worth of mods for indexing";
private static HashSet<Game> GamesToScan = new HashSet<Game>
{
Game.Fallout3, Game.Fallout4, Game.Skyrim, Game.SkyrimSpecialEdition, Game.SkyrimVR, Game.FalloutNewVegas, Game.Oblivion
};
public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
{
using (var queue = new WorkQueue())
{
var updates = await db.NexusUpdates.AsQueryable().ToListAsync();
var mods = updates
.Where(list => GamesToScan.Contains(GameRegistry.GetByNexusName(list.Game).Game))
.SelectMany(list =>
list.Data.Where(mod => DateTime.UtcNow - mod.LatestFileUpdate.AsUnixTime() < TimeSpan.FromDays(1))
.Select(mod => (list.Game, mod.ModId)));
var mod_files = (await mods.PMap(queue, async mod =>
{
var client = await NexusApiClient.Get();
try
{
var files = await client.GetModFiles(GameRegistry.GetByNexusName(mod.Game).Game,
(int)mod.ModId);
return (mod.Game, mod.ModId, files.files);
}
catch (Exception)
{
return default;
}
})).Where(t => t.Game != null).ToList();
var archives =
mod_files.SelectMany(mod => mod.files.Select(file => (mod.Game, mod.ModId, File:file)).Where(f => !string.IsNullOrEmpty(f.File.category_name) ))
.Select(tuple =>
{
var state = new NexusDownloader.State
{
GameName = tuple.Game, ModID = tuple.ModId.ToString(), FileID = tuple.File.file_id.ToString()
};
return new Archive {State = state, Name = tuple.File.file_name};
}).ToList();
Utils.Log($"Found {archives.Count} archives from recent Nexus updates to index");
var searching = archives.Select(a => a.State.PrimaryKeyString).Distinct().ToArray();
Utils.Log($"Looking for missing states");
var knownArchives = (await db.DownloadStates.AsQueryable().Where(s => searching.Contains(s.Key))
.Select(d => d.Key).ToListAsync()).ToDictionary(a => a);
Utils.Log($"Found {knownArchives.Count} pre-existing archives");
var missing = archives.Where(a => !knownArchives.ContainsKey(a.State.PrimaryKeyString))
.DistinctBy(d => d.State.PrimaryKeyString)
.ToList();
Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs");
var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}, Priority = Job.JobPriority.Low});
Utils.Log($"Writing jobs to the DB");
await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions {IsOrdered = false});
Utils.Log($"Done adding archives for Nexus Updates");
return JobResult.Success();
}
}
}
}

36
Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs
View File

@ -5,6 +5,7 @@ using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.NexusApi;
using MongoDB.Driver; using MongoDB.Driver;
using Newtonsoft.Json;
namespace Wabbajack.BuildServer.Models.Jobs namespace Wabbajack.BuildServer.Models.Jobs
@ -21,9 +22,17 @@ namespace Wabbajack.BuildServer.Models.Jobs
.Where(game => game.NexusName != null) .Where(game => game.NexusName != null)
.Select(async game => .Select(async game =>
{ {
return (game, var mods = await api.Get<List<NexusUpdateEntry>>(
mods: await api.Get<List<UpdatedMod>>( $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m");
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"));
var entry = new NexusCacheData<List<NexusUpdateEntry>>();
entry.Game = game.NexusName;
entry.Path = $"/v1/games/{game.NexusName}/mods/updated.json?period=1m";
entry.Data = mods;
await entry.Upsert(db.NexusUpdates);
return (game, mods);
}) })
.Select(async rTask => .Select(async rTask =>
{ {
@ -40,14 +49,16 @@ namespace Wabbajack.BuildServer.Models.Jobs
Utils.Log($"Found {purge.Count} updated mods in the last month"); Utils.Log($"Found {purge.Count} updated mods in the last month");
using (var queue = new WorkQueue()) using (var queue = new WorkQueue())
{ {
var collected = await purge.Select(d => var collected = purge.Select(d =>
{ {
var a = d.mod.latest_file_update.AsUnixTime(); var a = d.mod.LatestFileUpdate.AsUnixTime();
// Mod activity could hide files // Mod activity could hide files
var b = d.mod.latest_mod_activity.AsUnixTime(); var b = d.mod.LastestModActivity.AsUnixTime();
return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.mod_id.ToString()}; return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.ModId.ToString()};
}).PMap(queue, async t => });
var purged = await collected.PMap(queue, async t =>
{ {
var resultA = await db.NexusModInfos.DeleteManyAsync(f => var resultA = await db.NexusModInfos.DeleteManyAsync(f =>
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
@ -59,17 +70,12 @@ namespace Wabbajack.BuildServer.Models.Jobs
return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount; return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount;
}); });
Utils.Log($"Purged {collected.Sum()} cache entries"); Utils.Log($"Purged {purged.Sum()} cache entries");
} }
return JobResult.Success(); return JobResult.Success();
} }
class UpdatedMod
{
public long mod_id;
public long latest_file_update;
public long latest_mod_activity;
}
} }
} }

19
Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs
View File

@ -7,6 +7,8 @@ using Wabbajack.Common;
using Wabbajack.Lib; using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.Validation;
using File = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.BuildServer.Models.Jobs namespace Wabbajack.BuildServer.Models.Jobs
{ {
@ -20,11 +22,15 @@ namespace Wabbajack.BuildServer.Models.Jobs
using (var queue = new WorkQueue()) using (var queue = new WorkQueue())
{ {
var whitelists = new ValidateModlist(queue);
await whitelists.LoadListsFromGithub();
foreach (var list in modlists) foreach (var list in modlists)
{ {
try try
{ {
await ValidateList(db, list, queue); await ValidateList(db, list, queue, whitelists);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -36,7 +42,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success(); return JobResult.Success();
} }
private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue) private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
{ {
var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL); var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
@ -65,14 +71,14 @@ namespace Wabbajack.BuildServer.Models.Jobs
DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State)); DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
var validated = (await installer.Archives var validated = (await installer.Archives
.PMap(queue, async archive => .PMap(queue, async archive =>
{ {
Utils.Log($"Validating: {archive.Name}");
bool is_failed; bool is_failed;
try try
{ {
is_failed = !(await archive.State.Verify()); is_failed = !(await archive.State.Verify(archive)) || !archive.State.IsWhitelisted(whitelists.ServerWhitelist);
} }
catch (Exception) catch (Exception)
{ {
@ -105,7 +111,12 @@ namespace Wabbajack.BuildServer.Models.Jobs
DetailedStatus = status, DetailedStatus = status,
Metadata = list Metadata = list
}; };
Utils.Log(
$"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed");
await ModListStatus.Update(db, dto); await ModListStatus.Update(db, dto);
Utils.Log(
$"Done updating {dto.Summary.Name}");
} }
} }
} }

2
Wabbajack.BuildServer/Models/ModListStatus.cs
View File

@ -47,7 +47,7 @@ namespace Wabbajack.BuildServer.Models
public class DetailedStatus public class DetailedStatus
{ {
public string Name { get; set; } public string Name { get; set; }
public DateTime Checked { get; set; } = DateTime.Now; public DateTime Checked { get; set; } = DateTime.UtcNow;
public List<DetailedStatusItem> Archives { get; set; } public List<DetailedStatusItem> Archives { get; set; }
public DownloadMetadata DownloadMetaData { get; set; } public DownloadMetadata DownloadMetaData { get; set; }
public bool HasFailures { get; set; } public bool HasFailures { get; set; }

8
Wabbajack.BuildServer/Models/NexusCacheData.cs
View File

@ -1,5 +1,7 @@
using System; using System;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes; using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
namespace Wabbajack.BuildServer.Models namespace Wabbajack.BuildServer.Models
{ {
@ -9,6 +11,8 @@ namespace Wabbajack.BuildServer.Models
public string Path { get; set; } public string Path { get; set; }
public T Data { get; set; } public T Data { get; set; }
public string Game { get; set; } public string Game { get; set; }
[BsonIgnoreIfNull]
public string ModId { get; set; } public string ModId { get; set; }
public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow; public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow;
@ -16,5 +20,9 @@ namespace Wabbajack.BuildServer.Models
[BsonIgnoreIfNull] [BsonIgnoreIfNull]
public string FileId { get; set; } public string FileId { get; set; }
public async Task Upsert(IMongoCollection<NexusCacheData<T>> coll)
{
await coll.FindOneAndReplaceAsync<NexusCacheData<T>>(s => s.Path == Path, this, new FindOneAndReplaceOptions<NexusCacheData<T>> {IsUpsert = true});
}
} }
} }

16
Wabbajack.BuildServer/Models/NexusUpdateEntry.cs Normal file
View File

@ -0,0 +1,16 @@
using System;
using MongoDB.Bson.Serialization.Attributes;
using Newtonsoft.Json;
namespace Wabbajack.BuildServer.Models
{
public class NexusUpdateEntry
{
[JsonProperty("mod_id")]
public long ModId { get; set; }
[JsonProperty("latest_file_update")]
public long LatestFileUpdate { get; set; }
[JsonProperty("latest_mod_activity")]
public long LastestModActivity { get; set; }
}
}

1
Wabbajack.BuildServer/Wabbajack.BuildServer.csproj
View File

@ -18,6 +18,7 @@
<PackageReference Include="Microsoft.OpenApi" Version="1.1.4" /> <PackageReference Include="Microsoft.OpenApi" Version="1.1.4" />
<PackageReference Include="MongoDB.Driver" Version="2.10.0" /> <PackageReference Include="MongoDB.Driver" Version="2.10.0" />
<PackageReference Include="MongoDB.Driver.Core" Version="2.10.0" /> <PackageReference Include="MongoDB.Driver.Core" Version="2.10.0" />
<PackageReference Include="Nettle" Version="1.3.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.0.0-rc5" /> <PackageReference Include="Swashbuckle.AspNetCore" Version="5.0.0-rc5" />
</ItemGroup> </ItemGroup>

1
Wabbajack.BuildServer/appsettings.json
View File

@ -19,6 +19,7 @@
"NexusModInfos": "nexus_mod_infos", "NexusModInfos": "nexus_mod_infos",
"NexusModFiles": "nexus_mod_files", "NexusModFiles": "nexus_mod_files",
"NexusFileInfos": "nexus_file_infos", "NexusFileInfos": "nexus_file_infos",
"NexusUpdates": "nexus_updates",
"ModListStatus": "mod_lists", "ModListStatus": "mod_lists",
"JobQueue": "job_queue", "JobQueue": "job_queue",
"DownloadStates": "download_states", "DownloadStates": "download_states",

2
Wabbajack.Lib/ACompiler.cs
View File

@ -239,7 +239,7 @@ namespace Wabbajack.Lib
result.Meta = archive.Meta; result.Meta = archive.Meta;
result.Size = archive.File.Size; result.Size = archive.File.Size;
if (result.State != null && !await result.State.Verify()) if (result.State != null && !await result.State.Verify(result))
Error( Error(
$"Unable to resolve link for {archive.Name}. If this is hosted on the Nexus the file may have been removed."); $"Unable to resolve link for {archive.Name}. If this is hosted on the Nexus the file may have been removed.");

2
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -75,7 +75,7 @@ namespace Wabbajack.Lib.Downloaders
/// Returns true if this link is still valid /// Returns true if this link is still valid
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
public abstract Task<bool> Verify(); public abstract Task<bool> Verify(Archive archive);
public abstract IDownloader GetDownloader(); public abstract IDownloader GetDownloader();

2
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -129,7 +129,7 @@ namespace Wabbajack.Lib.Downloaders
public int CurrentTime { get; set; } public int CurrentTime { get; set; }
} }
public override async Task<bool> Verify() public override async Task<bool> Verify(Archive a)
{ {
var stream = await ResolveDownloadStream(); var stream = await ResolveDownloadStream();
if (stream == null) if (stream == null)

2
Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
View File

@ -72,7 +72,7 @@ namespace Wabbajack.Lib.Downloaders
} }
} }
public override async Task<bool> Verify() public override async Task<bool> Verify(Archive a)
{ {
return File.Exists(SourcePath) && SourcePath.FileHashCached() == Hash; return File.Exists(SourcePath) && SourcePath.FileHashCached() == Hash;
} }

4
Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs
View File

@ -64,10 +64,10 @@ namespace Wabbajack.Lib.Downloaders
return httpState; return httpState;
} }
public override async Task<bool> Verify() public override async Task<bool> Verify(Archive a)
{ {
var state = await ToHttpState(); var state = await ToHttpState();
return await state.Verify(); return await state.Verify(a);
} }
public override IDownloader GetDownloader() public override IDownloader GetDownloader()

19
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -119,12 +119,23 @@ namespace Wabbajack.Lib.Downloaders
return false; return false;
} }
if (!download)
return true;
var headerVar = a.Size == 0 ? "1" : a.Size.ToString(); var headerVar = a.Size == 0 ? "1" : a.Size.ToString();
long header_content_size = 0;
if (response.Content.Headers.Contains("Content-Length")) if (response.Content.Headers.Contains("Content-Length"))
{
headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault(); headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault();
if (headerVar != null)
long.TryParse(headerVar, out header_content_size);
}
if (!download)
{
if (a.Size != 0 && header_content_size != 0)
return a.Size == header_content_size;
return true;
}
var supportsResume = response.Headers.AcceptRanges.FirstOrDefault(f => f == "bytes") != null; var supportsResume = response.Headers.AcceptRanges.FirstOrDefault(f => f == "bytes") != null;
var contentSize = headerVar != null ? long.Parse(headerVar) : 1; var contentSize = headerVar != null ? long.Parse(headerVar) : 1;
@ -179,9 +190,9 @@ namespace Wabbajack.Lib.Downloaders
} }
} }
public override async Task<bool> Verify() public override async Task<bool> Verify(Archive a)
{ {
return await DoDownload(new Archive {Name = ""}, "", false); return await DoDownload(a, "", false);
} }
public override IDownloader GetDownloader() public override IDownloader GetDownloader()

2
Wabbajack.Lib/Downloaders/MEGADownloader.cs
View File

@ -37,7 +37,7 @@ namespace Wabbajack.Lib.Downloaders
client.DownloadFile(fileLink, destination); client.DownloadFile(fileLink, destination);
} }
public override async Task<bool> Verify() public override async Task<bool> Verify(Archive a)
{ {
var client = new MegaApiClient(); var client = new MegaApiClient();
Utils.Status("Logging into MEGA (as anonymous)"); Utils.Status("Logging into MEGA (as anonymous)");

2
Wabbajack.Lib/Downloaders/ManualDownloader.cs
View File

@ -111,7 +111,7 @@ namespace Wabbajack.Lib.Downloaders
} }
} }
public override async Task<bool> Verify() public override async Task<bool> Verify(Archive a)
{ {
return true; return true;
} }

2
Wabbajack.Lib/Downloaders/MediaFireDownloader.cs
View File

@ -37,7 +37,7 @@ namespace Wabbajack.Lib.Downloaders
await result.Download(a, destination); await result.Download(a, destination);
} }
public override async Task<bool> Verify() public override async Task<bool> Verify(Archive a)
{ {
return await Resolve() != null; return await Resolve() != null;
} }

2
Wabbajack.Lib/Downloaders/ModDBDownloader.cs
View File

@ -92,7 +92,7 @@ namespace Wabbajack.Lib.Downloaders
return mirrors.Select(d => d.Link).ToArray(); return mirrors.Select(d => d.Link).ToArray();
} }
public override async Task<bool> Verify() public override async Task<bool> Verify(Archive a)
{ {
await GetDownloadUrls(); await GetDownloadUrls();
return true; return true;

2
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -159,7 +159,7 @@ namespace Wabbajack.Lib.Downloaders
} }
public override async Task<bool> Verify() public override async Task<bool> Verify(Archive a)
{ {
try try
{ {

2
Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs
View File

@ -79,7 +79,7 @@ namespace Wabbajack.Lib.Downloaders
} }
} }
public override async Task<bool> Verify() public override async Task<bool> Verify(Archive a)
{ {
//TODO: find a way to verify steam workshop items //TODO: find a way to verify steam workshop items
throw new NotImplementedException(); throw new NotImplementedException();

22
Wabbajack.Test/DownloaderTests.cs
View File

@ -64,7 +64,7 @@ namespace Wabbajack.Test
var converted = await state.RoundTripState(); var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify()); Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString(); var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List<string>{"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k" } })); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List<string>{"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k" } }));
@ -94,7 +94,7 @@ namespace Wabbajack.Test
((HTTPDownloader.State)url_state).Url); ((HTTPDownloader.State)url_state).Url);
var converted = await state.RoundTripState(); var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify()); Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString(); var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?" } })); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?" } }));
@ -124,7 +124,7 @@ namespace Wabbajack.Test
((GoogleDriveDownloader.State)url_state).Id); ((GoogleDriveDownloader.State)url_state).Id);
var converted = await state.RoundTripState(); var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify()); Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString(); var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List<string> { "1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_" } })); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List<string> { "1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_" } }));
@ -153,7 +153,7 @@ namespace Wabbajack.Test
((HTTPDownloader.State)url_state).Url); ((HTTPDownloader.State)url_state).Url);
var converted = await state.RoundTripState(); var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify()); Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString(); var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "http://build.wabbajack.org/" } })); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "http://build.wabbajack.org/" } }));
@ -177,7 +177,7 @@ namespace Wabbajack.Test
Assert.IsNotNull(state); Assert.IsNotNull(state);
var converted = await state.RoundTripState(); var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify()); Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString(); var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "http://build.wabbajack.org/" } })); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "http://build.wabbajack.org/" } }));
@ -238,9 +238,9 @@ namespace Wabbajack.Test
var converted = await state.RoundTripState(); var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify()); Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
// Exercise the cache code // Exercise the cache code
Assert.IsTrue(await converted.Verify()); Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString(); var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> () })); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> () }));
@ -273,7 +273,7 @@ namespace Wabbajack.Test
((ModDBDownloader.State)url_state).Url); ((ModDBDownloader.State)url_state).Url);
var converted = await state.RoundTripState(); var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify()); Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString(); var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() })); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
@ -299,7 +299,7 @@ namespace Wabbajack.Test
((HTTPDownloader.State)url_state).Url); ((HTTPDownloader.State)url_state).Url);
*/ */
var converted = await state.RoundTripState(); var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify()); Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString(); var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() })); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
@ -327,7 +327,7 @@ namespace Wabbajack.Test
((HTTPDownloader.State)url_state).Url); ((HTTPDownloader.State)url_state).Url);
*/ */
var converted = await state.RoundTripState(); var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify()); Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString(); var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() })); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
@ -354,7 +354,7 @@ namespace Wabbajack.Test
Assert.IsNotNull(state); Assert.IsNotNull(state);
var converted = await state.RoundTripState(); var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify()); Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString(); var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() })); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));

3
Wabbajack.Test/ModlistMetadataTests.cs
View File

@ -1,6 +1,7 @@
using System.Linq; using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting; using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.ModListRegistry;
@ -25,7 +26,7 @@ namespace Wabbajack.Test
{ {
var logo_state = DownloadDispatcher.ResolveArchive(modlist.ImageUri); var logo_state = DownloadDispatcher.ResolveArchive(modlist.ImageUri);
Assert.IsNotNull(logo_state); Assert.IsNotNull(logo_state);
Assert.IsTrue(await logo_state.Verify(), $"{modlist.ImageUri} is not valid"); Assert.IsTrue(await logo_state.Verify(new Archive{Size = 0}), $"{modlist.ImageUri} is not valid");
} }
} }
} }