Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Validation service seems to be working well

Browse Source
This commit is contained in:
Timothy Baldridge 2020-04-13 17:31:48 -06:00
parent f9f2c349c0
commit 94514bd2cb
4 changed files with 151 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
Wabbajack.BuildServer.Test/ModListValidationTests.cs
View File

@ -73,8 +73,18 @@ namespace Wabbajack.BuildServer.Test
var archive = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
await archive.MoveToAsync(archive.WithExtension(new Extension(".moved")), true);
// We can revalidate but the non-nexus archives won't be checked yet since the list didn't change
await RevalidateLists();
data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
Assert.Equal(0, data.First().ValidationSummary.Failed);
Assert.Equal(1, data.First().ValidationSummary.Passed);
// Run the non-nexus validator
var evalService = new ValidateNonNexusArchives(Fixture.GetService<SqlService>(), Fixture.GetService<AppSettings>());
await evalService.Execute();
data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
Assert.Equal(1, data.First().ValidationSummary.Failed);
@ -86,6 +96,9 @@ namespace Wabbajack.BuildServer.Test
await archive.WithExtension(new Extension(".moved")).MoveToAsync(archive, false);
await RevalidateLists();
// Rerun the validation service to fix the list
await evalService.Execute();
data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
@ -98,20 +111,11 @@ namespace Wabbajack.BuildServer.Test
private async Task RevalidateLists()
{
var result = await AuthorAPI.UpdateServerModLists();
Assert.NotNull(result);
var sql = Fixture.GetService<SqlService>();
var settings = Fixture.GetService<AppSettings>();
var job = await sql.GetJob();
Assert.NotNull(job);
Assert.IsType<UpdateModLists>(job.Payload);
job.Result = await job.Payload.Execute(sql, settings);
await sql.FinishJob(job);
Assert.Equal(JobResultType.Success, job.Result.ResultType);
var jobService = new ListIngest(sql, settings);
await jobService.Execute();
}
private async Task CheckListFeeds(int failed, int passed)

12
Wabbajack.BuildServer/BackendServices/ValidateNonNexusArchives.cs
View File

@ -17,8 +17,16 @@ namespace Wabbajack.BuildServer.BackendServices
using var queue = new WorkQueue();
var results = await archives.PMap(queue, async archive =>
{
var isValid = await archive.State.Verify(archive);
return (Archive: archive, IsValid: isValid);
try
{
var isValid = await archive.State.Verify(archive);
return (Archive: archive, IsValid: isValid);
}
catch (Exception)
{
return (Archive: archive, IsValid: false);
}
});
await Sql.UpdateNonNexusModlistArchivesStatus(results);

90
Wabbajack.BuildServer/Controllers/ListValidation.cs
View File

@ -7,7 +7,9 @@ using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.BuildServer.Controllers
@ -19,12 +21,75 @@ namespace Wabbajack.BuildServer.Controllers
public ListValidation(ILogger<ListValidation> logger, SqlService sql) : base(logger, sql)
{
}
public async Task<IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)>> GetSummaries()
{
var data = await SQL.GetValidationData();
using var queue = new WorkQueue();
var results = data.ModLists.PMap(queue, list =>
{
var archives = list.ModList.Archives.Select(archive =>
{
switch (archive.State)
{
case NexusDownloader.State nexusState when data.NexusFiles.Contains((
nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
return (archive, true);
case NexusDownloader.State nexusState:
return (archive, false);
case ManualDownloader.State _:
return (archive, true);
default:
{
if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash),
out var isValid))
{
return (archive, isValid);
}
return (archive, false);
}
}
}).ToList();
var failedCount = archives.Count(f => !f.Item2);
var passCount = archives.Count(f => f.Item2);
var summary = new ModListSummary
{
Checked = DateTime.UtcNow,
Failed = failedCount,
MachineURL = list.Metadata.Links.MachineURL,
Name = list.Metadata.Title,
Passed = passCount
};
var detailed = new DetailedStatus
{
Name = list.Metadata.Title,
Checked = DateTime.UtcNow,
DownloadMetaData = list.Metadata.DownloadMetadata,
HasFailures = failedCount > 0,
MachineName = list.Metadata.Links.MachineURL,
Archives = archives.Select(a => new DetailedStatusItem
{
Archive = a.archive, IsFailing = !a.Item2
}).ToList()
};
return (summary, detailed);
});
return await results;
}
[HttpGet]
[Route("status.json")]
public async Task<IEnumerable<ModListSummary>> HandleGetLists()
{
return await SQL.GetModListSummaries();
return (await GetSummaries()).Select(d => d.Summary);
}
private static readonly Func<object, string> HandleGetRssFeedTemplate = NettleEngine.GetCompiler().Compile(@"
@ -48,7 +113,7 @@ namespace Wabbajack.BuildServer.Controllers
[Route("status/{Name}/broken.rss")]
public async Task<ContentResult> HandleGetRSSFeed(string Name)
{
var lst = await SQL.GetDetailedModlistStatus(Name);
var lst = await DetailedStatus(Name);
var response = HandleGetRssFeedTemplate(new
{
lst,
@ -86,7 +151,7 @@ namespace Wabbajack.BuildServer.Controllers
public async Task<ContentResult> HandleGetListHtml(string Name)
{
var lst = await SQL.GetDetailedModlistStatus(Name);
var lst = await DetailedStatus(Name);
var response = HandleGetListTemplate(new
{
lst,
@ -104,19 +169,16 @@ namespace Wabbajack.BuildServer.Controllers
[HttpGet]
[Route("status/{Name}.json")]
public async Task<ContentResult> HandleGetListJson(string Name)
public async Task<IActionResult> HandleGetListJson(string Name)
{
var lst = await SQL.GetDetailedModlistStatus(Name);
lst.Archives.Do(a => a.Archive.Meta = null);
return new ContentResult
{
ContentType = "application/json",
StatusCode = (int) HttpStatusCode.OK,
Content = lst.ToJson()
};
return Ok((await DetailedStatus(Name)).ToJson());
}
private async Task<DetailedStatus> DetailedStatus(string Name)
{
return (await GetSummaries())
.Select(d => d.Detailed)
.FirstOrDefault(d => d.MachineName == Name);
}
}
}

50
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -6,11 +6,13 @@ using System.Data.SqlClient;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Microsoft.AspNetCore.Mvc.ModelBinding.Metadata;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Wabbajack.BuildServer.Model.Models.Results;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
@ -682,5 +684,53 @@ namespace Wabbajack.BuildServer.Model.Models
await trans.CommitAsync();
}
public async Task<ValidationData> GetValidationData()
{
var nexusFiles = AllNexusFiles();
var archiveStatus = AllModListArchivesStatus();
var modLists = AllModLists();
return new ValidationData
{
NexusFiles = await nexusFiles,
ArchiveStatus = await archiveStatus,
ModLists = await modLists
};
}
public async Task<Dictionary<(string PrimaryKeyString, Hash Hash), bool>> AllModListArchivesStatus()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<(string, Hash, bool)>(
@"SELECT PrimaryKeyString, Hash, IsValid FROM dbo.ModListArchiveStatus");
return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3);
}
public async Task<HashSet<(long NexusGameId, long ModId, long FileId)>> AllNexusFiles()
{
await using var conn = await Open();
var results = await conn.QueryAsync<(long, long, long)>(@"SELECT Game, ModId, p.file_id
FROM [NexusModFiles] files
CROSS APPLY
OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p
WHERE p.category is not null");
return results.ToHashSet();
}
public async Task<List<(ModlistMetadata, ModList)>> AllModLists()
{
await using var conn = await Open();
var results = await conn.QueryAsync<(string, string)>(@"SELECT Metadata, ModList FROM dbo.ModLists");
return results.Select(m => (m.Item1.FromJsonString<ModlistMetadata>(), m.Item2.FromJsonString<ModList>())).ToList();
}
public class ValidationData
{
public HashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; }
public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
public List<(ModlistMetadata Metadata, ModList ModList)> ModLists { get; set; }
}
}
}