Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ModListdownloader and supporting tests

Browse Source
This commit is contained in:
Timothy Baldridge 2020-05-12 21:39:03 -06:00
parent da28e2325b
commit 087875fba3
5 changed files with 292 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
Wabbajack.Server.Test/ModListValidationTests.cs Normal file
View File

@ -0,0 +1,108 @@
using System;
using System.Collections.Generic;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class ModListValidationTests : ABuildServerSystemTest
{
public ModListValidationTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanLoadMetadataFromTestServer()
{
var modlist = await MakeModList();
Consts.ModlistMetadataURL = modlist.ToString();
var data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
Assert.Equal("test_list", data.First().Links.MachineURL);
}
[Fact]
public async Task CanIngestModLists()
{
var modlist = await MakeModList();
Consts.ModlistMetadataURL = modlist.ToString();
var sql = Fixture.GetService<SqlService>();
var downloader = Fixture.GetService<ModListDownloader>();
await downloader.CheckForNewLists();
foreach (var list in ModListMetaData)
{
Assert.True(await sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash));
}
}
private async Task<Uri> MakeModList()
{
var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!");
var test_archive_path = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
await test_archive_path.WriteAllBytesAsync(archive_data);
ModListData = new ModList();
ModListData.Archives.Add(
new Archive(new HTTPDownloader.State(MakeURL("test_archive.txt")))
{
Hash = await test_archive_path.FileHashAsync(),
Name = "test_archive",
Size = test_archive_path.Size,
});
var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder);
await using (var fs = modListPath.Create())
{
using var za = new ZipArchive(fs, ZipArchiveMode.Create);
var entry = za.CreateEntry("modlist");
await using var es = entry.Open();
ModListData.ToJson(es);
}
ModListMetaData = new List<ModlistMetadata>
{
new ModlistMetadata
{
Official = false,
Author = "Test Suite",
Description = "A test",
DownloadMetadata = new DownloadMetadata
{
Hash = await modListPath.FileHashAsync(),
Size = modListPath.Size
},
Links = new ModlistMetadata.LinksObject
{
MachineURL = "test_list",
Download = MakeURL("test_modlist.wabbajack")
}
}
};
var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder);
ModListMetaData.ToJson(metadataPath);
return new Uri(MakeURL("test_mod_list_metadata.json"));
}
public ModList ModListData { get; set; }
public List<ModlistMetadata> ModListMetaData { get; set; }
}
}

62
Wabbajack.Server/DataLayer/ModLists.cs Normal file
View File

@ -0,0 +1,62 @@
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Lib;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Common;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist)
{
await using var conn = await Open();
await using var tran = await conn.BeginTransactionAsync();
await conn.ExecuteAsync(@"DELETE FROM dbo.ModLists Where MachineUrl = @MachineUrl",
new {MachineUrl = metadata.Links.MachineURL}, tran);
await conn.ExecuteAsync(
@"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList) VALUES (@MachineUrl, @Hash, @Metadata, @ModList)",
new
{
MachineUrl = metadata.Links.MachineURL,
Hash = hash,
MetaData = metadata.ToJson(),
ModList = modlist.ToJson()
}, tran);
var entries = modlist.Archives.Select(a =>
new
{
MachineUrl = metadata.Links.MachineURL,
Hash = a.Hash,
Size = a.Size,
State = a.State.ToJson(),
PrimaryKeyString = a.State.PrimaryKeyString
}).ToArray();
await conn.ExecuteAsync(@"DELETE FROM dbo.ModListArchives WHERE MachineURL = @machineURL",
new {MachineUrl = metadata.Links.MachineURL}, tran);
foreach (var entry in entries)
{
await conn.ExecuteAsync(
"INSERT INTO dbo.ModListArchives (MachineURL, Hash, Size, PrimaryKeyString, State) VALUES (@MachineURL, @Hash, @Size, @PrimaryKeyString, @State)",
entry, tran);
}
await tran.CommitAsync();
}
public async Task<bool> HaveIndexedModlist(string machineUrl, Hash hash)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT MachineURL from dbo.Modlists WHERE MachineURL = @MachineUrl AND Hash = @Hash",
new {MachineUrl = machineUrl, Hash = hash});
return result != null;
}
}
}

119
Wabbajack.Server/Services/ModListDownloader.cs Normal file
View File

@ -0,0 +1,119 @@
using System;
using System.IO.Compression;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.Services
{
public class ModListDownloader
{
private ILogger<ModListDownloader> _logger;
private AppSettings _settings;
private ArchiveMaintainer _maintainer;
private SqlService _sql;
public ModListDownloader(ILogger<ModListDownloader> logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql)
{
_logger = logger;
_settings = settings;
_maintainer = maintainer;
_sql = sql;
}
public void Start()
{
if (_settings.RunBackEndJobs)
{
Task.Run(async () =>
{
while (true)
{
try
{
await CheckForNewLists();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error checking list");
}
await Task.Delay(TimeSpan.FromMinutes(5));
}
});
}
}
public async Task CheckForNewLists()
{
var lists = await ModlistMetadata.LoadFromGithub();
foreach (var list in lists)
{
try
{
if (_maintainer.HaveArchive(list.DownloadMetadata!.Hash))
continue;
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
var tf = new TempFile();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
if (state == null)
{
_logger.Log(LogLevel.Error,
$"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}");
continue;
}
await state.Download(new Archive(state) {Name = $"{list.Links.MachineURL}.wabbajack"}, tf.Path);
var modistPath = await _maintainer.Ingest(tf.Path);
ModList modlist;
await using (var fs = modistPath.OpenRead())
using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
await using (var entry = zip.GetEntry("modlist")?.Open())
{
if (entry == null)
{
Utils.Log($"Bad Modlist {list.Links.MachineURL}");
continue;
}
try
{
modlist = entry.FromJson<ModList>();
}
catch (JsonReaderException ex)
{
Utils.Log($"Bad JSON format for {list.Links.MachineURL}");
continue;
}
}
await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
}
}
}
}
public static class ModListDownloaderExtensions
{
public static void UseModListDownloader(this IApplicationBuilder b)
{
var poll = (ModListDownloader)b.ApplicationServices.GetService(typeof(ModListDownloader));
poll.Start();
}
}
}

1
Wabbajack.Server/Services/NexusPoll.cs
View File

@ -170,6 +170,5 @@ namespace Wabbajack.Server.Services
var poll = (NexusPoll)b.ApplicationServices.GetService(typeof(NexusPoll));
poll.Start();
}
}
}

3
Wabbajack.Server/Startup.cs
View File

@ -58,6 +58,7 @@ namespace Wabbajack.Server
services.AddSingleton<GlobalInformation>();
services.AddSingleton<NexusPoll>();
services.AddSingleton<ArchiveMaintainer>();
services.AddSingleton<ModListDownloader>();
services.AddMvc();
services.AddControllers()
.AddNewtonsoftJson(o =>
@ -100,6 +101,8 @@ namespace Wabbajack.Server
app.UseAuthentication();
app.UseAuthorization();
app.UseNexusPoll();
app.UseArchiveMaintainer();
app.UseModListDownloader();
app.Use(next =>
{