diff --git a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs index 5d0cd97a..a494a9df 100644 --- a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs +++ b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.IO.Compression; +using System.IO.MemoryMappedFiles; using System.Linq; using System.Threading.Tasks; using Wabbajack.Common; @@ -58,15 +59,18 @@ namespace Wabbajack.Lib.Downloaders { destination.Parent.CreateDirectory(); var definition = await GetDefinition(); - await using var fs = destination.Create(); + using var fs = destination.Create(); + using var mmfile = MemoryMappedFile.CreateFromFile(fs, null, definition.Size, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, false); var client = new Common.Http.Client(); - await definition.Parts.DoProgress($"Downloading {a.Name}", async part => + using var queue = new WorkQueue(); + await definition.Parts.PMap(queue, async part => { - fs.Position = part.Offset; + Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(definition.Parts.Length - part.Index, definition.Parts.Length)); + await using var ostream = mmfile.CreateViewStream(part.Offset, part.Size); using var response = await client.GetAsync($"{Url}/parts/{part.Index}"); if (!response.IsSuccessStatusCode) throw new HttpException((int)response.StatusCode, response.ReasonPhrase); - await response.Content.CopyToAsync(fs); + await response.Content.CopyToAsync(ostream); }); return true; } diff --git a/Wabbajack.Server.Test/ABuildServerSystemTest.cs b/Wabbajack.Server.Test/ABuildServerSystemTest.cs index 253e8d0c..5190ab37 100644 --- a/Wabbajack.Server.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.Server.Test/ABuildServerSystemTest.cs @@ -32,6 +32,9 @@ namespace Wabbajack.BuildServer.Test public BuildServerFixture() { + ServerArchivesFolder.DeleteDirectory(); + ServerArchivesFolder.CreateDirectory(); + var builder = Program.CreateHostBuilder( new[] { @@ -54,6 +57,7 @@ namespace Wabbajack.BuildServer.Test "ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllText( "GoogleIDs:\nAllowedPrefixes:\n - http://localhost"); + } ~BuildServerFixture() @@ -149,6 +153,7 @@ namespace Wabbajack.BuildServer.Test Consts.ModlistSummaryURL = MakeURL("lists/status.json"); Consts.ServerWhitelistURL = MakeURL("ServerWhitelist.yaml"); + } public WorkQueue Queue { get; set; } diff --git a/Wabbajack.Server.Test/ArchiveMaintainerTests.cs b/Wabbajack.Server.Test/ArchiveMaintainerTests.cs new file mode 100644 index 00000000..6906255e --- /dev/null +++ b/Wabbajack.Server.Test/ArchiveMaintainerTests.cs @@ -0,0 +1,49 @@ +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Server.Services; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.BuildServer.Test +{ + public class ArchiveMaintainerTests : ABuildServerSystemTest + { + public ArchiveMaintainerTests(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, fixture) + { + } + + [Fact] + public async Task CanIngestFiles() + { + var maintainer = Fixture.GetService(); + using var tf = new TempFile(); + using var tf2 = new TempFile(); + + await tf.Path.WriteAllBytesAsync(RandomData(1024)); + await tf.Path.CopyToAsync(tf2.Path); + + + var hash = await tf.Path.FileHashAsync(); + await maintainer.Ingest(tf.Path); + + Assert.True(maintainer.TryGetPath(hash, out var found)); + Assert.Equal(await tf2.Path.ReadAllBytesAsync(), await found.ReadAllBytesAsync()); + } + + + [Fact] + public async Task IngestsExistingFiles() + { + var maintainer = Fixture.GetService(); + using var tf = new TempFile(); + + await tf.Path.WriteAllBytesAsync(RandomData(1024)); + var hash = await tf.Path.FileHashAsync(); + + await tf.Path.CopyToAsync(Fixture.ServerArchivesFolder.Combine(hash.ToHex())); + maintainer.Start(); + + Assert.True(maintainer.TryGetPath(hash, out var found)); + } + } +} diff --git a/Wabbajack.Server.Test/ModListValidationTests.cs b/Wabbajack.Server.Test/ModListValidationTests.cs new file mode 100644 index 00000000..f64d5da8 --- /dev/null +++ b/Wabbajack.Server.Test/ModListValidationTests.cs @@ -0,0 +1,108 @@ +using System; +using System.Collections.Generic; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.ModListRegistry; +using Wabbajack.Server.DataLayer; +using Wabbajack.Server.Services; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.BuildServer.Test +{ + public class ModListValidationTests : ABuildServerSystemTest + { + public ModListValidationTests(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, fixture) + { + } + + [Fact] + public async Task CanLoadMetadataFromTestServer() + { + var modlist = await MakeModList(); + Consts.ModlistMetadataURL = modlist.ToString(); + var data = await ModlistMetadata.LoadFromGithub(); + Assert.Single(data); + Assert.Equal("test_list", data.First().Links.MachineURL); + } + + [Fact] + public async Task CanIngestModLists() + { + var modlist = await MakeModList(); + Consts.ModlistMetadataURL = modlist.ToString(); + var sql = Fixture.GetService(); + var downloader = Fixture.GetService(); + await downloader.CheckForNewLists(); + + foreach (var list in ModListMetaData) + { + Assert.True(await sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash)); + } + } + + private async Task MakeModList() + { + var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!"); + var test_archive_path = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder); + await test_archive_path.WriteAllBytesAsync(archive_data); + + + + ModListData = new ModList(); + ModListData.Archives.Add( + new Archive(new HTTPDownloader.State(MakeURL("test_archive.txt"))) + { + Hash = await test_archive_path.FileHashAsync(), + Name = "test_archive", + Size = test_archive_path.Size, + }); + + var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder); + + await using (var fs = modListPath.Create()) + { + using var za = new ZipArchive(fs, ZipArchiveMode.Create); + var entry = za.CreateEntry("modlist"); + await using var es = entry.Open(); + ModListData.ToJson(es); + } + + ModListMetaData = new List + { + new ModlistMetadata + { + Official = false, + Author = "Test Suite", + Description = "A test", + DownloadMetadata = new DownloadMetadata + { + Hash = await modListPath.FileHashAsync(), + Size = modListPath.Size + }, + Links = new ModlistMetadata.LinksObject + { + MachineURL = "test_list", + Download = MakeURL("test_modlist.wabbajack") + } + } + }; + + var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder); + + ModListMetaData.ToJson(metadataPath); + + return new Uri(MakeURL("test_mod_list_metadata.json")); + } + + public ModList ModListData { get; set; } + + public List ModListMetaData { get; set; } + + } +} diff --git a/Wabbajack.Server/DataLayer/ModLists.cs b/Wabbajack.Server/DataLayer/ModLists.cs new file mode 100644 index 00000000..d73357e6 --- /dev/null +++ b/Wabbajack.Server/DataLayer/ModLists.cs @@ -0,0 +1,62 @@ +using System.Linq; +using System.Threading.Tasks; +using Dapper; +using Wabbajack.Lib; +using Wabbajack.Lib.ModListRegistry; +using Wabbajack.Common; + +namespace Wabbajack.Server.DataLayer +{ + public partial class SqlService + { + public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist) + { + await using var conn = await Open(); + await using var tran = await conn.BeginTransactionAsync(); + + await conn.ExecuteAsync(@"DELETE FROM dbo.ModLists Where MachineUrl = @MachineUrl", + new {MachineUrl = metadata.Links.MachineURL}, tran); + + await conn.ExecuteAsync( + @"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList) VALUES (@MachineUrl, @Hash, @Metadata, @ModList)", + new + { + MachineUrl = metadata.Links.MachineURL, + Hash = hash, + MetaData = metadata.ToJson(), + ModList = modlist.ToJson() + }, tran); + + var entries = modlist.Archives.Select(a => + new + { + MachineUrl = metadata.Links.MachineURL, + Hash = a.Hash, + Size = a.Size, + State = a.State.ToJson(), + PrimaryKeyString = a.State.PrimaryKeyString + }).ToArray(); + + await conn.ExecuteAsync(@"DELETE FROM dbo.ModListArchives WHERE MachineURL = @machineURL", + new {MachineUrl = metadata.Links.MachineURL}, tran); + + foreach (var entry in entries) + { + await conn.ExecuteAsync( + "INSERT INTO dbo.ModListArchives (MachineURL, Hash, Size, PrimaryKeyString, State) VALUES (@MachineURL, @Hash, @Size, @PrimaryKeyString, @State)", + entry, tran); + } + + await tran.CommitAsync(); + } + + public async Task HaveIndexedModlist(string machineUrl, Hash hash) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync( + "SELECT MachineURL from dbo.Modlists WHERE MachineURL = @MachineUrl AND Hash = @Hash", + new {MachineUrl = machineUrl, Hash = hash}); + return result != null; + } + } +} diff --git a/Wabbajack.Server/Services/ArchiveMaintainer.cs b/Wabbajack.Server/Services/ArchiveMaintainer.cs new file mode 100644 index 00000000..f6237461 --- /dev/null +++ b/Wabbajack.Server/Services/ArchiveMaintainer.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer; +using Wabbajack.Common; +using File = System.IO.File; + +namespace Wabbajack.Server.Services +{ + /// + /// Maintains a concurrent cache of all the files we've downloaded, indexed by Hash + /// + public class ArchiveMaintainer + { + private AppSettings _settings; + private ILogger _logger; + private ConcurrentDictionary _archives = new ConcurrentDictionary(); + + public ArchiveMaintainer(ILogger logger, AppSettings settings) + { + _settings = settings; + _logger = logger; + } + + public void Start() + { + foreach (var path in _settings.ArchivePath.EnumerateFiles(false)) + { + try + { + var hash = Hash.FromHex((string)path.FileNameWithoutExtension); + _archives[hash] = path; + } + catch (Exception ex) + { + _logger.Log(LogLevel.Error, ex.ToString()); + } + } + _logger.Log(LogLevel.Information, $"Found {_archives.Count} archives"); + } + + public async Task Ingest(AbsolutePath file) + { + var hash = await file.FileHashAsync(); + if (HaveArchive(hash)) + { + file.Delete(); + return _archives[hash]; + } + + var newPath = _settings.ArchivePath.Combine(hash.ToHex()); + await file.MoveToAsync(newPath); + _archives[hash] = newPath; + return _archives[hash]; + } + + public bool HaveArchive(Hash hash) + { + return _archives.ContainsKey(hash); + } + + public bool TryGetPath(Hash hash, out AbsolutePath path) + { + return _archives.TryGetValue(hash, out path); + } + } + + public static class ArchiveMaintainerExtensions + { + public static void UseArchiveMaintainer(this IApplicationBuilder b) + { + var poll = (ArchiveMaintainer)b.ApplicationServices.GetService(typeof(ArchiveMaintainer)); + poll.Start(); + } + + } +} diff --git a/Wabbajack.Server/Services/ModListDownloader.cs b/Wabbajack.Server/Services/ModListDownloader.cs new file mode 100644 index 00000000..07638860 --- /dev/null +++ b/Wabbajack.Server/Services/ModListDownloader.cs @@ -0,0 +1,119 @@ +using System; +using System.IO.Compression; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Wabbajack.BuildServer; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.ModListRegistry; +using Wabbajack.Server.DataLayer; + +namespace Wabbajack.Server.Services +{ + public class ModListDownloader + { + private ILogger _logger; + private AppSettings _settings; + private ArchiveMaintainer _maintainer; + private SqlService _sql; + + public ModListDownloader(ILogger logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql) + { + _logger = logger; + _settings = settings; + _maintainer = maintainer; + _sql = sql; + } + + public void Start() + { + if (_settings.RunBackEndJobs) + { + Task.Run(async () => + { + while (true) + { + try + { + await CheckForNewLists(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error checking list"); + } + + await Task.Delay(TimeSpan.FromMinutes(5)); + + } + }); + } + } + + public async Task CheckForNewLists() + { + var lists = await ModlistMetadata.LoadFromGithub(); + foreach (var list in lists) + { + try + { + if (_maintainer.HaveArchive(list.DownloadMetadata!.Hash)) + continue; + + _logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}"); + var tf = new TempFile(); + var state = DownloadDispatcher.ResolveArchive(list.Links.Download); + if (state == null) + { + _logger.Log(LogLevel.Error, + $"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}"); + continue; + } + + await state.Download(new Archive(state) {Name = $"{list.Links.MachineURL}.wabbajack"}, tf.Path); + var modistPath = await _maintainer.Ingest(tf.Path); + + ModList modlist; + await using (var fs = modistPath.OpenRead()) + using (var zip = new ZipArchive(fs, ZipArchiveMode.Read)) + await using (var entry = zip.GetEntry("modlist")?.Open()) + { + if (entry == null) + { + Utils.Log($"Bad Modlist {list.Links.MachineURL}"); + continue; + } + + try + { + modlist = entry.FromJson(); + } + catch (JsonReaderException ex) + { + Utils.Log($"Bad JSON format for {list.Links.MachineURL}"); + continue; + } + } + + await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}"); + } + } + } + } + + public static class ModListDownloaderExtensions + { + public static void UseModListDownloader(this IApplicationBuilder b) + { + var poll = (ModListDownloader)b.ApplicationServices.GetService(typeof(ModListDownloader)); + poll.Start(); + } + + } +} diff --git a/Wabbajack.Server/Services/NexusPoll.cs b/Wabbajack.Server/Services/NexusPoll.cs index 64c1f18e..d3691adc 100644 --- a/Wabbajack.Server/Services/NexusPoll.cs +++ b/Wabbajack.Server/Services/NexusPoll.cs @@ -170,6 +170,5 @@ namespace Wabbajack.Server.Services var poll = (NexusPoll)b.ApplicationServices.GetService(typeof(NexusPoll)); poll.Start(); } - } } diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index ffb66314..61935204 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -56,7 +56,9 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddMvc(); services.AddControllers() .AddNewtonsoftJson(o => @@ -99,6 +101,8 @@ namespace Wabbajack.Server app.UseAuthentication(); app.UseAuthorization(); app.UseNexusPoll(); + app.UseArchiveMaintainer(); + app.UseModListDownloader(); app.Use(next => {