diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 4de27607..ff2cb1b0 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -198,10 +198,15 @@ namespace Wabbajack.Lib await using var patchStream = new MemoryStream(); Status($"Patching {toPatch.To.FileName}"); // Read in the patch data + + Status($"Verifying unpatched file {toPatch.To.FileName}"); + var toFile = OutputFolder.Combine(toPatch.To); + var hash = await toFile.FileHashAsync(); + if (hash != toPatch.FromHash) + throw new InvalidDataException($"Invalid Hash for {toPatch.To} before patching"); byte[] patchData = await LoadBytesFromPath(toPatch.PatchID); - var toFile = OutputFolder.Combine(toPatch.To); var oldData = new MemoryStream(await toFile.ReadAllBytesAsync()); // Remove the file we're about to patch @@ -214,8 +219,8 @@ namespace Wabbajack.Lib } Status($"Verifying Patch {toPatch.To.FileName}"); - var resultSha = await toFile.FileHashAsync(); - if (resultSha != toPatch.Hash) + hash = await toFile.FileHashAsync(); + if (hash != toPatch.Hash) throw new InvalidDataException($"Invalid Hash for {toPatch.To} after patching"); }); } diff --git a/Wabbajack.Server.Test/ArchiveDownloadsTests.cs b/Wabbajack.Server.Test/ArchiveDownloadsTests.cs new file mode 100644 index 00000000..9556c58e --- /dev/null +++ b/Wabbajack.Server.Test/ArchiveDownloadsTests.cs @@ -0,0 +1,55 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Server.DataLayer; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.BuildServer.Test +{ + public class ArchiveDownloadsTests : ABuildServerSystemTest + { + public ArchiveDownloadsTests(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, fixture) + { + } + + [Fact] + public async Task CanEnqueueDequeueAndUpdateDownloads() + { + await ClearDownloaderQueue(); + var state = new HTTPDownloader.State("http://www.google.com"); + var archive = new Archive(state); + + var service = Fixture.GetService(); + var id = await service.EnqueueDownload(archive); + + var toRun = await service.GetNextPendingDownload(); + + Assert.Equal(id, toRun.Id); + + await toRun.Finish(service); + await service.UpdatePendingDownload(toRun); + + toRun = await service.GetNextPendingDownload(); + Assert.Null(toRun); + + var allStates = await service.GetAllArchiveDownloads(); + Assert.Contains(state.PrimaryKeyString, allStates.Select(s => s.PrimaryKeyString)); + + } + + private async Task ClearDownloaderQueue() + { + var service = Fixture.GetService(); + while (true) + { + var job = await service.GetNextPendingDownload(); + if (job == null) break; + + await job.Fail(service, "Canceled"); + } + } + } +} diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index a51319a5..20afb36a 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -465,56 +465,41 @@ CREATE UNIQUE NONCLUSTERED INDEX [ByAPIKey] ON [dbo].[ApiKeys] INCLUDE([Owner]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] GO -/****** Object: Table [dbo].[DownloadStates] Script Date: 3/31/2020 6:22:47 AM ******/ - -CREATE TABLE [dbo].[DownloadStates]( - [Id] [binary](32) NOT NULL, - [Hash] [bigint] NOT NULL, - [PrimaryKey] [nvarchar](max) NOT NULL, - [IniState] [nvarchar](max) NOT NULL, - [JsonState] [nvarchar](max) NOT NULL, - CONSTRAINT [PK_DownloadStates] PRIMARY KEY CLUSTERED - ( - [Id] ASC - )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +CREATE TABLE [dbo].[ArchiveDownloads]( +[Id] [uniqueidentifier] NOT NULL, +[PrimaryKeyString] [nvarchar](255) NOT NULL, +[Size] [bigint] NULL, +[Hash] [bigint] NULL, +[IsFailed] [tinyint] NULL, +[DownloadFinished] [datetime] NULL, +[DownloadState] [nvarchar](max) NOT NULL, +[Downloader] [nvarchar](50) NOT NULL, +[FailMessage] [nvarchar](MAX) NULL, +CONSTRAINT [PK_ArchiveDownloads] PRIMARY KEY CLUSTERED + ( + [Id] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] GO -CREATE NONCLUSTERED INDEX [ByHash] ON [dbo].[DownloadStates] + +/****** Object: Index [ByDownloaderAndFinished] Script Date: 5/13/2020 8:47:58 PM ******/ +CREATE NONCLUSTERED INDEX [ByDownloaderAndFinished] ON [dbo].[ArchiveDownloads] ( + [DownloadFinished] ASC, + [Downloader] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +GO + +/****** Object: Index [ByPrimaryKeyAndHash] Script Date: 5/13/2020 8:48:01 PM ******/ +CREATE NONCLUSTERED INDEX [ByPrimaryKeyAndHash] ON [dbo].[ArchiveDownloads] + ( + [PrimaryKeyString] ASC, [Hash] ASC - ) - INCLUDE([IniState]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] GO -/****** Object: View [dbo].[GameFiles] Script Date: 4/30/2020 4:23:25 PM ******/ - -CREATE VIEW [dbo].[GameFiles] - WITH SCHEMABINDING -AS - -Select - Id, - CONVERT(NVARCHAR(20), JSON_VALUE(JsonState,'$.GameVersion')) as GameVersion, - CONVERT(NVARCHAR(32),JSON_VALUE(JsonState,'$.Game')) as Game, - JSON_VALUE(JsonState,'$.GameFile') as Path, - Hash as Hash -FROM dbo.DownloadStates -WHERE PrimaryKey like 'GameFileSourceDownloader+State|%' - AND JSON_VALUE(JsonState,'$.GameFile') NOT LIKE '%.xxhash' -GO - -CREATE UNIQUE CLUSTERED INDEX [ByGameAndVersion] ON [dbo].[GameFiles] - ( - [Game] ASC, - [GameVersion] ASC, - [Id] ASC - )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] -GO - - - /****** Object: Index [IX_Child] Script Date: 3/28/2020 4:58:59 PM ******/ CREATE NONCLUSTERED INDEX [IX_Child] ON [dbo].[AllFilesInArchive] ( diff --git a/Wabbajack.Server/DTOs/ArchiveDownload.cs b/Wabbajack.Server/DTOs/ArchiveDownload.cs new file mode 100644 index 00000000..bf43b82e --- /dev/null +++ b/Wabbajack.Server/DTOs/ArchiveDownload.cs @@ -0,0 +1,31 @@ +using System; +using System.Threading.Tasks; +using Wabbajack.Lib; +using Wabbajack.Server.DataLayer; + +namespace Wabbajack.Server.DTOs +{ + public class ArchiveDownload + { + public Guid Id { get; set; } + public Archive Archive { get; set; } + public bool? IsFailed { get; set; } + public DateTime? DownloadFinished { get; set; } + public string FailMessage { get; set; } + + public async Task Fail(SqlService service, string message) + { + IsFailed = true; + DownloadFinished = DateTime.UtcNow; + FailMessage = message; + await service.UpdatePendingDownload(this); + } + + public async Task Finish(SqlService service) + { + IsFailed = true; + DownloadFinished = DateTime.UtcNow; + await service.UpdatePendingDownload(this); + } + } +} diff --git a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs new file mode 100644 index 00000000..359a9fd1 --- /dev/null +++ b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Dapper; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.AuthorApi; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Server.DTOs; + +namespace Wabbajack.Server.DataLayer +{ + public partial class SqlService + { + public async Task EnqueueDownload(Archive a) + { + await using var conn = await Open(); + var Id = Guid.NewGuid(); + await conn.ExecuteAsync( + "INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)", + new + { + Id = Id, + PrimaryKeyString = a.State.PrimaryKeyString, + Size = a.Size == 0 ? null : (long?)a.Size, + Hash = a.Hash == default ? null : (Hash?)a.Hash, + DownloadState = a.State, + Downloader = AbstractDownloadState.TypeToName[a.State.GetType()] + }); + return Id; + } + + public async Task> GetAllArchiveDownloads() + { + await using var conn = await Open(); + return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, PrimaryKeyString FROM ArchiveDownloads")).ToHashSet(); + } + + public async Task GetNextPendingDownload(bool ignoreNexus = false) + { + await using var conn = await Open(); + (Guid, long?, Hash?, AbstractDownloadState) result; + + if (ignoreNexus) + { + result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, AbstractDownloadState)>( + "SELECT Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL AND Downloader != 'NexusDownloader+State'"); + } + else + { + result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, AbstractDownloadState)>( + "SELECT Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL"); + } + + if (result == default) + return null; + + return new ArchiveDownload + { + Id = result.Item1, + Archive = new Archive(result.Item4) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default,}, + }; + } + + public async Task UpdatePendingDownload(ArchiveDownload ad) + { + await using var conn = await Open(); + await conn.ExecuteAsync( + "UPDATE dbo.ArchiveDownloads SET IsFailed = @IsFailed, DownloadFinished = @DownloadFinished, Hash = @Hash, Size = @Size, FailMessage = @FailMessage WHERE Id = @Id", + new + { + Id = ad.Id, + IsFailed = ad.IsFailed, + DownloadFinished = ad.DownloadFinished, + Size = ad.Archive.Size, + Hash = ad.Archive.Hash, + FailMessage = ad.FailMessage + }); + } + + public async Task EnqueueModListFilesForIndexing() + { + await using var conn = await Open(); + return await conn.ExecuteAsync(@" + INSERT INTO dbo.ArchiveDownloads (Id, PrimaryKeyString, Hash, DownloadState, Size, Downloader) + SELECT DISTINCT NEWID(), mla.PrimaryKeyString, mla.Hash, mla.State, mla.Size, SUBSTRING(mla.PrimaryKeyString, 0, CHARINDEX('|', mla.PrimaryKeyString)) + FROM [dbo].[ModListArchives] mla + LEFT JOIN dbo.ArchiveDownloads ad on mla.PrimaryKeyString = ad.PrimaryKeyString AND mla.Hash = ad.Hash + WHERE ad.PrimaryKeyString is null"); + } + + } +} diff --git a/Wabbajack.Server/Services/ArchiveDownloader.cs b/Wabbajack.Server/Services/ArchiveDownloader.cs new file mode 100644 index 00000000..eea0b99d --- /dev/null +++ b/Wabbajack.Server/Services/ArchiveDownloader.cs @@ -0,0 +1,96 @@ +using System; +using System.Reflection.Metadata.Ecma335; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer; +using Wabbajack.Common; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.NexusApi; +using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; + +namespace Wabbajack.Server.Services +{ + public class ArchiveDownloader : AbstractService + { + private SqlService _sql; + private ArchiveMaintainer _archiveMaintainer; + private NexusApiClient _nexusClient; + + public ArchiveDownloader(ILogger logger, AppSettings settings, SqlService sql, ArchiveMaintainer archiveMaintainer) : base(logger, settings, TimeSpan.FromMinutes(10)) + { + _sql = sql; + _archiveMaintainer = archiveMaintainer; + } + + public override async Task Execute() + { + _nexusClient ??= await NexusApiClient.Get(); + await _nexusClient.GetUserStatus(); + int count = 0; + + while (true) + { + bool ignoreNexus = _nexusClient.HourlyRemaining < 25; + + var nextDownload = await _sql.GetNextPendingDownload(ignoreNexus); + + if (nextDownload == null) + break; + + if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash)) + { + await nextDownload.Finish(_sql); + continue; + } + + if (nextDownload.Archive.State is ManualDownloader.State) + { + await nextDownload.Finish(_sql); + continue; + } + + try + { + _logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"); + await DownloadDispatcher.PrepareAll(new[] {nextDownload.Archive.State}); + + using var tempPath = new TempFile(); + await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path); + + var hash = await tempPath.Path.FileHashAsync(); + + if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash) + { + await nextDownload.Fail(_sql, "Invalid Hash"); + continue; + } + + if (nextDownload.Archive.Size != default && + tempPath.Path.Size != nextDownload.Archive.Size) + { + await nextDownload.Fail(_sql, "Invalid Size"); + continue; + } + nextDownload.Archive.Hash = hash; + nextDownload.Archive.Size = tempPath.Path.Size; + + _logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}"); + await _archiveMaintainer.Ingest(tempPath.Path); + + _logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}"); + await nextDownload.Finish(_sql); + + } + catch (Exception ex) + { + await nextDownload.Fail(_sql, ex.ToString()); + } + + count++; + } + + return count; + } + } +} diff --git a/Wabbajack.Server/Services/ModListDownloader.cs b/Wabbajack.Server/Services/ModListDownloader.cs index dc35ab5c..d8d4b17c 100644 --- a/Wabbajack.Server/Services/ModListDownloader.cs +++ b/Wabbajack.Server/Services/ModListDownloader.cs @@ -123,6 +123,10 @@ namespace Wabbajack.Server.Services } } _logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists"); + + var fc = await _sql.EnqueueModListFilesForIndexing(); + _logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading"); + return downloaded; } } diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 8f74ded4..a2de2e82 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -62,6 +62,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddMvc(); services.AddControllers() @@ -110,6 +111,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); + app.UseService(); app.Use(next => {