Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Create a mod file archiver.

Browse Source
This commit is contained in:
Timothy Baldridge 2020-05-13 22:08:27 -06:00
parent 2a8687a720
commit a57794479f
7 changed files with 309 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
Wabbajack.Server.Test/ArchiveDownloadsTests.cs Normal file
View File

@ -0,0 +1,55 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DataLayer;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class ArchiveDownloadsTests : ABuildServerSystemTest
{
public ArchiveDownloadsTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanEnqueueDequeueAndUpdateDownloads()
{
await ClearDownloaderQueue();
var state = new HTTPDownloader.State("http://www.google.com");
var archive = new Archive(state);
var service = Fixture.GetService<SqlService>();
var id = await service.EnqueueDownload(archive);
var toRun = await service.GetNextPendingDownload();
Assert.Equal(id, toRun.Id);
await toRun.Finish(service);
await service.UpdatePendingDownload(toRun);
toRun = await service.GetNextPendingDownload();
Assert.Null(toRun);
var allStates = await service.GetAllArchiveDownloads();
Assert.Contains(state.PrimaryKeyString, allStates.Select(s => s.PrimaryKeyString));
}
private async Task ClearDownloaderQueue()
{
var service = Fixture.GetService<SqlService>();
while (true)
{
var job = await service.GetNextPendingDownload();
if (job == null) break;
await job.Fail(service, "Canceled");
}
}
}
}

69
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -465,56 +465,41 @@ CREATE UNIQUE NONCLUSTERED INDEX [ByAPIKey] ON [dbo].[ApiKeys]
INCLUDE([Owner]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
GO
/****** Object: Table [dbo].[DownloadStates] Script Date: 3/31/2020 6:22:47 AM ******/
CREATE TABLE [dbo].[DownloadStates](
[Id] [binary](32) NOT NULL,
[Hash] [bigint] NOT NULL,
[PrimaryKey] [nvarchar](max) NOT NULL,
[IniState] [nvarchar](max) NOT NULL,
[JsonState] [nvarchar](max) NOT NULL,
CONSTRAINT [PK_DownloadStates] PRIMARY KEY CLUSTERED
(
[Id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
CREATE TABLE [dbo].[ArchiveDownloads](
[Id] [uniqueidentifier] NOT NULL,
[PrimaryKeyString] [nvarchar](255) NOT NULL,
[Size] [bigint] NULL,
[Hash] [bigint] NULL,
[IsFailed] [tinyint] NULL,
[DownloadFinished] [datetime] NULL,
[DownloadState] [nvarchar](max) NOT NULL,
[Downloader] [nvarchar](50) NOT NULL,
[FailMessage] [nvarchar](MAX) NULL,
CONSTRAINT [PK_ArchiveDownloads] PRIMARY KEY CLUSTERED
(
[Id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
CREATE NONCLUSTERED INDEX [ByHash] ON [dbo].[DownloadStates]
/****** Object: Index [ByDownloaderAndFinished] Script Date: 5/13/2020 8:47:58 PM ******/
CREATE NONCLUSTERED INDEX [ByDownloaderAndFinished] ON [dbo].[ArchiveDownloads]
(
[DownloadFinished] ASC,
[Downloader] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
GO
/****** Object: Index [ByPrimaryKeyAndHash] Script Date: 5/13/2020 8:48:01 PM ******/
CREATE NONCLUSTERED INDEX [ByPrimaryKeyAndHash] ON [dbo].[ArchiveDownloads]
(
[PrimaryKeyString] ASC,
[Hash] ASC
)
INCLUDE([IniState]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
GO
/****** Object: View [dbo].[GameFiles] Script Date: 4/30/2020 4:23:25 PM ******/
CREATE VIEW [dbo].[GameFiles]
WITH SCHEMABINDING
AS
Select
Id,
CONVERT(NVARCHAR(20), JSON_VALUE(JsonState,'$.GameVersion')) as GameVersion,
CONVERT(NVARCHAR(32),JSON_VALUE(JsonState,'$.Game')) as Game,
JSON_VALUE(JsonState,'$.GameFile') as Path,
Hash as Hash
FROM dbo.DownloadStates
WHERE PrimaryKey like 'GameFileSourceDownloader+State|%'
AND JSON_VALUE(JsonState,'$.GameFile') NOT LIKE '%.xxhash'
GO
CREATE UNIQUE CLUSTERED INDEX [ByGameAndVersion] ON [dbo].[GameFiles]
(
[Game] ASC,
[GameVersion] ASC,
[Id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
GO
/****** Object: Index [IX_Child] Script Date: 3/28/2020 4:58:59 PM ******/
CREATE NONCLUSTERED INDEX [IX_Child] ON [dbo].[AllFilesInArchive]
(

31
Wabbajack.Server/DTOs/ArchiveDownload.cs Normal file
View File

@ -0,0 +1,31 @@
using System;
using System.Threading.Tasks;
using Wabbajack.Lib;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.DTOs
{
public class ArchiveDownload
{
public Guid Id { get; set; }
public Archive Archive { get; set; }
public bool? IsFailed { get; set; }
public DateTime? DownloadFinished { get; set; }
public string FailMessage { get; set; }
public async Task Fail(SqlService service, string message)
{
IsFailed = true;
DownloadFinished = DateTime.UtcNow;
FailMessage = message;
await service.UpdatePendingDownload(this);
}
public async Task Finish(SqlService service)
{
IsFailed = true;
DownloadFinished = DateTime.UtcNow;
await service.UpdatePendingDownload(this);
}
}
}

94
Wabbajack.Server/DataLayer/ArchiveDownloads.cs Normal file
View File

@ -0,0 +1,94 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.AuthorApi;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task<Guid> EnqueueDownload(Archive a)
{
await using var conn = await Open();
var Id = Guid.NewGuid();
await conn.ExecuteAsync(
"INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)",
new
{
Id = Id,
PrimaryKeyString = a.State.PrimaryKeyString,
Size = a.Size == 0 ? null : (long?)a.Size,
Hash = a.Hash == default ? null : (Hash?)a.Hash,
DownloadState = a.State,
Downloader = AbstractDownloadState.TypeToName[a.State.GetType()]
});
return Id;
}
public async Task<HashSet<(Hash Hash, string PrimaryKeyString)>> GetAllArchiveDownloads()
{
await using var conn = await Open();
return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, PrimaryKeyString FROM ArchiveDownloads")).ToHashSet();
}
public async Task<ArchiveDownload> GetNextPendingDownload(bool ignoreNexus = false)
{
await using var conn = await Open();
(Guid, long?, Hash?, AbstractDownloadState) result;
if (ignoreNexus)
{
result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, AbstractDownloadState)>(
"SELECT Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL AND Downloader != 'NexusDownloader+State'");
}
else
{
result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, AbstractDownloadState)>(
"SELECT Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL");
}
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
Archive = new Archive(result.Item4) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default,},
};
}
public async Task UpdatePendingDownload(ArchiveDownload ad)
{
await using var conn = await Open();
await conn.ExecuteAsync(
"UPDATE dbo.ArchiveDownloads SET IsFailed = @IsFailed, DownloadFinished = @DownloadFinished, Hash = @Hash, Size = @Size, FailMessage = @FailMessage WHERE Id = @Id",
new
{
Id = ad.Id,
IsFailed = ad.IsFailed,
DownloadFinished = ad.DownloadFinished,
Size = ad.Archive.Size,
Hash = ad.Archive.Hash,
FailMessage = ad.FailMessage
});
}
public async Task<int> EnqueueModListFilesForIndexing()
{
await using var conn = await Open();
return await conn.ExecuteAsync(@"
INSERT INTO dbo.ArchiveDownloads (Id, PrimaryKeyString, Hash, DownloadState, Size, Downloader)
SELECT DISTINCT NEWID(), mla.PrimaryKeyString, mla.Hash, mla.State, mla.Size, SUBSTRING(mla.PrimaryKeyString, 0, CHARINDEX('|', mla.PrimaryKeyString))
FROM [dbo].[ModListArchives] mla
LEFT JOIN dbo.ArchiveDownloads ad on mla.PrimaryKeyString = ad.PrimaryKeyString AND mla.Hash = ad.Hash
WHERE ad.PrimaryKeyString is null");
}
}
}

96
Wabbajack.Server/Services/ArchiveDownloader.cs Normal file
View File

@ -0,0 +1,96 @@
using System;
using System.Reflection.Metadata.Ecma335;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class ArchiveDownloader : AbstractService<ArchiveDownloader, int>
{
private SqlService _sql;
private ArchiveMaintainer _archiveMaintainer;
private NexusApiClient _nexusClient;
public ArchiveDownloader(ILogger<ArchiveDownloader> logger, AppSettings settings, SqlService sql, ArchiveMaintainer archiveMaintainer) : base(logger, settings, TimeSpan.FromMinutes(10))
{
_sql = sql;
_archiveMaintainer = archiveMaintainer;
}
public override async Task<int> Execute()
{
_nexusClient ??= await NexusApiClient.Get();
await _nexusClient.GetUserStatus();
int count = 0;
while (true)
{
bool useNexus = _nexusClient.HourlyRemaining > 25;
var nextDownload = await _sql.GetNextPendingDownload(useNexus);
if (nextDownload == null)
break;
if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash))
{
await nextDownload.Finish(_sql);
continue;
}
if (nextDownload.Archive.State is ManualDownloader.State)
{
await nextDownload.Finish(_sql);
continue;
}
try
{
_logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
await DownloadDispatcher.PrepareAll(new[] {nextDownload.Archive.State});
using var tempPath = new TempFile();
await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path);
var hash = await tempPath.Path.FileHashAsync();
if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
{
await nextDownload.Fail(_sql, "Invalid Hash");
continue;
}
if (nextDownload.Archive.Size != default &&
tempPath.Path.Size != nextDownload.Archive.Size)
{
await nextDownload.Fail(_sql, "Invalid Size");
continue;
}
nextDownload.Archive.Hash = hash;
nextDownload.Archive.Size = tempPath.Path.Size;
_logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await _archiveMaintainer.Ingest(tempPath.Path);
_logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Finish(_sql);
}
catch (Exception ex)
{
await nextDownload.Fail(_sql, ex.ToString());
}
count++;
}
return count;
}
}
}

4
Wabbajack.Server/Services/ModListDownloader.cs
View File

@ -123,6 +123,10 @@ namespace Wabbajack.Server.Services
}
}
_logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
var fc = await _sql.EnqueueModListFilesForIndexing();
_logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading");
return downloaded;
}
}

2
Wabbajack.Server/Startup.cs
View File

@ -62,6 +62,7 @@ namespace Wabbajack.Server
services.AddSingleton<ModListDownloader>();
services.AddSingleton<NonNexusDownloadValidator>();
services.AddSingleton<ListValidator>();
services.AddSingleton<ArchiveDownloader>();
services.AddMvc();
services.AddControllers()
@ -110,6 +111,7 @@ namespace Wabbajack.Server
app.UseService<NonNexusDownloadValidator>();
app.UseService<ListValidator>();
app.UseService<ArchiveDownloader>();
app.Use(next =>
{