Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updates to smooth out server update features

Browse Source
This commit is contained in:
Timothy Baldridge 2020-11-01 17:30:49 -07:00
parent 65a908aa1e
commit f80ae8b142
10 changed files with 135 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Wabbajack.Server.Test/MirroredFilesTests.cs
View File

@ -50,6 +50,13 @@ namespace Wabbajack.Server.Test
var file2 = new TempFile();
await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file2.Path);
}
[Fact]
public async Task CanQueueFiles()
{
var service = Fixture.GetService<MirrorQueueService>();
Assert.Equal(1, await service.Execute());
}
}
}

21
Wabbajack.Server.Test/ModFileTests.cs
View File

@ -1,10 +1,12 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Xunit;
using Xunit.Abstractions;
@ -22,23 +24,38 @@ namespace Wabbajack.Server.Test
public async Task CanGetDownloadStates()
{
var sql = Fixture.GetService<SqlService>();
var hash = Hash.FromBase64("eSIyd+KOG3s=");
var archive =
new Archive(new WabbajackCDNDownloader.State(new Uri(
"https://wabbajack.b-cdn.net/WABBAJACK_TEST_FILE.zip_a1a3e961-5c0b-4ccf-84b4-7aa437d9640d")))
{
Size = 20, Hash = Hash.FromBase64("eSIyd+KOG3s=")
Size = 20, Hash = hash
};
await sql.EnqueueDownload(archive);
await sql.UpsertMirroredFile(new MirroredFile()
{
Created = DateTime.UtcNow,
Uploaded = DateTime.UtcNow,
Hash = hash,
Rationale = "Test File"
});
var dld = await sql.GetNextPendingDownload();
await dld.Finish(sql);
var state = await ClientAPI.InferDownloadState(archive.Hash);
Assert.Equal(archive.State.GetMetaIniString(), state!.GetMetaIniString());
var archives = await (await ClientAPI.GetClient()).GetJsonAsync<Archive[]>(
$"{Consts.WabbajackBuildServerUri}mod_files/by_hash/{hash.ToHex()}");
Assert.True(archives.Length >= 2);
Assert.NotNull(archives.FirstOrDefault(a => a.State is WabbajackCDNDownloader.State));
await sql.DeleteMirroredFile(hash);
}
}
}

17
Wabbajack.Server/DataLayer/ArchiveDownloads.cs
View File

@ -241,10 +241,19 @@ namespace Wabbajack.Server.DataLayer
var files = (await conn.QueryAsync<(long, Hash, AbstractDownloadState)>(
@"SELECT Size, Hash, DownloadState from dbo.ArchiveDownloads WHERE Hash = @Hash AND IsFailed = 0 AND DownloadFinished IS NOT NULL ORDER BY DownloadFinished DESC",
new {Hash = hash})
);
return files.Select(e =>
new Archive(e.Item3) {Size = e.Item1, Hash = e.Item2}
).ToArray();
).Select(e =>
new Archive(e.Item3) {Size = e.Item1, Hash = e.Item2}
).ToList();
if (await HaveMirror(hash) && files.Count > 0)
{
var ffile = files.First();
var url = new Uri($"https://{(await _mirrorCreds).Username}.b-cdn.net/{hash.ToHex()}");
files.Add(new Archive(
new WabbajackCDNDownloader.State(url)) {Hash = hash, Size = ffile.Size, Name = ffile.Name});
}
return files.ToArray();
}
}
}

48
Wabbajack.Server/DataLayer/MirroredFiles.cs
View File

@ -49,6 +49,13 @@ namespace Wabbajack.Server.DataLayer
await trans.CommitAsync();
}
public async Task DeleteMirroredFile(Hash hash)
{
await using var conn = await Open();
await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash",
new {Hash = hash});
}
public async Task InsertAllNexusMirrors()
{
var permissions = (await GetNexusPermissions()).Where(p => p.Value == HTMLInterface.PermissionValue.Yes);
@ -82,5 +89,46 @@ namespace Wabbajack.Server.DataLayer
return await conn.QueryFirstOrDefaultAsync<Hash>("SELECT Hash FROM dbo.MirroredArchives WHERE Hash = @Hash",
new {Hash = hash}) != default;
}
public async Task QueueMirroredFiles()
{
await using var conn = await Open();
await conn.ExecuteAsync(@"
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT hs.Hash, GETUTCDATE(), 'File has re-upload permissions on the Nexus' FROM
(SELECT DISTINCT ad.Hash FROM dbo.NexusModPermissions p
INNER JOIN GameMetadata md on md.NexusGameId = p.NexusGameID
INNER JOIN dbo.ArchiveDownloads ad on ad.PrimaryKeyString like 'NexusDownloader+State|'+md.WabbajackName+'|'+CAST(p.ModID as nvarchar)+'|%'
WHERE p.Permissions = 1
AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
) hs
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT DISTINCT Hash, GETUTCDATE(), 'File is hosted on GitHub'
FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%github.com/%'
AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT DISTINCT Hash, GETUTCDATE(), 'File license allows uploading to any Non-nexus site'
FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%enbdev.com/%'
AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT DISTINCT Hash, GETUTCDATE(), 'DynDOLOD file' /*, Name*/
from dbo.ModListArchives mla WHERE Name like '%DynDoLOD%standalone%'
and Hash not in (select Hash from dbo.MirroredArchives)
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT DISTINCT Hash, GETUTCDATE(), 'Distribution allowed by author' /*, Name*/
from dbo.ModListArchives mla WHERE Name like '%particle%patch%'
and Hash not in (select Hash from dbo.MirroredArchives)
");
}
}
}

3
Wabbajack.Server/DataLayer/SqlService.cs
View File

@ -1,16 +1,19 @@
using System.Data.SqlClient;
using System.Threading.Tasks;
using Wabbajack.BuildServer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
private AppSettings _settings;
private Task<BunnyCdnFtpInfo> _mirrorCreds;
public SqlService(AppSettings settings)
{
_settings = settings;
_mirrorCreds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
}

28
Wabbajack.Server/Services/MirrorQueueService.cs Normal file
View File

@ -0,0 +1,28 @@
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.Services
{
public class MirrorQueueService : AbstractService<MirrorQueueService, int>
{
private DiscordWebHook _discord;
private SqlService _sql;
public MirrorQueueService(ILogger<MirrorQueueService> logger, AppSettings settings, QuickSync quickSync,
DiscordWebHook discordWebHook, SqlService sqlService) :
base(logger, settings, quickSync, TimeSpan.FromMinutes(5))
{
_discord = discordWebHook;
_sql = sqlService;
}
public override async Task<int> Execute()
{
await _sql.QueueMirroredFiles();
return 1;
}
}
}

11
Wabbajack.Server/Services/MirrorUploader.cs
View File

@ -23,11 +23,14 @@ namespace Wabbajack.Server.Services
{
private SqlService _sql;
private ArchiveMaintainer _archives;
private DiscordWebHook _discord;
public MirrorUploader(ILogger<MirrorUploader> logger, AppSettings settings, SqlService sql, QuickSync quickSync, ArchiveMaintainer archives) : base(logger, settings, quickSync, TimeSpan.FromHours(1))
public MirrorUploader(ILogger<MirrorUploader> logger, AppSettings settings, SqlService sql, QuickSync quickSync, ArchiveMaintainer archives, DiscordWebHook discord)
: base(logger, settings, quickSync, TimeSpan.FromHours(1))
{
_sql = sql;
_archives = archives;
_discord = discord;
}
public override async Task<int> Execute()
@ -61,6 +64,12 @@ namespace Wabbajack.Server.Services
goto TOP;
}
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Uploading {toUpload.Hash} - {toUpload.Created} because {toUpload.Rationale}"
});
var definition = await Client.GenerateFileDefinition(queue, path, (s, percent) => { });
using (var client = await GetClient(creds))

2
Wabbajack.Server/Startup.cs
View File

@ -70,6 +70,7 @@ namespace Wabbajack.Server
services.AddSingleton<CDNMirrorList>();
services.AddSingleton<NexusPermissionsUpdater>();
services.AddSingleton<MirrorUploader>();
services.AddSingleton<MirrorQueueService>();
services.AddMvc();
services.AddControllers()
@ -127,6 +128,7 @@ namespace Wabbajack.Server
app.UseService<CDNMirrorList>();
app.UseService<NexusPermissionsUpdater>();
app.UseService<MirrorUploader>();
app.UseService<MirrorQueueService>();
app.Use(next =>
{

2
Wabbajack.Test/EndToEndTests.cs
View File

@ -63,7 +63,7 @@ namespace Wabbajack.Test
DownloadAndInstall(Game.SkyrimSpecialEdition, 4783, "Frost Armor UNP"),
DownloadAndInstall(Game.SkyrimSpecialEdition, 32359, "Frost Armor HDT"),
DownloadAndInstall("https://github.com/ShikyoKira/Project-New-Reign---Nemesis-Main/releases/download/v0.84-beta/Nemesis.Unlimited.Behavior.Engine.v0.84-beta.rar", "Nemesis.Unlimited.Behavior.Engine.v0.84-beta.rar", "Nemesis"),
DownloadAndInstall(Game.Fallout4, 40136, "RAR test File"));
DownloadAndInstall(Game.Fallout4, 40136, "RAR test File")); // ShouldPullFrom Mirror
// We're going to fully patch this mod from another source.
await modfiles[3].Download.DeleteAsync();

8
Wabbajack.Test/TestUtils.cs
View File

@ -172,13 +172,13 @@ namespace Wabbajack.Test
}
await DownloadsPath.Combine(name + Consts.MetaFileExtension).WriteAllLinesAsync(
"[General]",
"manualURL=<TESTING>"
);
"[General]",
"manualURL=<TESTING>"
);
return name;
}
public async Task VerifyInstalledFile(string mod, string file)
{
var src = SourcePath.Combine((string)Consts.MO2ModFolderName, mod, file);