Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Mirrored files, permissions, and download support

Browse Source
This commit is contained in:
Timothy Baldridge 2020-08-04 18:34:09 -06:00
parent abe30fe8a3
commit 7f892d26d7
19 changed files with 349 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Wabbajack.CLI/OptionsDefinition.cs
View File

@ -26,7 +26,9 @@ namespace Wabbajack.CLI
typeof(InlinedFileReport),
typeof(ExtractBSA),
typeof(PurgeNexusCache),
typeof(ForceHealing)
typeof(ForceHealing),
typeof(HashVariants),
typeof(ParseMeta)
};
}
}

1
Wabbajack.Common/Consts.cs
View File

@ -149,6 +149,7 @@ namespace Wabbajack.Common
public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2;
public static string ArchiveUpdatesCDNFolder = "archive_updates";
public static Uri WabbajackMirror = new Uri("https://wabbajack-mirror.b-cdn.net");
}
}

4
Wabbajack.Lib/ACompiler.cs
View File

@ -150,6 +150,9 @@ namespace Wabbajack.Lib
await using (var of = await ModListOutputFolder.Combine("modlist").Create())
ModList.ToJson(of);
await ModListOutputFolder.Combine("sig")
.WriteAllBytesAsync((await ModListOutputFolder.Combine("modlist").FileHashAsync()).ToArray());
await ClientAPI.SendModListDefinition(ModList);
await ModListOutputFile.DeleteAsync();
@ -157,6 +160,7 @@ namespace Wabbajack.Lib
await using (var fs = await ModListOutputFile.Create())
{
using var za = new ZipArchive(fs, ZipArchiveMode.Create);
await ModListOutputFolder.EnumerateFiles()
.DoProgress("Compressing ModList",
async f =>

2
Wabbajack.Lib/AuthorApi/Client.cs
View File

@ -39,7 +39,7 @@ namespace Wabbajack.Lib.AuthorApi
}
public async Task<CDNFileDefinition> GenerateFileDefinition(WorkQueue queue, AbsolutePath path, Action<string, Percent> progressFn)
public static async Task<CDNFileDefinition> GenerateFileDefinition(WorkQueue queue, AbsolutePath path, Action<string, Percent> progressFn)
{
IEnumerable<CDNFilePartDefinition> Blocks(AbsolutePath path)
{

28
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -93,8 +93,14 @@ namespace Wabbajack.Lib.Downloaders
public static async Task<bool> DownloadWithPossibleUpgrade(Archive archive, AbsolutePath destination)
{
var success = await Download(archive, destination);
if (success)
if (await Download(archive, destination))
{
await destination.FileHashCachedAsync();
return true;
}
if (await DownloadFromMirror(archive, destination))
{
await destination.FileHashCachedAsync();
return true;
@ -147,6 +153,24 @@ namespace Wabbajack.Lib.Downloaders
return true;
}
private static async Task<bool> DownloadFromMirror(Archive archive, AbsolutePath destination)
{
try
{
var newArchive =
new Archive(
new WabbajackCDNDownloader.State(new Uri($"{Consts.WabbajackMirror}{archive.Hash.ToHex()}")))
{
Hash = archive.Hash, Size = archive.Size, Name = archive.Name
};
return await Download(newArchive, destination);
}
catch (Exception ex)
{
return false;
}
}
private static async Task<bool> Download(Archive archive, AbsolutePath destination)
{
try

1
Wabbajack.Server.Test/ABuildServerSystemTest.cs
View File

@ -61,6 +61,7 @@ namespace Wabbajack.BuildServer.Test
_token = new CancellationTokenSource();
_task = _host.RunAsync(_token.Token);
Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080");
Consts.WabbajackMirror = new Uri("https://wabbajack-test.b-cdn.net");
await "ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllTextAsync(
"GoogleIDs:\nAllowedPrefixes:\n - http://localhost");

55
Wabbajack.Server.Test/MirroredFilesTests.cs Normal file
View File

@ -0,0 +1,55 @@
using System;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Server.Test
{
public class MirroredFilesTests : ABuildServerSystemTest
{
public MirroredFilesTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanUploadAndDownloadMirroredFiles()
{
var file = new TempFile();
await file.Path.WriteAllBytesAsync(RandomData(1024 * 1024 * 6));
var dataHash = await file.Path.FileHashAsync();
await Fixture.GetService<ArchiveMaintainer>().Ingest(file.Path);
Assert.True(Fixture.GetService<ArchiveMaintainer>().HaveArchive(dataHash));
var sql = Fixture.GetService<SqlService>();
await sql.UpsertMirroredFile(new MirroredFile
{
Created = DateTime.UtcNow,
Rationale = "Test File",
Hash = dataHash
});
var uploader = Fixture.GetService<MirrorUploader>();
Assert.Equal(1, await uploader.Execute());
var archive = new Archive(new HTTPDownloader.State(MakeURL(dataHash.ToString())))
{
Hash = dataHash,
Size = file.Path.Size
};
var file2 = new TempFile();
await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file2.Path);
}
}
}

2
Wabbajack.Server.Test/Wabbajack.Server.Test.csproj
View File

@ -2,7 +2,7 @@
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
<IsPackable>false</IsPackable>
</PropertyGroup>

41
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -677,17 +677,42 @@ SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[MirroredArchives](
[Hash] [bigint] NOT NULL,
[Created] [datetime] NOT NULL,
[Uploaded] [datetime] NULL,
[Rationale] [nvarchar](max) NOT NULL,
CONSTRAINT [PK_MirroredArchives] PRIMARY KEY CLUSTERED
(
[Hash] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
[Hash] [bigint] NOT NULL,
[Created] [datetime] NOT NULL,
[Uploaded] [datetime] NULL,
[Rationale] [nvarchar](max) NOT NULL,
[FailMessage] [nvarchar](max) NULL,
CONSTRAINT [PK_MirroredArchives] PRIMARY KEY CLUSTERED
(
[Hash] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
/****** Object: Table [dbo].[GameMetadata] Script Date: 8/3/2020 8:39:33 PM ******/
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[GameMetadata](
[NexusGameId] [bigint] NULL,
[WabbajackName] [nvarchar](50) NOT NULL,
CONSTRAINT [PK_GameMetadata] PRIMARY KEY CLUSTERED
(
[WabbajackName] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
)
GO
CREATE NONCLUSTERED INDEX [IDX_GameAndName-20200804-164236] ON [dbo].[GameMetadata]
(
[NexusGameId] ASC,
[WabbajackName] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
GO
/****** Object: StoredProcedure [dbo].[MergeAllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/

2
Wabbajack.Server/Controllers/AuthoredFiles.cs
View File

@ -106,7 +106,7 @@ namespace Wabbajack.BuildServer.Controllers
private async Task<FtpClient> GetBunnyCdnFtpClient()
{
var info = await Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
var info = await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles);
var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
await client.ConnectAsync();
return client;

18
Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs
View File

@ -1,9 +1,25 @@
namespace Wabbajack.Server.DTOs
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.Server.DTOs
{
public enum StorageSpace
{
AuthoredFiles,
Patches,
Mirrors
}
public class BunnyCdnFtpInfo
{
public string Username { get; set; }
public string Password { get; set; }
public string Hostname { get; set; }
public static async Task<BunnyCdnFtpInfo> GetCreds(StorageSpace space)
{
return (await Utils.FromEncryptedJson<Dictionary<string, BunnyCdnFtpInfo>>("bunnycdn"))[space.ToString()];
}
}
}

17
Wabbajack.Server/DTOs/MirroredFile.cs
View File

@ -1,5 +1,7 @@
using System;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.DTOs
{
@ -9,5 +11,20 @@ namespace Wabbajack.Server.DTOs
public DateTime Created { get; set; }
public DateTime? Uploaded { get; set; }
public string Rationale { get; set; }
public string FailMessage { get; set; }
public async Task Finish(SqlService sql)
{
Uploaded = DateTime.UtcNow;
await sql.UpsertMirroredFile(this);
}
public async Task Fail(SqlService sql, string message)
{
Uploaded = DateTime.UtcNow;
FailMessage = message;
await sql.UpsertMirroredFile(this);
}
}
}

18
Wabbajack.Server/DataLayer/MirroredFiles.cs
View File

@ -16,11 +16,12 @@ namespace Wabbajack.Server.DataLayer
public async Task<MirroredFile> GetNextMirroredFile()
{
await using var conn = await Open();
var results = await conn.QueryFirstOrDefaultAsync<(Hash, DateTime, DateTime, string)>(
"SELECT Hash, Created, Uploaded, Rationale from dbo.MirroredArchives WHERE Uploaded IS NULL");
var result = await conn.QueryFirstOrDefaultAsync<(Hash, DateTime, DateTime, string, string)>(
"SELECT Hash, Created, Uploaded, Rationale, FailMessage from dbo.MirroredArchives WHERE Uploaded IS NULL");
if (result == default) return null;
return new MirroredFile
{
Hash = results.Item1, Created = results.Item2, Uploaded = results.Item3, Rationale = results.Item4
Hash = result.Item1, Created = result.Item2, Uploaded = result.Item3, Rationale = result.Item4, FailMessage = result.Item5
};
}
@ -37,8 +38,15 @@ namespace Wabbajack.Server.DataLayer
await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash", new {file.Hash}, trans);
await conn.ExecuteAsync(
"INSERT INTO dbo.MirroredArchives (Hash, Created, Updated, Rationale) VALUES (@Hash, @Created, @Updated, @Rationale)",
file, trans);
"INSERT INTO dbo.MirroredArchives (Hash, Created, Uploaded, Rationale, FailMessage) VALUES (@Hash, @Created, @Uploaded, @Rationale, @FailMessage)",
new
{
Hash = file.Hash,
Created = file.Created,
Uploaded = file.Uploaded,
Rationale = file.Rationale,
FailMessage = file.FailMessage
}, trans);
await trans.CommitAsync();
}

33
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -146,5 +146,38 @@ namespace Wabbajack.Server.DataLayer
await tx.CommitAsync();
}
public async Task UpdateGameMetadata()
{
await using var conn = await Open();
var existing = (await conn.QueryAsync<string>("SELECT WabbajackName FROM dbo.GameMetadata")).ToHashSet();
var missing = GameRegistry.Games.Values.Where(g => !existing.Contains(g.Game.ToString())).ToList();
foreach (var add in missing.Where(g => g.NexusGameId != 0))
{
await conn.ExecuteAsync(
"INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, WabbajackName)",
new {NexusGameId = add.NexusGameId, WabbajackName = add.ToString()});
}
}
public async Task SetNexusPermission(Game game, long modId, HTMLInterface.PermissionValue perm)
{
await using var conn = await Open();
var tx = await conn.BeginTransactionAsync();
await conn.ExecuteAsync("DELETE FROM NexusModPermissions WHERE GameID = @GameID AND ModID = @ModID", new
{
GameID = game.MetaData().NexusGameId,
ModID = modId
},
transaction:tx);
await conn.ExecuteAsync(
"INSERT INTO NexusModPermissions (NexusGameID, ModID, Permissions) VALUES (@NexusGameID, @ModID, @Permissions)",
new {NexusGameID = game.MetaData().NexusGameId, ModID = modId, Permissions = (int)perm}, tx);
await tx.CommitAsync();
}
}
}

13
Wabbajack.Server/Services/AbstractService.cs
View File

@ -17,6 +17,7 @@ namespace Wabbajack.Server.Services
private TimeSpan _delay;
protected ILogger<TP> _logger;
protected QuickSync _quickSync;
private bool _isSetup;
public AbstractService(ILogger<TP> logger, AppSettings settings, QuickSync quickSync, TimeSpan delay)
{
@ -24,14 +25,26 @@ namespace Wabbajack.Server.Services
_delay = delay;
_logger = logger;
_quickSync = quickSync;
_isSetup = false;
}
public virtual async Task Setup()
{
}
public void Start()
{
if (_settings.RunBackEndJobs)
{
Task.Run(async () =>
{
await Setup();
_isSetup = true;
while (true)
{
await _quickSync.ResetToken<TP>();

111
Wabbajack.Server/Services/MirrorUploader.cs Normal file
View File

@ -0,0 +1,111 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.BuildServer.Controllers;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.AuthorApi;
using Wabbajack.Lib.FileUploader;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class MirrorUploader : AbstractService<MirrorUploader, int>
{
private SqlService _sql;
private ArchiveMaintainer _archives;
public MirrorUploader(ILogger<MirrorUploader> logger, AppSettings settings, SqlService sql, QuickSync quickSync, ArchiveMaintainer archives) : base(logger, settings, quickSync, TimeSpan.FromHours(1))
{
_sql = sql;
_archives = archives;
}
public override async Task<int> Execute()
{
int uploaded = 0;
TOP:
var toUpload = await _sql.GetNextMirroredFile();
if (toUpload == default) return uploaded;
uploaded += 1;
try
{
using var queue = new WorkQueue();
if (_archives.TryGetPath(toUpload.Hash, out var path))
{
_logger.LogInformation($"Uploading mirror file {toUpload.Hash} {path.Size.FileSizeToString()}");
var definition = await Client.GenerateFileDefinition(queue, path, (s, percent) => { });
var creds = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
using (var client = await GetClient(creds))
{
await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}");
await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}/parts");
}
string MakePath(long idx)
{
return $"{definition.Hash.ToHex()}/parts/{idx}";
}
await definition.Parts.PMap(queue, async part =>
{
_logger.LogInformation($"Uploading mirror part ({part.Index}/{definition.Parts.Length})");
var name = MakePath(part.Index);
var buffer = new byte[part.Size];
await using (var fs = await path.OpenShared())
{
fs.Position = part.Offset;
await fs.ReadAsync(buffer);
}
using var client = await GetClient(creds);
await client.UploadAsync(new MemoryStream(buffer), name);
});
using (var client = await GetClient(creds))
{
_logger.LogInformation($"Finishing mirror upload");
await using var ms = new MemoryStream();
await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
{
definition.ToJson(gz);
}
ms.Position = 0;
await client.UploadAsync(ms, $"{definition.Hash.ToHex()}/definition.json.gz");
}
await toUpload.Finish(_sql);
}
else
{
await toUpload.Fail(_sql, "Archive not found");
}
}
catch (Exception ex)
{
_logger.LogInformation($"{toUpload.Created} {toUpload.Uploaded}");
_logger.LogError(ex, "Error uploading");
await toUpload.Fail(_sql, ex.ToString());
}
goto TOP;
}
private static async Task<FtpClient> GetClient(BunnyCdnFtpInfo creds)
{
var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password));
await ftpClient.ConnectAsync();
return ftpClient;
}
}
}

37
Wabbajack.Server/Services/NexusPermissionsUpdater.cs
View File

@ -24,6 +24,8 @@ namespace Wabbajack.Server.Services
public override async Task<int> Execute()
{
await _sql.UpdateGameMetadata();
var permissions = await _sql.GetNexusPermissions();
var data = await _sql.ModListArchives();
@ -33,38 +35,31 @@ namespace Wabbajack.Server.Services
_logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods");
using var queue = new WorkQueue();
using var queue = new WorkQueue(2);
var results = await nexusArchives.PMap(queue, async archive =>
{
var permissions = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID);
return (archive.Game, archive.ModID, permissions);
});
var prev = await _sql.GetNexusPermissions();
var updated = 0;
foreach (var result in results)
await nexusArchives.PMap(queue, async archive =>
{
if (permissions.TryGetValue((result.Game, result.ModID), out var oldPermission))
var result = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID);
await _sql.SetNexusPermission(archive.Game, archive.ModID, result);
if (prev.TryGetValue((archive.Game, archive.ModID), out var oldPermission))
{
if (oldPermission != result.permissions)
if (oldPermission != result)
{
await _discord.Send(Channel.Spam,
new DiscordMessage {
Content = $"Permissions status of {result.Game} {result.ModID} was {oldPermission} is now {result.permissions} "
Content = $"Permissions status of {archive.Game} {archive.ModID} was {oldPermission} is now {result}"
});
await _sql.PurgeNexusCache(result.ModID);
updated += 1;
await _sql.PurgeNexusCache(archive.ModID);
await _quickSync.Notify<ListValidator>();
}
}
}
});
await _sql.SetNexusPermissions(results);
if (updated > 0)
await _quickSync.Notify<ListValidator>();
return updated;
return 1;
}
}
}

2
Wabbajack.Server/Services/PatchBuilder.cs
View File

@ -230,7 +230,7 @@ namespace Wabbajack.Server.Services
private async Task<FtpClient> GetBunnyCdnFtpClient()
{
var info = await Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
var info = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches);
var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
await client.ConnectAsync();
return client;

2
Wabbajack.Server/Startup.cs
View File

@ -69,6 +69,7 @@ namespace Wabbajack.Server
services.AddSingleton<PatchBuilder>();
services.AddSingleton<CDNMirrorList>();
services.AddSingleton<NexusPermissionsUpdater>();
services.AddSingleton<MirrorUploader>();
services.AddMvc();
services.AddControllers()
@ -125,6 +126,7 @@ namespace Wabbajack.Server
app.UseService<PatchBuilder>();
app.UseService<CDNMirrorList>();
app.UseService<NexusPermissionsUpdater>();
app.UseService<MirrorUploader>();
app.Use(next =>
{