Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1008 from wabbajack-tools/mirrored-files

Browse Source 
Mirrored files
This commit is contained in:
Timothy Baldridge 2020-08-05 18:18:04 -07:00 committed by GitHub
commit e14501afdf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 526 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Wabbajack.CLI/OptionsDefinition.cs
View File

@ -26,7 +26,9 @@ namespace Wabbajack.CLI
typeof(InlinedFileReport),
typeof(ExtractBSA),
typeof(PurgeNexusCache),
typeof(ForceHealing)
typeof(ForceHealing),
typeof(HashVariants),
typeof(ParseMeta)
};
}
}

24
Wabbajack.CLI/Verbs/HashVariants.cs Normal file
View File

@ -0,0 +1,24 @@
using System;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Common;
namespace Wabbajack.CLI.Verbs
{
[Verb("hash-variants", HelpText = "Print all the known variants (formats) of a hash")]
public class HashVariants : AVerb
{
[Option('i', "input", Required = true, HelpText = "Input Hash")]
public string Input { get; set; } = "";
protected override async Task<ExitCode> Run()
{
var hash = Hash.Interpret(Input);
Console.WriteLine($"Base64: {hash.ToBase64()}");
Console.WriteLine($"Hex: {hash.ToHex()}");
Console.WriteLine($"Long: {(long)hash}");
Console.WriteLine($"ULong (uncommon): {(ulong)hash}");
return ExitCode.Ok;
}
}
}

28
Wabbajack.CLI/Verbs/ParseMeta.cs Normal file
View File

@ -0,0 +1,28 @@
using System;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.CLI.Verbs
{
[Verb("parse-meta", HelpText = "Parse a .meta file, figure out the download state and print it")]
public class ParseMeta : AVerb
{
[Option('i', "input", Required = true, HelpText = "Input meta file to parse")]
public string Input { get; set; } = "";
protected override async Task<ExitCode> Run()
{
var meta = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(((AbsolutePath)Input).LoadIniFile());
if (meta == null)
{
Console.WriteLine("Cannot resolve meta!");
return ExitCode.Error;
}
Console.WriteLine($"PrimaryKeyString : {meta.PrimaryKeyString}");
return ExitCode.Ok;
}
}
}

2
Wabbajack.Common/Consts.cs
View File

@ -148,7 +148,7 @@ namespace Wabbajack.Common
public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2;
public static string ArchiveUpdatesCDNFolder = "archive_updates";
public static Uri WabbajackMirror = new Uri("https://wabbajack-mirror.b-cdn.net");
}
}

10
Wabbajack.Common/Hash.cs
View File

@ -103,6 +103,16 @@ namespace Wabbajack.Common
{
return BitConverter.GetBytes(_code);
}
public static Hash Interpret(string input)
{
return input.Length switch
{
16 => FromHex(input),
12 when input.EndsWith('=') => FromBase64(input),
_ => FromLong(long.Parse(input))
};
}
}
public static partial class Utils

4
Wabbajack.Lib/ACompiler.cs
View File

@ -150,6 +150,9 @@ namespace Wabbajack.Lib
await using (var of = await ModListOutputFolder.Combine("modlist").Create())
ModList.ToJson(of);
await ModListOutputFolder.Combine("sig")
.WriteAllBytesAsync((await ModListOutputFolder.Combine("modlist").FileHashAsync()).ToArray());
await ClientAPI.SendModListDefinition(ModList);
await ModListOutputFile.DeleteAsync();
@ -157,6 +160,7 @@ namespace Wabbajack.Lib
await using (var fs = await ModListOutputFile.Create())
{
using var za = new ZipArchive(fs, ZipArchiveMode.Create);
await ModListOutputFolder.EnumerateFiles()
.DoProgress("Compressing ModList",
async f =>

2
Wabbajack.Lib/AuthorApi/Client.cs
View File

@ -39,7 +39,7 @@ namespace Wabbajack.Lib.AuthorApi
}
public async Task<CDNFileDefinition> GenerateFileDefinition(WorkQueue queue, AbsolutePath path, Action<string, Percent> progressFn)
public static async Task<CDNFileDefinition> GenerateFileDefinition(WorkQueue queue, AbsolutePath path, Action<string, Percent> progressFn)
{
IEnumerable<CDNFilePartDefinition> Blocks(AbsolutePath path)
{

28
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -93,8 +93,14 @@ namespace Wabbajack.Lib.Downloaders
public static async Task<bool> DownloadWithPossibleUpgrade(Archive archive, AbsolutePath destination)
{
var success = await Download(archive, destination);
if (success)
if (await Download(archive, destination))
{
await destination.FileHashCachedAsync();
return true;
}
if (await DownloadFromMirror(archive, destination))
{
await destination.FileHashCachedAsync();
return true;
@ -147,6 +153,24 @@ namespace Wabbajack.Lib.Downloaders
return true;
}
private static async Task<bool> DownloadFromMirror(Archive archive, AbsolutePath destination)
{
try
{
var newArchive =
new Archive(
new WabbajackCDNDownloader.State(new Uri($"{Consts.WabbajackMirror}{archive.Hash.ToHex()}")))
{
Hash = archive.Hash, Size = archive.Size, Name = archive.Name
};
return await Download(newArchive, destination);
}
catch (Exception ex)
{
return false;
}
}
private static async Task<bool> Download(Archive archive, AbsolutePath destination)
{
try

1
Wabbajack.Server.Test/ABuildServerSystemTest.cs
View File

@ -61,6 +61,7 @@ namespace Wabbajack.BuildServer.Test
_token = new CancellationTokenSource();
_task = _host.RunAsync(_token.Token);
Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080");
Consts.WabbajackMirror = new Uri("https://wabbajack-test.b-cdn.net");
await "ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllTextAsync(
"GoogleIDs:\nAllowedPrefixes:\n - http://localhost");

55
Wabbajack.Server.Test/MirroredFilesTests.cs Normal file
View File

@ -0,0 +1,55 @@
using System;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Server.Test
{
public class MirroredFilesTests : ABuildServerSystemTest
{
public MirroredFilesTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanUploadAndDownloadMirroredFiles()
{
var file = new TempFile();
await file.Path.WriteAllBytesAsync(RandomData(1024 * 1024 * 6));
var dataHash = await file.Path.FileHashAsync();
await Fixture.GetService<ArchiveMaintainer>().Ingest(file.Path);
Assert.True(Fixture.GetService<ArchiveMaintainer>().HaveArchive(dataHash));
var sql = Fixture.GetService<SqlService>();
await sql.UpsertMirroredFile(new MirroredFile
{
Created = DateTime.UtcNow,
Rationale = "Test File",
Hash = dataHash
});
var uploader = Fixture.GetService<MirrorUploader>();
Assert.Equal(1, await uploader.Execute());
var archive = new Archive(new HTTPDownloader.State(MakeURL(dataHash.ToString())))
{
Hash = dataHash,
Size = file.Path.Size
};
var file2 = new TempFile();
await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file2.Path);
}
}
}

2
Wabbajack.Server.Test/ModlistUpdater.cs
View File

@ -70,7 +70,7 @@ namespace Wabbajack.Server.Test
await Assert.ThrowsAsync<HttpException>(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
Assert.True(await patcher.Execute() > 1);
Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_updates/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
}
[Fact]

2
Wabbajack.Server.Test/Wabbajack.Server.Test.csproj
View File

@ -2,7 +2,7 @@
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
<IsPackable>false</IsPackable>
</PropertyGroup>

44
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -669,6 +669,50 @@ CONSTRAINT [PK_NexusModPermissions] PRIMARY KEY CLUSTERED
) ON [PRIMARY]
GO
/****** Object: Table [dbo].[MirroredArchives] Script Date: 8/3/2020 8:39:33 PM ******/
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[MirroredArchives](
[Hash] [bigint] NOT NULL,
[Created] [datetime] NOT NULL,
[Uploaded] [datetime] NULL,
[Rationale] [nvarchar](max) NOT NULL,
[FailMessage] [nvarchar](max) NULL,
CONSTRAINT [PK_MirroredArchives] PRIMARY KEY CLUSTERED
(
[Hash] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
/****** Object: Table [dbo].[GameMetadata] Script Date: 8/3/2020 8:39:33 PM ******/
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[GameMetadata](
[NexusGameId] [bigint] NULL,
[WabbajackName] [nvarchar](50) NOT NULL,
CONSTRAINT [PK_GameMetadata] PRIMARY KEY CLUSTERED
(
[WabbajackName] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
)
GO
CREATE NONCLUSTERED INDEX [IDX_GameAndName-20200804-164236] ON [dbo].[GameMetadata]
(
[NexusGameId] ASC,
[WabbajackName] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
GO
/****** Object: StoredProcedure [dbo].[MergeAllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/

2
Wabbajack.Server/Controllers/AuthoredFiles.cs
View File

@ -106,7 +106,7 @@ namespace Wabbajack.BuildServer.Controllers
private async Task<FtpClient> GetBunnyCdnFtpClient()
{
var info = await Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
var info = await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles);
var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
await client.ConnectAsync();
return client;

5
Wabbajack.Server/Controllers/ModUpgrade.cs
View File

@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
@ -20,6 +21,7 @@ namespace Wabbajack.BuildServer.Controllers
private DiscordWebHook _discord;
private AppSettings _settings;
private QuickSync _quickSync;
private Task<BunnyCdnFtpInfo> _creds;
public ModUpgrade(ILogger<ModUpgrade> logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync, AppSettings settings)
{
@ -28,6 +30,7 @@ namespace Wabbajack.BuildServer.Controllers
_discord = discord;
_settings = settings;
_quickSync = quickSync;
_creds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches);
}
[HttpPost]
@ -93,7 +96,7 @@ namespace Wabbajack.BuildServer.Controllers
await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id);
return
Ok(
$"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{Consts.ArchiveUpdatesCDNFolder}/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
$"https://{(await _creds).Username}.b-cdn.net/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
}
_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed");

18
Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs
View File

@ -1,9 +1,25 @@
namespace Wabbajack.Server.DTOs
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.Server.DTOs
{
public enum StorageSpace
{
AuthoredFiles,
Patches,
Mirrors
}
public class BunnyCdnFtpInfo
{
public string Username { get; set; }
public string Password { get; set; }
public string Hostname { get; set; }
public static async Task<BunnyCdnFtpInfo> GetCreds(StorageSpace space)
{
return (await Utils.FromEncryptedJson<Dictionary<string, BunnyCdnFtpInfo>>("bunnycdn"))[space.ToString()];
}
}
}

30
Wabbajack.Server/DTOs/MirroredFile.cs Normal file
View File

@ -0,0 +1,30 @@
using System;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.DTOs
{
public class MirroredFile
{
public Hash Hash { get; set; }
public DateTime Created { get; set; }
public DateTime? Uploaded { get; set; }
public string Rationale { get; set; }
public string FailMessage { get; set; }
public async Task Finish(SqlService sql)
{
Uploaded = DateTime.UtcNow;
await sql.UpsertMirroredFile(this);
}
public async Task Fail(SqlService sql, string message)
{
Uploaded = DateTime.UtcNow;
FailMessage = message;
await sql.UpsertMirroredFile(this);
}
}
}

6
Wabbajack.Server/DataLayer/ArchiveDownloads.cs
View File

@ -57,6 +57,12 @@ namespace Wabbajack.Server.DataLayer
return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, PrimaryKeyString FROM ArchiveDownloads")).ToHashSet();
}
public async Task<HashSet<(Hash Hash, AbstractDownloadState State)>> GetAllArchiveDownloadStates()
{
await using var conn = await Open();
return (await conn.QueryAsync<(Hash, AbstractDownloadState)>("SELECT Hash, DownloadState FROM ArchiveDownloads")).ToHashSet();
}
public async Task<ArchiveDownload> GetArchiveDownload(Guid id)
{

79
Wabbajack.Server/DataLayer/MirroredFiles.cs Normal file
View File

@ -0,0 +1,79 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using AngleSharp.Common;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task<MirroredFile> GetNextMirroredFile()
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Hash, DateTime, DateTime, string, string)>(
"SELECT Hash, Created, Uploaded, Rationale, FailMessage from dbo.MirroredArchives WHERE Uploaded IS NULL");
if (result == default) return null;
return new MirroredFile
{
Hash = result.Item1, Created = result.Item2, Uploaded = result.Item3, Rationale = result.Item4, FailMessage = result.Item5
};
}
public async Task<HashSet<Hash>> GetAllMirroredHashes()
{
await using var conn = await Open();
return (await conn.QueryAsync<Hash>("SELECT Hash FROM dbo.MirroredArchives")).ToHashSet();
}
public async Task UpsertMirroredFile(MirroredFile file)
{
await using var conn = await Open();
await using var trans = await conn.BeginTransactionAsync();
await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash", new {file.Hash}, trans);
await conn.ExecuteAsync(
"INSERT INTO dbo.MirroredArchives (Hash, Created, Uploaded, Rationale, FailMessage) VALUES (@Hash, @Created, @Uploaded, @Rationale, @FailMessage)",
new
{
Hash = file.Hash,
Created = file.Created,
Uploaded = file.Uploaded,
Rationale = file.Rationale,
FailMessage = file.FailMessage
}, trans);
await trans.CommitAsync();
}
public async Task InsertAllNexusMirrors()
{
var permissions = (await GetNexusPermissions()).Where(p => p.Value == HTMLInterface.PermissionValue.Yes);
var downloads = (await GetAllArchiveDownloadStates()).Where(a => a.State is NexusDownloader.State).ToDictionary(a =>
{
var nd = (NexusDownloader.State)a.State;
return (nd.Game, nd.ModID);
}, a => a.Hash);
var existing = await GetAllMirroredHashes();
foreach (var (key, _) in permissions)
{
if (!downloads.TryGetValue(key, out var hash)) continue;
if (existing.Contains(hash)) continue;
await UpsertMirroredFile(new MirroredFile
{
Hash = hash,
Created = DateTime.UtcNow,
Rationale =
$"Mod ({key.Item1} {key.Item2}) has allowed re-upload permissions on the Nexus"
});
}
}
}
}

33
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -146,5 +146,38 @@ namespace Wabbajack.Server.DataLayer
await tx.CommitAsync();
}
public async Task UpdateGameMetadata()
{
await using var conn = await Open();
var existing = (await conn.QueryAsync<string>("SELECT WabbajackName FROM dbo.GameMetadata")).ToHashSet();
var missing = GameRegistry.Games.Values.Where(g => !existing.Contains(g.Game.ToString())).ToList();
foreach (var add in missing.Where(g => g.NexusGameId != 0))
{
await conn.ExecuteAsync(
"INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, @WabbajackName)",
new {NexusGameId = add.NexusGameId, WabbajackName = add.Game.ToString()});
}
}
public async Task SetNexusPermission(Game game, long modId, HTMLInterface.PermissionValue perm)
{
await using var conn = await Open();
var tx = await conn.BeginTransactionAsync();
await conn.ExecuteAsync("DELETE FROM NexusModPermissions WHERE GameID = @GameID AND ModID = @ModID", new
{
GameID = game.MetaData().NexusGameId,
ModID = modId
},
transaction:tx);
await conn.ExecuteAsync(
"INSERT INTO NexusModPermissions (NexusGameID, ModID, Permissions) VALUES (@NexusGameID, @ModID, @Permissions)",
new {NexusGameID = game.MetaData().NexusGameId, ModID = modId, Permissions = (int)perm}, tx);
await tx.CommitAsync();
}
}
}

14
Wabbajack.Server/Services/AbstractService.cs
View File

@ -17,6 +17,7 @@ namespace Wabbajack.Server.Services
private TimeSpan _delay;
protected ILogger<TP> _logger;
protected QuickSync _quickSync;
private bool _isSetup;
public AbstractService(ILogger<TP> logger, AppSettings settings, QuickSync quickSync, TimeSpan delay)
{
@ -24,19 +25,32 @@ namespace Wabbajack.Server.Services
_delay = delay;
_logger = logger;
_quickSync = quickSync;
_isSetup = false;
}
public virtual async Task Setup()
{
}
public void Start()
{
if (_settings.RunBackEndJobs)
{
Task.Run(async () =>
{
await Setup();
_isSetup = true;
while (true)
{
await _quickSync.ResetToken<TP>();
try
{
_logger.LogInformation($"Running: {GetType().Name}");
await Execute();
}
catch (Exception ex)

111
Wabbajack.Server/Services/MirrorUploader.cs Normal file
View File

@ -0,0 +1,111 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.BuildServer.Controllers;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.AuthorApi;
using Wabbajack.Lib.FileUploader;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class MirrorUploader : AbstractService<MirrorUploader, int>
{
private SqlService _sql;
private ArchiveMaintainer _archives;
public MirrorUploader(ILogger<MirrorUploader> logger, AppSettings settings, SqlService sql, QuickSync quickSync, ArchiveMaintainer archives) : base(logger, settings, quickSync, TimeSpan.FromHours(1))
{
_sql = sql;
_archives = archives;
}
public override async Task<int> Execute()
{
int uploaded = 0;
TOP:
var toUpload = await _sql.GetNextMirroredFile();
if (toUpload == default) return uploaded;
uploaded += 1;
try
{
using var queue = new WorkQueue();
if (_archives.TryGetPath(toUpload.Hash, out var path))
{
_logger.LogInformation($"Uploading mirror file {toUpload.Hash} {path.Size.FileSizeToString()}");
var definition = await Client.GenerateFileDefinition(queue, path, (s, percent) => { });
var creds = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
using (var client = await GetClient(creds))
{
await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}");
await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}/parts");
}
string MakePath(long idx)
{
return $"{definition.Hash.ToHex()}/parts/{idx}";
}
await definition.Parts.PMap(queue, async part =>
{
_logger.LogInformation($"Uploading mirror part ({part.Index}/{definition.Parts.Length})");
var name = MakePath(part.Index);
var buffer = new byte[part.Size];
await using (var fs = await path.OpenShared())
{
fs.Position = part.Offset;
await fs.ReadAsync(buffer);
}
using var client = await GetClient(creds);
await client.UploadAsync(new MemoryStream(buffer), name);
});
using (var client = await GetClient(creds))
{
_logger.LogInformation($"Finishing mirror upload");
await using var ms = new MemoryStream();
await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
{
definition.ToJson(gz);
}
ms.Position = 0;
await client.UploadAsync(ms, $"{definition.Hash.ToHex()}/definition.json.gz");
}
await toUpload.Finish(_sql);
}
else
{
await toUpload.Fail(_sql, "Archive not found");
}
}
catch (Exception ex)
{
_logger.LogInformation($"{toUpload.Created} {toUpload.Uploaded}");
_logger.LogError(ex, "Error uploading");
await toUpload.Fail(_sql, ex.ToString());
}
goto TOP;
}
private static async Task<FtpClient> GetClient(BunnyCdnFtpInfo creds)
{
var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password));
await ftpClient.ConnectAsync();
return ftpClient;
}
}
}

47
Wabbajack.Server/Services/NexusPermissionsUpdater.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
@ -16,7 +17,9 @@ namespace Wabbajack.Server.Services
private DiscordWebHook _discord;
private SqlService _sql;
public NexusPermissionsUpdater(ILogger<NexusKeyMaintainance> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromHours(4))
public static TimeSpan MaxSync = TimeSpan.FromHours(4);
public NexusPermissionsUpdater(ILogger<NexusKeyMaintainance> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromSeconds(1))
{
_discord = discord;
_sql = sql;
@ -24,7 +27,8 @@ namespace Wabbajack.Server.Services
public override async Task<int> Execute()
{
var permissions = await _sql.GetNexusPermissions();
await _sql.UpdateGameMetadata();
var data = await _sql.ModListArchives();
var nexusArchives = data.Select(a => a.State).OfType<NexusDownloader.State>().Select(d => (d.Game, d.ModID))
@ -33,38 +37,37 @@ namespace Wabbajack.Server.Services
_logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods");
using var queue = new WorkQueue();
using var queue = new WorkQueue(1);
var results = await nexusArchives.PMap(queue, async archive =>
{
var permissions = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID);
return (archive.Game, archive.ModID, permissions);
});
var prev = await _sql.GetNexusPermissions();
var updated = 0;
foreach (var result in results)
var lag = MaxSync / nexusArchives.Count * 2;
await nexusArchives.PMap(queue, async archive =>
{
if (permissions.TryGetValue((result.Game, result.ModID), out var oldPermission))
_logger.LogInformation($"Checking permissions for {archive.Game} {archive.ModID}");
var result = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID);
await _sql.SetNexusPermission(archive.Game, archive.ModID, result);
if (prev.TryGetValue((archive.Game, archive.ModID), out var oldPermission))
{
if (oldPermission != result.permissions)
if (oldPermission != result)
{
await _discord.Send(Channel.Spam,
new DiscordMessage {
Content = $"Permissions status of {result.Game} {result.ModID} was {oldPermission} is now {result.permissions} "
Content = $"Permissions status of {archive.Game} {archive.ModID} was {oldPermission} is now {result}"
});
await _sql.PurgeNexusCache(result.ModID);
updated += 1;
await _sql.PurgeNexusCache(archive.ModID);
await _quickSync.Notify<ListValidator>();
}
}
}
await _sql.SetNexusPermissions(results);
await Task.Delay(lag);
});
if (updated > 0)
await _quickSync.Notify<ListValidator>();
return updated;
return 1;
}
}
}

9
Wabbajack.Server/Services/PatchBuilder.cs
View File

@ -125,7 +125,7 @@ namespace Wabbajack.Server.Services
private static string PatchName(Hash oldHash, Hash newHash)
{
return $"{Consts.ArchiveUpdatesCDNFolder}\\{oldHash.ToHex()}_{newHash.ToHex()}";
return $"\\{oldHash.ToHex()}_{newHash.ToHex()}";
}
private async Task CleanupOldPatches()
@ -155,7 +155,7 @@ namespace Wabbajack.Server.Services
if (pendingPatch != default) break;
}
var files = await client.GetListingAsync($"{Consts.ArchiveUpdatesCDNFolder}\\");
var files = await client.GetListingAsync($"\\");
_logger.LogInformation($"Found {files.Length} on the CDN");
var sqlFiles = await _sql.AllPatchHashes();
@ -206,9 +206,6 @@ namespace Wabbajack.Server.Services
$"Uploading {patchFile.Size.ToFileSizeString()} patch file to CDN");
using var client = await GetBunnyCdnFtpClient();
if (!await client.DirectoryExistsAsync(Consts.ArchiveUpdatesCDNFolder))
await client.CreateDirectoryAsync(Consts.ArchiveUpdatesCDNFolder);
await client.UploadFileAsync((string)patchFile, patchName, FtpRemoteExists.Overwrite);
return;
}
@ -230,7 +227,7 @@ namespace Wabbajack.Server.Services
private async Task<FtpClient> GetBunnyCdnFtpClient()
{
var info = await Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
var info = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches);
var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
await client.ConnectAsync();
return client;

2
Wabbajack.Server/Startup.cs
View File

@ -69,6 +69,7 @@ namespace Wabbajack.Server
services.AddSingleton<PatchBuilder>();
services.AddSingleton<CDNMirrorList>();
services.AddSingleton<NexusPermissionsUpdater>();
services.AddSingleton<MirrorUploader>();
services.AddMvc();
services.AddControllers()
@ -125,6 +126,7 @@ namespace Wabbajack.Server
app.UseService<PatchBuilder>();
app.UseService<CDNMirrorList>();
app.UseService<NexusPermissionsUpdater>();
app.UseService<MirrorUploader>();
app.Use(next =>
{