diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 8d9b681a..3150c51b 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -148,7 +148,6 @@ namespace Wabbajack.Common public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2; - public static string ArchiveUpdatesCDNFolder = "archive_updates"; public static Uri WabbajackMirror = new Uri("https://wabbajack-mirror.b-cdn.net"); } diff --git a/Wabbajack.Server/Controllers/ModUpgrade.cs b/Wabbajack.Server/Controllers/ModUpgrade.cs index 835776a8..f26c24c2 100644 --- a/Wabbajack.Server/Controllers/ModUpgrade.cs +++ b/Wabbajack.Server/Controllers/ModUpgrade.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; using Wabbajack.Server.Services; namespace Wabbajack.BuildServer.Controllers @@ -20,6 +21,7 @@ namespace Wabbajack.BuildServer.Controllers private DiscordWebHook _discord; private AppSettings _settings; private QuickSync _quickSync; + private Task _creds; public ModUpgrade(ILogger logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync, AppSettings settings) { @@ -28,6 +30,7 @@ namespace Wabbajack.BuildServer.Controllers _discord = discord; _settings = settings; _quickSync = quickSync; + _creds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches); } [HttpPost] @@ -93,7 +96,7 @@ namespace Wabbajack.BuildServer.Controllers await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id); return Ok( - $"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{Consts.ArchiveUpdatesCDNFolder}/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); + $"https://{(await _creds).Username}.b-cdn.net/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); } _logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed"); diff --git a/Wabbajack.Server/DataLayer/Nexus.cs b/Wabbajack.Server/DataLayer/Nexus.cs index 20e26645..487ca3a5 100644 --- a/Wabbajack.Server/DataLayer/Nexus.cs +++ b/Wabbajack.Server/DataLayer/Nexus.cs @@ -156,8 +156,8 @@ namespace Wabbajack.Server.DataLayer foreach (var add in missing.Where(g => g.NexusGameId != 0)) { await conn.ExecuteAsync( - "INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, WabbajackName)", - new {NexusGameId = add.NexusGameId, WabbajackName = add.ToString()}); + "INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, @WabbajackName)", + new {NexusGameId = add.NexusGameId, WabbajackName = add.Game.ToString()}); } } diff --git a/Wabbajack.Server/Services/AbstractService.cs b/Wabbajack.Server/Services/AbstractService.cs index 1998c266..f54b9d6a 100644 --- a/Wabbajack.Server/Services/AbstractService.cs +++ b/Wabbajack.Server/Services/AbstractService.cs @@ -50,6 +50,7 @@ namespace Wabbajack.Server.Services await _quickSync.ResetToken(); try { + _logger.LogInformation($"Running: {GetType().Name}"); await Execute(); } catch (Exception ex) diff --git a/Wabbajack.Server/Services/NexusPermissionsUpdater.cs b/Wabbajack.Server/Services/NexusPermissionsUpdater.cs index b85c28bb..6190c1ae 100644 --- a/Wabbajack.Server/Services/NexusPermissionsUpdater.cs +++ b/Wabbajack.Server/Services/NexusPermissionsUpdater.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Wabbajack.BuildServer; @@ -15,8 +16,10 @@ namespace Wabbajack.Server.Services { private DiscordWebHook _discord; private SqlService _sql; + + public static TimeSpan MaxSync = TimeSpan.FromHours(4); - public NexusPermissionsUpdater(ILogger logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromHours(4)) + public NexusPermissionsUpdater(ILogger logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromSeconds(1)) { _discord = discord; _sql = sql; @@ -26,8 +29,7 @@ namespace Wabbajack.Server.Services { await _sql.UpdateGameMetadata(); - var permissions = await _sql.GetNexusPermissions(); - + var data = await _sql.ModListArchives(); var nexusArchives = data.Select(a => a.State).OfType().Select(d => (d.Game, d.ModID)) .Distinct() @@ -35,12 +37,16 @@ namespace Wabbajack.Server.Services _logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods"); - using var queue = new WorkQueue(2); + using var queue = new WorkQueue(1); var prev = await _sql.GetNexusPermissions(); + var lag = MaxSync / nexusArchives.Count * 2; + + await nexusArchives.PMap(queue, async archive => { + _logger.LogInformation($"Checking permissions for {archive.Game} {archive.ModID}"); var result = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID); await _sql.SetNexusPermission(archive.Game, archive.ModID, result); @@ -56,6 +62,8 @@ namespace Wabbajack.Server.Services await _quickSync.Notify(); } } + + await Task.Delay(lag); }); return 1; diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index 0d9f29e6..b73fe9f4 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -125,7 +125,7 @@ namespace Wabbajack.Server.Services private static string PatchName(Hash oldHash, Hash newHash) { - return $"{Consts.ArchiveUpdatesCDNFolder}\\{oldHash.ToHex()}_{newHash.ToHex()}"; + return $"\\{oldHash.ToHex()}_{newHash.ToHex()}"; } private async Task CleanupOldPatches() @@ -155,7 +155,7 @@ namespace Wabbajack.Server.Services if (pendingPatch != default) break; } - var files = await client.GetListingAsync($"{Consts.ArchiveUpdatesCDNFolder}\\"); + var files = await client.GetListingAsync($"\\"); _logger.LogInformation($"Found {files.Length} on the CDN"); var sqlFiles = await _sql.AllPatchHashes(); @@ -206,9 +206,6 @@ namespace Wabbajack.Server.Services $"Uploading {patchFile.Size.ToFileSizeString()} patch file to CDN"); using var client = await GetBunnyCdnFtpClient(); - if (!await client.DirectoryExistsAsync(Consts.ArchiveUpdatesCDNFolder)) - await client.CreateDirectoryAsync(Consts.ArchiveUpdatesCDNFolder); - await client.UploadFileAsync((string)patchFile, patchName, FtpRemoteExists.Overwrite); return; }