Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes and logging for mirrored files and associated server changes

Browse Source
This commit is contained in:
Timothy Baldridge 2020-08-05 14:53:19 -06:00
parent 7f892d26d7
commit b9a3f9a3c4
6 changed files with 21 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Wabbajack.Common/Consts.cs
View File

@ -148,7 +148,6 @@ namespace Wabbajack.Common
public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2; public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2;
public static string ArchiveUpdatesCDNFolder = "archive_updates";
public static Uri WabbajackMirror = new Uri("https://wabbajack-mirror.b-cdn.net"); public static Uri WabbajackMirror = new Uri("https://wabbajack-mirror.b-cdn.net");
} }

5
Wabbajack.Server/Controllers/ModUpgrade.cs
View File

@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Lib; using Wabbajack.Lib;
using Wabbajack.Server.DataLayer; using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services; using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers namespace Wabbajack.BuildServer.Controllers
@ -20,6 +21,7 @@ namespace Wabbajack.BuildServer.Controllers
private DiscordWebHook _discord; private DiscordWebHook _discord;
private AppSettings _settings; private AppSettings _settings;
private QuickSync _quickSync; private QuickSync _quickSync;
private Task<BunnyCdnFtpInfo> _creds;
public ModUpgrade(ILogger<ModUpgrade> logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync, AppSettings settings) public ModUpgrade(ILogger<ModUpgrade> logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync, AppSettings settings)
{ {
@ -28,6 +30,7 @@ namespace Wabbajack.BuildServer.Controllers
_discord = discord; _discord = discord;
_settings = settings; _settings = settings;
_quickSync = quickSync; _quickSync = quickSync;
_creds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches);
} }
[HttpPost] [HttpPost]
@ -93,7 +96,7 @@ namespace Wabbajack.BuildServer.Controllers
await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id); await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id);
return return
Ok( Ok(
$"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{Consts.ArchiveUpdatesCDNFolder}/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); $"https://{(await _creds).Username}.b-cdn.net/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
} }
_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed"); _logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed");

4
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -156,8 +156,8 @@ namespace Wabbajack.Server.DataLayer
foreach (var add in missing.Where(g => g.NexusGameId != 0)) foreach (var add in missing.Where(g => g.NexusGameId != 0))
{ {
await conn.ExecuteAsync( await conn.ExecuteAsync(
"INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, WabbajackName)", "INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, @WabbajackName)",
new {NexusGameId = add.NexusGameId, WabbajackName = add.ToString()}); new {NexusGameId = add.NexusGameId, WabbajackName = add.Game.ToString()});
} }
} }

1
Wabbajack.Server/Services/AbstractService.cs
View File

@ -50,6 +50,7 @@ namespace Wabbajack.Server.Services
await _quickSync.ResetToken<TP>(); await _quickSync.ResetToken<TP>();
try try
{ {
_logger.LogInformation($"Running: {GetType().Name}");
await Execute(); await Execute();
} }
catch (Exception ex) catch (Exception ex)

16
Wabbajack.Server/Services/NexusPermissionsUpdater.cs
View File

@ -1,5 +1,6 @@
using System; using System;
using System.Linq; using System.Linq;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer; using Wabbajack.BuildServer;
@ -15,8 +16,10 @@ namespace Wabbajack.Server.Services
{ {
private DiscordWebHook _discord; private DiscordWebHook _discord;
private SqlService _sql; private SqlService _sql;
public static TimeSpan MaxSync = TimeSpan.FromHours(4);
public NexusPermissionsUpdater(ILogger<NexusKeyMaintainance> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromHours(4)) public NexusPermissionsUpdater(ILogger<NexusKeyMaintainance> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromSeconds(1))
{ {
_discord = discord; _discord = discord;
_sql = sql; _sql = sql;
@ -26,8 +29,7 @@ namespace Wabbajack.Server.Services
{ {
await _sql.UpdateGameMetadata(); await _sql.UpdateGameMetadata();
var permissions = await _sql.GetNexusPermissions();
var data = await _sql.ModListArchives(); var data = await _sql.ModListArchives();
var nexusArchives = data.Select(a => a.State).OfType<NexusDownloader.State>().Select(d => (d.Game, d.ModID)) var nexusArchives = data.Select(a => a.State).OfType<NexusDownloader.State>().Select(d => (d.Game, d.ModID))
.Distinct() .Distinct()
@ -35,12 +37,16 @@ namespace Wabbajack.Server.Services
_logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods"); _logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods");
using var queue = new WorkQueue(2); using var queue = new WorkQueue(1);
var prev = await _sql.GetNexusPermissions(); var prev = await _sql.GetNexusPermissions();
var lag = MaxSync / nexusArchives.Count * 2;
await nexusArchives.PMap(queue, async archive => await nexusArchives.PMap(queue, async archive =>
{ {
_logger.LogInformation($"Checking permissions for {archive.Game} {archive.ModID}");
var result = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID); var result = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID);
await _sql.SetNexusPermission(archive.Game, archive.ModID, result); await _sql.SetNexusPermission(archive.Game, archive.ModID, result);
@ -56,6 +62,8 @@ namespace Wabbajack.Server.Services
await _quickSync.Notify<ListValidator>(); await _quickSync.Notify<ListValidator>();
} }
} }
await Task.Delay(lag);
}); });
return 1; return 1;

7
Wabbajack.Server/Services/PatchBuilder.cs
View File

@ -125,7 +125,7 @@ namespace Wabbajack.Server.Services
private static string PatchName(Hash oldHash, Hash newHash) private static string PatchName(Hash oldHash, Hash newHash)
{ {
return $"{Consts.ArchiveUpdatesCDNFolder}\\{oldHash.ToHex()}_{newHash.ToHex()}"; return $"\\{oldHash.ToHex()}_{newHash.ToHex()}";
} }
private async Task CleanupOldPatches() private async Task CleanupOldPatches()
@ -155,7 +155,7 @@ namespace Wabbajack.Server.Services
if (pendingPatch != default) break; if (pendingPatch != default) break;
} }
var files = await client.GetListingAsync($"{Consts.ArchiveUpdatesCDNFolder}\\"); var files = await client.GetListingAsync($"\\");
_logger.LogInformation($"Found {files.Length} on the CDN"); _logger.LogInformation($"Found {files.Length} on the CDN");
var sqlFiles = await _sql.AllPatchHashes(); var sqlFiles = await _sql.AllPatchHashes();
@ -206,9 +206,6 @@ namespace Wabbajack.Server.Services
$"Uploading {patchFile.Size.ToFileSizeString()} patch file to CDN"); $"Uploading {patchFile.Size.ToFileSizeString()} patch file to CDN");
using var client = await GetBunnyCdnFtpClient(); using var client = await GetBunnyCdnFtpClient();
if (!await client.DirectoryExistsAsync(Consts.ArchiveUpdatesCDNFolder))
await client.CreateDirectoryAsync(Consts.ArchiveUpdatesCDNFolder);
await client.UploadFileAsync((string)patchFile, patchName, FtpRemoteExists.Overwrite); await client.UploadFileAsync((string)patchFile, patchName, FtpRemoteExists.Overwrite);
return; return;
} }