Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes and logging for mirrored files and associated server changes

Browse Source
This commit is contained in:
Timothy Baldridge 2020-08-05 14:53:19 -06:00
parent 7f892d26d7
commit b9a3f9a3c4
6 changed files with 21 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Wabbajack.Common/Consts.cs
View File

@ -148,7 +148,6 @@ namespace Wabbajack.Common
public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2;
public static string ArchiveUpdatesCDNFolder = "archive_updates";
public static Uri WabbajackMirror = new Uri("https://wabbajack-mirror.b-cdn.net");
}

5
Wabbajack.Server/Controllers/ModUpgrade.cs
View File

@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
@ -20,6 +21,7 @@ namespace Wabbajack.BuildServer.Controllers
private DiscordWebHook _discord;
private AppSettings _settings;
private QuickSync _quickSync;
private Task<BunnyCdnFtpInfo> _creds;
public ModUpgrade(ILogger<ModUpgrade> logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync, AppSettings settings)
{
@ -28,6 +30,7 @@ namespace Wabbajack.BuildServer.Controllers
_discord = discord;
_settings = settings;
_quickSync = quickSync;
_creds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches);
}
[HttpPost]
@ -93,7 +96,7 @@ namespace Wabbajack.BuildServer.Controllers
await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id);
return
Ok(
$"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{Consts.ArchiveUpdatesCDNFolder}/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
$"https://{(await _creds).Username}.b-cdn.net/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
}
_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed");

4
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -156,8 +156,8 @@ namespace Wabbajack.Server.DataLayer
foreach (var add in missing.Where(g => g.NexusGameId != 0))
{
await conn.ExecuteAsync(
"INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, WabbajackName)",
new {NexusGameId = add.NexusGameId, WabbajackName = add.ToString()});
"INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, @WabbajackName)",
new {NexusGameId = add.NexusGameId, WabbajackName = add.Game.ToString()});
}
}

1
Wabbajack.Server/Services/AbstractService.cs
View File

@ -50,6 +50,7 @@ namespace Wabbajack.Server.Services
await _quickSync.ResetToken<TP>();
try
{
_logger.LogInformation($"Running: {GetType().Name}");
await Execute();
}
catch (Exception ex)

16
Wabbajack.Server/Services/NexusPermissionsUpdater.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
@ -15,8 +16,10 @@ namespace Wabbajack.Server.Services
{
private DiscordWebHook _discord;
private SqlService _sql;
public static TimeSpan MaxSync = TimeSpan.FromHours(4);
public NexusPermissionsUpdater(ILogger<NexusKeyMaintainance> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromHours(4))
public NexusPermissionsUpdater(ILogger<NexusKeyMaintainance> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromSeconds(1))
{
_discord = discord;
_sql = sql;
@ -26,8 +29,7 @@ namespace Wabbajack.Server.Services
{
await _sql.UpdateGameMetadata();
var permissions = await _sql.GetNexusPermissions();
var data = await _sql.ModListArchives();
var nexusArchives = data.Select(a => a.State).OfType<NexusDownloader.State>().Select(d => (d.Game, d.ModID))
.Distinct()
@ -35,12 +37,16 @@ namespace Wabbajack.Server.Services
_logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods");
using var queue = new WorkQueue(2);
using var queue = new WorkQueue(1);
var prev = await _sql.GetNexusPermissions();
var lag = MaxSync / nexusArchives.Count * 2;
await nexusArchives.PMap(queue, async archive =>
{
_logger.LogInformation($"Checking permissions for {archive.Game} {archive.ModID}");
var result = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID);
await _sql.SetNexusPermission(archive.Game, archive.ModID, result);
@ -56,6 +62,8 @@ namespace Wabbajack.Server.Services
await _quickSync.Notify<ListValidator>();
}
}
await Task.Delay(lag);
});
return 1;

7
Wabbajack.Server/Services/PatchBuilder.cs
View File

@ -125,7 +125,7 @@ namespace Wabbajack.Server.Services
private static string PatchName(Hash oldHash, Hash newHash)
{
return $"{Consts.ArchiveUpdatesCDNFolder}\\{oldHash.ToHex()}_{newHash.ToHex()}";
return $"\\{oldHash.ToHex()}_{newHash.ToHex()}";
}
private async Task CleanupOldPatches()
@ -155,7 +155,7 @@ namespace Wabbajack.Server.Services
if (pendingPatch != default) break;
}
var files = await client.GetListingAsync($"{Consts.ArchiveUpdatesCDNFolder}\\");
var files = await client.GetListingAsync($"\\");
_logger.LogInformation($"Found {files.Length} on the CDN");
var sqlFiles = await _sql.AllPatchHashes();
@ -206,9 +206,6 @@ namespace Wabbajack.Server.Services
$"Uploading {patchFile.Size.ToFileSizeString()} patch file to CDN");
using var client = await GetBunnyCdnFtpClient();
if (!await client.DirectoryExistsAsync(Consts.ArchiveUpdatesCDNFolder))
await client.CreateDirectoryAsync(Consts.ArchiveUpdatesCDNFolder);
await client.UploadFileAsync((string)patchFile, patchName, FtpRemoteExists.Overwrite);
return;
}