Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Patch usage tracking, quick waking for polling tasks, several logging tweaks

Browse Source
This commit is contained in:
Timothy Baldridge 2020-06-06 15:44:30 -06:00
parent 58022d648e
commit 2d85e96214
14 changed files with 123 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -4,6 +4,7 @@ using System.Linq;
using System.Reactive;
using System.Reactive.Linq;
using System.Threading.Tasks;
using F23.StringSimilarity;
using Newtonsoft.Json;
using ReactiveUI;
using Wabbajack.Common;
@ -246,14 +247,16 @@ namespace Wabbajack.Lib.Downloaders
var mod = await client.GetModInfo(Game, ModID);
var files = await client.GetModFiles(Game, ModID);
var oldFile = files.files.FirstOrDefault(f => f.file_id == FileID);
var newFile = files.files.Where(f => f.category_name != null).OrderByDescending(f => f.uploaded_timestamp).FirstOrDefault();
var nl = new Levenshtein();
var newFile = files.files.Where(f => f.category_name != null)
.OrderBy(f => nl.Distance(oldFile.name.ToLowerInvariant(), f.name.ToLowerInvariant())).FirstOrDefault();
if (!mod.available || oldFile == default || newFile == default)
{
return default;
}
// Size is in KB
if (oldFile.size > 2_500_000 || newFile.size > 2_500_000 || oldFile.file_id == newFile.file_id)
if (oldFile.size > 4_500_000 || newFile.size > 4_500_000 || oldFile.file_id == newFile.file_id)
{
return default;
}

3
Wabbajack.Lib/Wabbajack.Lib.csproj
View File

@ -13,6 +13,9 @@
<PackageReference Include="CefSharp.OffScreen">
<Version>81.3.100</Version>
</PackageReference>
<PackageReference Include="F23.StringSimilarity">
<Version>3.1.0</Version>
</PackageReference>
<PackageReference Include="Fody">
<Version>6.1.2</Version>
</PackageReference>

2
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -560,6 +560,8 @@ CREATE TABLE [dbo].[Patches](
[Finished] [datetime] NULL,
[IsFailed] [tinyint] NULL,
[FailMessage] [varchar](MAX) NULL,
[LastUsed] [datetime] NULL,
[Downloads] [bigint] NOT NULL DEFAULT 0,
CONSTRAINT [PK_Patches] PRIMARY KEY CLUSTERED
(
[SrcId] ASC,

15
Wabbajack.Server/Controllers/ModUpgrade.cs
View File

@ -15,13 +15,15 @@ namespace Wabbajack.BuildServer.Controllers
private SqlService _sql;
private DiscordWebHook _discord;
private AppSettings _settings;
private QuickSync _quickSync;
public ModUpgrade(ILogger<ModUpgrade> logger, SqlService sql, DiscordWebHook discord, AppSettings settings)
public ModUpgrade(ILogger<ModUpgrade> logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync, AppSettings settings)
{
_logger = logger;
_sql = sql;
_discord = discord;
_settings = settings;
_quickSync = quickSync;
}
[HttpPost]
@ -49,6 +51,7 @@ namespace Wabbajack.BuildServer.Controllers
if (patch.PatchSize != 0)
{
_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found");
await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id);
return
Ok(
$"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{Consts.ArchiveUpdatesCDNFolder}/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
@ -57,6 +60,16 @@ namespace Wabbajack.BuildServer.Controllers
return NotFound("Patch creation failed");
}
if (!newDownload.DownloadFinished.HasValue)
{
await _quickSync.Notify<ArchiveDownloader>();
}
else
{
await _quickSync.Notify<PatchBuilder>();
}
_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found is processing");
// Still processing
return Accepted();

14
Wabbajack.Server/DataLayer/Patches.cs
View File

@ -110,7 +110,10 @@ namespace Wabbajack.Server.DataLayer
{
await using var conn = await Open();
var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
"SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE Finished is NULL");
@"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads src ON src.Id = p.SrcId
LEFT JOIN dbo.ArchiveDownloads dest ON dest.Id = p.DestId
WHERE p.Finished is NULL AND src.IsFailed = 0 AND dest.IsFailed = 0 ");
if (patch == default)
return default(Patch);
@ -144,5 +147,14 @@ namespace Wabbajack.Server.DataLayer
}
return results;
}
public async Task MarkPatchUsage(Guid srcId, Guid destId)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"UPDATE dbo.Patches SET Downloads = Downloads + 1, LastUsed = GETUTCDATE() WHERE SrcId = @srcId AND DestID = @destId",
new {SrcId = srcId, DestId = destId});
}
}
}

10
Wabbajack.Server/Services/AbstractService.cs
View File

@ -16,12 +16,14 @@ namespace Wabbajack.Server.Services
protected AppSettings _settings;
private TimeSpan _delay;
protected ILogger<TP> _logger;
protected QuickSync _quickSync;
public AbstractService(ILogger<TP> logger, AppSettings settings, TimeSpan delay)
public AbstractService(ILogger<TP> logger, AppSettings settings, QuickSync quickSync, TimeSpan delay)
{
_settings = settings;
_delay = delay;
_logger = logger;
_quickSync = quickSync;
}
public void Start()
@ -32,7 +34,7 @@ namespace Wabbajack.Server.Services
{
while (true)
{
await _quickSync.ResetToken<TP>();
try
{
await Execute();
@ -42,9 +44,9 @@ namespace Wabbajack.Server.Services
_logger.LogError(ex, "Running Service Loop");
}
await Task.Delay(_delay);
var token = await _quickSync.GetToken<TP>();
await Task.Delay(_delay, token);
}
});
}
}

15
Wabbajack.Server/Services/ArchiveDownloader.cs
View File

@ -16,11 +16,14 @@ namespace Wabbajack.Server.Services
private SqlService _sql;
private ArchiveMaintainer _archiveMaintainer;
private NexusApiClient _nexusClient;
private DiscordWebHook _discord;
public ArchiveDownloader(ILogger<ArchiveDownloader> logger, AppSettings settings, SqlService sql, ArchiveMaintainer archiveMaintainer) : base(logger, settings, TimeSpan.FromMinutes(10))
public ArchiveDownloader(ILogger<ArchiveDownloader> logger, AppSettings settings, SqlService sql, ArchiveMaintainer archiveMaintainer, DiscordWebHook discord, QuickSync quickSync)
: base(logger, settings, quickSync, TimeSpan.FromMinutes(10))
{
_sql = sql;
_archiveMaintainer = archiveMaintainer;
_discord = discord;
}
public override async Task<int> Execute()
@ -58,6 +61,7 @@ namespace Wabbajack.Server.Services
try
{
_logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"});
await DownloadDispatcher.PrepareAll(new[] {nextDownload.Archive.State});
await using var tempPath = new TempFile();
@ -86,17 +90,26 @@ namespace Wabbajack.Server.Services
_logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Finish(_sql);
await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}"});
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, ex.ToString());
await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"});
}
count++;
}
if (count > 0)
{
// Wake the Patch builder up in case it needs to build a patch now
await _quickSync.Notify<PatchBuilder>();
}
return count;
}
}

2
Wabbajack.Server/Services/DiscordWebHook.cs
View File

@ -23,7 +23,7 @@ namespace Wabbajack.Server.Services
private ILogger<DiscordWebHook> _logger;
private Random _random = new Random();
public DiscordWebHook(ILogger<DiscordWebHook> logger, AppSettings settings) : base(logger, settings, TimeSpan.FromHours(1))
public DiscordWebHook(ILogger<DiscordWebHook> logger, AppSettings settings, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(1))
{
_settings = settings;
_logger = logger;

10
Wabbajack.Server/Services/ListValidator.cs
View File

@ -30,8 +30,8 @@ namespace Wabbajack.Server.Services
new (ModListSummary Summary, DetailedStatus Detailed)[0];
public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives)
: base(logger, settings, TimeSpan.FromMinutes(5))
public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives, QuickSync quickSync)
: base(logger, settings, quickSync, TimeSpan.FromMinutes(5))
{
_sql = sql;
_discord = discord;
@ -56,7 +56,7 @@ namespace Wabbajack.Server.Services
{
var (_, result) = await ValidateArchive(data, archive);
if (result == ArchiveStatus.InValid)
return await TryToHeal(data, archive);
return await TryToHeal(data, archive, metadata);
return (archive, result);
});
@ -133,7 +133,7 @@ namespace Wabbajack.Server.Services
}
private AsyncLock _healLock = new AsyncLock();
private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive)
private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive, ModlistMetadata modList)
{
using var _ = await _healLock.WaitAsync();
@ -182,7 +182,7 @@ namespace Wabbajack.Server.Services
await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload});
_logger.Log(LogLevel.Information, $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}");
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}" });
await _discord.Send(Channel.Ham, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`" });
await upgrade.NewFile.DisposeAsync();

2
Wabbajack.Server/Services/NexusKeyMaintainance.cs
View File

@ -13,7 +13,7 @@ namespace Wabbajack.Server.Services
{
private SqlService _sql;
public NexusKeyMaintainance(ILogger<NexusKeyMaintainance> logger, AppSettings settings, SqlService sql) : base(logger, settings, TimeSpan.FromHours(1))
public NexusKeyMaintainance(ILogger<NexusKeyMaintainance> logger, AppSettings settings, SqlService sql, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(1))
{
_sql = sql;
}

4
Wabbajack.Server/Services/NonNexusDownloadValidator.cs
View File

@ -16,8 +16,8 @@ namespace Wabbajack.Server.Services
{
private SqlService _sql;
public NonNexusDownloadValidator(ILogger<NonNexusDownloadValidator> logger, AppSettings settings, SqlService sql)
: base(logger, settings, TimeSpan.FromHours(2))
public NonNexusDownloadValidator(ILogger<NonNexusDownloadValidator> logger, AppSettings settings, SqlService sql, QuickSync quickSync)
: base(logger, settings, quickSync, TimeSpan.FromHours(2))
{
_sql = sql;
}

8
Wabbajack.Server/Services/PatchBuilder.cs
View File

@ -22,7 +22,7 @@ namespace Wabbajack.Server.Services
private ArchiveMaintainer _maintainer;
public PatchBuilder(ILogger<PatchBuilder> logger, SqlService sql, AppSettings settings, ArchiveMaintainer maintainer,
DiscordWebHook discordWebHook) : base(logger, settings, TimeSpan.FromMinutes(1))
DiscordWebHook discordWebHook, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromMinutes(1))
{
_discordWebHook = discordWebHook;
_sql = sql;
@ -103,6 +103,12 @@ namespace Wabbajack.Server.Services
}
}
if (count > 0)
{
// Notify the List Validator that we may have more patches
await _quickSync.Notify<ListValidator>();
}
return count;
}

49
Wabbajack.Server/Services/QuickSync.cs Normal file
View File

@ -0,0 +1,49 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.Server.Services
{
public class QuickSync
{
private Dictionary<Type, CancellationTokenSource> _syncs = new Dictionary<Type, CancellationTokenSource>();
private AsyncLock _lock = new AsyncLock();
public async Task<CancellationToken> GetToken<T>()
{
using var _ = await _lock.WaitAsync();
if (_syncs.TryGetValue(typeof(T), out var result))
{
return result.Token;
}
var token = new CancellationTokenSource();
_syncs[typeof(T)] = token;
return token.Token;
}
public async Task ResetToken<T>()
{
using var _ = await _lock.WaitAsync();
if (_syncs.TryGetValue(typeof(T), out var ct))
{
ct.Cancel();
}
_syncs[typeof(T)] = new CancellationTokenSource();
}
public async Task Notify<T>()
{
using var _ = await _lock.WaitAsync();
if (_syncs.TryGetValue(typeof(T), out var ct))
{
ct.Cancel();
}
}
}
}

1
Wabbajack.Server/Startup.cs
View File

@ -55,6 +55,7 @@ namespace Wabbajack.Server
});
services.AddSingleton<AppSettings>();
services.AddSingleton<QuickSync>();
services.AddSingleton<SqlService>();
services.AddSingleton<GlobalInformation>();
services.AddSingleton<NexusPoll>();