From 132774511506933900f7b63afe6a71b1a4bb755a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 25 Apr 2020 23:13:42 -0600 Subject: [PATCH] Bug fixes encountered during the 2.0 release --- Wabbajack.BuildServer.Test/MetricsTests.cs | 7 ++ .../BackendServices/ListIngest.cs | 14 +++- Wabbajack.BuildServer/Controllers/GraphQL.cs | 3 +- .../Controllers/IndexedFiles.cs | 25 ++++--- .../Controllers/ListValidation.cs | 8 ++- .../Controllers/ModlistUpdater.cs | 2 +- .../Controllers/NexusCache.cs | 3 - Wabbajack.BuildServer/GraphQL/Query.cs | 3 +- Wabbajack.BuildServer/JobManager.cs | 8 ++- .../Models/Sql/SqlService.cs | 67 ++++++++++++++----- Wabbajack.Common/Json.cs | 2 +- Wabbajack.Common/Paths.cs | 2 + 12 files changed, 104 insertions(+), 40 deletions(-) diff --git a/Wabbajack.BuildServer.Test/MetricsTests.cs b/Wabbajack.BuildServer.Test/MetricsTests.cs index 91fcc8d6..daa76652 100644 --- a/Wabbajack.BuildServer.Test/MetricsTests.cs +++ b/Wabbajack.BuildServer.Test/MetricsTests.cs @@ -30,5 +30,12 @@ namespace Wabbajack.BuildServer.Test Assert.Equal(subject, result); } + + [Fact] + public async Task CanLoadMetricsFromSQL() + { + var sql = Fixture.GetService(); + var results = await sql.MetricsReport("finish_install"); + } } } diff --git a/Wabbajack.BuildServer/BackendServices/ListIngest.cs b/Wabbajack.BuildServer/BackendServices/ListIngest.cs index 4f950f70..0e652404 100644 --- a/Wabbajack.BuildServer/BackendServices/ListIngest.cs +++ b/Wabbajack.BuildServer/BackendServices/ListIngest.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO.Compression; using System.Threading.Tasks; +using Newtonsoft.Json; using Wabbajack.BuildServer.Model.Models; using Wabbajack.Common; using Wabbajack.Lib; @@ -42,14 +43,23 @@ namespace Wabbajack.BuildServer.BackendServices ModList modlist; await using (var fs = modlistPath.OpenRead()) using (var zip = new ZipArchive(fs, ZipArchiveMode.Read)) - await using (var entry = zip.GetEntry("modlist.json")?.Open()) + await using (var entry = zip.GetEntry("modlist")?.Open()) { if (entry == null) { Utils.Log($"Bad Modlist {list.Links.MachineURL}"); continue; } - modlist = entry.FromJson(); + + try + { + modlist = entry.FromJson(); + } + catch (JsonReaderException ex) + { + Utils.Log($"Bad JSON format for {list.Links.MachineURL}"); + continue; + } } newData = true; diff --git a/Wabbajack.BuildServer/Controllers/GraphQL.cs b/Wabbajack.BuildServer/Controllers/GraphQL.cs index 6afa21d1..6190b8a1 100644 --- a/Wabbajack.BuildServer/Controllers/GraphQL.cs +++ b/Wabbajack.BuildServer/Controllers/GraphQL.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging; using Wabbajack.BuildServer.GraphQL; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; +using Wabbajack.Common; namespace Wabbajack.BuildServer.Controllers { @@ -35,7 +36,7 @@ namespace Wabbajack.BuildServer.Controllers if (result.Errors?.Count > 0) { - return BadRequest(); + return BadRequest(result.Errors); } return Ok(result); diff --git a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs index 30587e1a..574c9b3f 100644 --- a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs @@ -52,19 +52,25 @@ namespace Wabbajack.BuildServer.Controllers var fullPath = folder.RelativeTo((AbsolutePath)_settings.TempFolder); Utils.Log($"Ingesting Inis from {fullPath}"); int loadCount = 0; - foreach (var file in fullPath.EnumerateFiles().Where(f => f.Extension == Consts.IniExtension)) - { - var loaded = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile(), true)); - if (loaded == null) + using var queue = new WorkQueue(); + await fullPath.EnumerateFiles().Where(f => f.Extension == Consts.IniExtension) + .PMap(queue, async file => { + + try { - Utils.Log($"Unsupported Ini {file}"); - continue; + var loaded = + (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile(), true)); + + var hash = Hash.FromHex(((string)file.FileNameWithoutExtension).Split("_").First()); + await SQL.AddDownloadState(hash, loaded); + } + catch (Exception ex) + { + Utils.Log($"Failure for {file}"); } - var hash = Hash.FromHex(((string)file.FileNameWithoutExtension).Split("_").First()); - await SQL.AddDownloadState(hash, loaded); loadCount += 1; - } + }); return Ok(loadCount); } @@ -82,7 +88,6 @@ namespace Wabbajack.BuildServer.Controllers { await using var ins = entry.Open(); var iniString = Encoding.UTF8.GetString(await ins.ReadAllAsync()); - Utils.Log(iniString); var data = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(iniString.LoadIniString(), true)); if (data == null) diff --git a/Wabbajack.BuildServer/Controllers/ListValidation.cs b/Wabbajack.BuildServer/Controllers/ListValidation.cs index f1e2aaa8..305cdcce 100644 --- a/Wabbajack.BuildServer/Controllers/ListValidation.cs +++ b/Wabbajack.BuildServer/Controllers/ListValidation.cs @@ -68,7 +68,10 @@ namespace Wabbajack.BuildServer.Controllers var (_, result) = ValidateArchive(data, archive); if (result == ArchiveStatus.InValid) { - return await TryToFix(data, archive); + var fixResult = await TryToFix(data, archive); + + return fixResult; + } return (archive, result); @@ -105,7 +108,7 @@ namespace Wabbajack.BuildServer.Controllers }); - var cacheOptions = new MemoryCacheEntryOptions().SetSlidingExpiration(TimeSpan.FromMinutes(1)); + var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1)); Cache.Set(ModListSummariesKey, results, cacheOptions); return results; } @@ -142,6 +145,7 @@ namespace Wabbajack.BuildServer.Controllers var result = await _updater.GetAlternative(archive.Hash.ToHex()); return result switch { + OkObjectResult ok => (archive, ArchiveStatus.Updated), OkResult ok => (archive, ArchiveStatus.Updated), AcceptedResult accept => (archive, ArchiveStatus.Updating), _ => (archive, ArchiveStatus.InValid) diff --git a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs index 8faa57de..f6216adb 100644 --- a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs +++ b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs @@ -216,7 +216,7 @@ namespace Wabbajack.BuildServer.Controllers private async Task FindNexusAlternative(NexusDownloader.State state, Hash srcHash) { var origSize = _settings.PathForArchive(srcHash).Size; - var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); + var api = await NexusApiClient.Get(Request?.Headers["apikey"].FirstOrDefault()); var allMods = await api.GetModFiles(state.Game, state.ModID); var archive = allMods.files.Where(m => !string.IsNullOrEmpty(m.category_name)) .OrderBy(s => Math.Abs((long)s.size - origSize)) diff --git a/Wabbajack.BuildServer/Controllers/NexusCache.cs b/Wabbajack.BuildServer/Controllers/NexusCache.cs index 2dffd915..83bde6b9 100644 --- a/Wabbajack.BuildServer/Controllers/NexusCache.cs +++ b/Wabbajack.BuildServer/Controllers/NexusCache.cs @@ -41,7 +41,6 @@ namespace Wabbajack.BuildServer.Controllers [Route("{GameName}/mods/{ModId}.json")] public async Task GetModInfo(string GameName, long ModId) { - Utils.Log($"Nexus Mod Info {GameName} {ModId}"); var game = GameRegistry.GetByFuzzyName(GameName).Game; var result = await SQL.GetNexusModInfoString(game, ModId); @@ -69,8 +68,6 @@ namespace Wabbajack.BuildServer.Controllers [Route("{GameName}/mods/{ModId}/files.json")] public async Task GetModFiles(string GameName, long ModId) { - Utils.Log($"Nexus Mod Files {GameName} {ModId}"); - var game = GameRegistry.GetByFuzzyName(GameName).Game; var result = await SQL.GetModFiles(game, ModId); diff --git a/Wabbajack.BuildServer/GraphQL/Query.cs b/Wabbajack.BuildServer/GraphQL/Query.cs index 0a5b0f41..08e0e075 100644 --- a/Wabbajack.BuildServer/GraphQL/Query.cs +++ b/Wabbajack.BuildServer/GraphQL/Query.cs @@ -8,6 +8,7 @@ namespace Wabbajack.BuildServer.GraphQL { public Query(SqlService sql) { + /* FieldAsync>("modLists", arguments: new QueryArguments(new QueryArgument { @@ -28,7 +29,7 @@ namespace Wabbajack.BuildServer.GraphQL } }); - +*/ FieldAsync>("dailyUniqueMetrics", arguments: new QueryArguments( new QueryArgument {Name = "metric_type", Description = "The grouping of metric data to query"} diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index a98ab1a2..1a89b5d4 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -1,9 +1,11 @@ using System; using System.Linq; using System.Reactive.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Nettle; +using Wabbajack.BuildServer.BackendServices; using Wabbajack.BuildServer.Controllers; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; @@ -77,6 +79,7 @@ namespace Wabbajack.BuildServer if (!Settings.JobScheduler) return; var task = RunNexusCacheLoop(); + var listIngest = (new ListIngest(Sql, Settings)).RunLoop(CancellationToken.None); while (true) { @@ -99,6 +102,7 @@ namespace Wabbajack.BuildServer } } + private async Task KillOrphanedJobs() { try @@ -126,7 +130,7 @@ namespace Wabbajack.BuildServer if (!Settings.RunFrontEndJobs && typeof(T).ImplementsInterface(typeof(IFrontEndJob))) return; try { - var jobs = (await Sql.GetUnfinishedJobs()) + var jobs = (await Sql.GetAllJobs(span)) .Where(j => j.Payload is T) .OrderByDescending(j => j.Created) .Take(10); @@ -134,7 +138,7 @@ namespace Wabbajack.BuildServer foreach (var job in jobs) { if (job.Started == null || job.Ended == null) return; - if (DateTime.Now - job.Ended < span) return; + if (DateTime.UtcNow - job.Ended < span) return; } await Sql.EnqueueJob(new Job { diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index a5dca44b..948d5611 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -167,7 +167,7 @@ namespace Wabbajack.BuildServer.Model.Models AND DATEADD(DAY, number+1, dbo.MinMetricDate()) < dbo.MaxMetricDate()) as d ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action WHERE d.Action = @action - AND d.Date >= DATEADD(month, -1, GETDATE()) + AND d.Date >= DATEADD(month, -1, GETUTCDATE()) group by d.Date, d.GroupingSubject, d.Action ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action})) .ToList(); @@ -184,7 +184,7 @@ namespace Wabbajack.BuildServer.Model.Models { await using var conn = await Open(); await conn.ExecuteAsync( - @"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)", + @"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETUTCDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)", new { job.Priority, PrimaryKeyString = job.Payload.PrimaryKeyString, @@ -201,7 +201,7 @@ namespace Wabbajack.BuildServer.Model.Models { await using var conn = await Open(); await conn.ExecuteAsync( - @"UPDATE dbo.Jobs SET Ended = GETDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id", + @"UPDATE dbo.Jobs SET Ended = GETUTCDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id", new { job.Id, Success = job.Result.ResultType == JobResultType.Success, @@ -221,7 +221,7 @@ namespace Wabbajack.BuildServer.Model.Models { await using var conn = await Open(); var result = await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>( - @"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy + @"UPDATE jobs SET Started = GETUTCDATE(), RunBy = @RunBy WHERE ID in (SELECT TOP(1) ID FROM Jobs WHERE Started is NULL AND PrimaryKeyString NOT IN (SELECT PrimaryKeyString from jobs WHERE Started IS NOT NULL and Ended IS NULL) @@ -229,14 +229,14 @@ namespace Wabbajack.BuildServer.Model.Models SELECT TOP(1) Id, Started, Ended, Created, Payload, Priority FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC", new {RunBy = Guid.NewGuid().ToString()}); return result.Select(k => - new Job { - Id = k.Item1, - Started = k.Item2, - Ended = k.Item3, - Created = k.Item4, - Payload = k.Item5, - Priority = (Job.JobPriority)k.Item6 - }).FirstOrDefault(); + new Job { + Id = k.Item1, + Started = k.Item2, + Ended = k.Item3, + Created = k.Item4, + Payload = k.Item5, + Priority = (Job.JobPriority)k.Item6 + }).FirstOrDefault(); } @@ -244,8 +244,16 @@ namespace Wabbajack.BuildServer.Model.Models { await using var conn = await Open(); var results = - await conn.QueryAsync("SELECT * from dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL "); - return results; + await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority FROM dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL "); + return results.Select(k => + new Job { + Id = k.Item1, + Started = k.Item2, + Ended = k.Item3, + Created = k.Item4, + Payload = k.Item5, + Priority = (Job.JobPriority)k.Item6 + }); } @@ -253,8 +261,16 @@ namespace Wabbajack.BuildServer.Model.Models { await using var conn = await Open(); var results = - await conn.QueryAsync("SELECT * from dbo.Jobs WHERE Ended IS NULL "); - return results; + await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority from dbo.Jobs WHERE Ended IS NULL "); + return results.Select(k => + new Job { + Id = k.Item1, + Started = k.Item2, + Ended = k.Item3, + Created = k.Item4, + Payload = k.Item5, + Priority = (Job.JobPriority)k.Item6 + }); } @@ -353,7 +369,7 @@ namespace Wabbajack.BuildServer.Model.Models public async Task> AllUploadedFiles() { await using var conn = await Open(); - return await conn.QueryAsync("SELECT * FROM dbo.UploadedFiles ORDER BY UploadDate DESC"); + return await conn.QueryAsync("SELECT Id, Name, Size, UploadedBy as Uploader, Hash, UploadDate, CDNName FROM dbo.UploadedFiles ORDER BY UploadDate DESC"); } public async Task DeleteUploadedFile(Guid dupId) @@ -853,5 +869,22 @@ namespace Wabbajack.BuildServer.Model.Models #endregion + + public async Task> GetAllJobs(TimeSpan from) + { + await using var conn = await Open(); + var results = + await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority from dbo.Jobs WHERE Created >= @FromTime ", + new {FromTime = DateTime.UtcNow - from}); + return results.Select(k => + new Job { + Id = k.Item1, + Started = k.Item2, + Ended = k.Item3, + Created = k.Item4, + Payload = k.Item5, + Priority = (Job.JobPriority)k.Item6 + }); + } } } diff --git a/Wabbajack.Common/Json.cs b/Wabbajack.Common/Json.cs index c4708842..87d68522 100644 --- a/Wabbajack.Common/Json.cs +++ b/Wabbajack.Common/Json.cs @@ -98,7 +98,7 @@ namespace Wabbajack.Common } } - private class RelativePathConverter : JsonConverter + public class RelativePathConverter : JsonConverter { public override void WriteJson(JsonWriter writer, RelativePath value, JsonSerializer serializer) { diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index 195651bb..c3400ef0 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -9,6 +9,7 @@ using System.Reflection; using System.Text; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; +using Newtonsoft.Json; using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; @@ -405,6 +406,7 @@ namespace Wabbajack.Common } } + [JsonConverter(typeof(Utils.RelativePathConverter))] public struct RelativePath : IPath, IEquatable, IComparable { private readonly string? _nullable_path;