Bug fixes encountered during the 2.0 release

Wabbajack.BuildServer.Test/MetricsTests.cs

@@ -30,5 +30,12 @@ namespace Wabbajack.BuildServer.Test
 
             Assert.Equal(subject, result);
         }
+
+        [Fact]
+        public async Task CanLoadMetricsFromSQL()
+        {
+            var sql = Fixture.GetService<SqlService>();
+            var results = await sql.MetricsReport("finish_install");
+        }
     }
 }

Wabbajack.BuildServer/BackendServices/ListIngest.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO.Compression;
 using System.Threading.Tasks;
+using Newtonsoft.Json;
 using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.Common;
 using Wabbajack.Lib;
@@ -42,14 +43,23 @@ namespace Wabbajack.BuildServer.BackendServices
                 ModList modlist;
                 await using (var fs = modlistPath.OpenRead())
                 using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
-                await using (var entry = zip.GetEntry("modlist.json")?.Open())
+                await using (var entry = zip.GetEntry("modlist")?.Open())
                 {
                     if (entry == null)
                     {
                         Utils.Log($"Bad Modlist {list.Links.MachineURL}");
                         continue;
                     }
-                    modlist = entry.FromJson<ModList>();
+
+                    try
+                    {
+                        modlist = entry.FromJson<ModList>();
+                    }
+                    catch (JsonReaderException ex)
+                    {
+                        Utils.Log($"Bad JSON format for {list.Links.MachineURL}");
+                        continue;
+                    }
                 }
 
                 newData = true;

Wabbajack.BuildServer/Controllers/GraphQL.cs

@@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging;
 using Wabbajack.BuildServer.GraphQL;
 using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.BuildServer.Models;
+using Wabbajack.Common;
 
 namespace Wabbajack.BuildServer.Controllers
 {
@@ -35,7 +36,7 @@ namespace Wabbajack.BuildServer.Controllers
 
             if (result.Errors?.Count > 0)
             {
-                return BadRequest();
+                return BadRequest(result.Errors);
             }
 
             return Ok(result);

Wabbajack.BuildServer/Controllers/IndexedFiles.cs

@@ -52,19 +52,25 @@ namespace Wabbajack.BuildServer.Controllers
             var fullPath = folder.RelativeTo((AbsolutePath)_settings.TempFolder);
             Utils.Log($"Ingesting Inis from {fullPath}");
             int loadCount = 0;
-            foreach (var file in fullPath.EnumerateFiles().Where(f => f.Extension == Consts.IniExtension))
+            using var queue = new WorkQueue();
-            {
+            await fullPath.EnumerateFiles().Where(f => f.Extension == Consts.IniExtension)
-                var loaded = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile(), true));
+                    .PMap(queue, async file => {
-                if (loaded == null)
+            
+                try
                 {
-                    Utils.Log($"Unsupported Ini {file}");
+                    var loaded =
-                    continue;
+                        (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile(), true));
+
+                    var hash = Hash.FromHex(((string)file.FileNameWithoutExtension).Split("_").First());
+                    await SQL.AddDownloadState(hash, loaded);
+                }
+                catch (Exception ex)
+                {
+                    Utils.Log($"Failure for {file}");
                 }
 
-                var hash = Hash.FromHex(((string)file.FileNameWithoutExtension).Split("_").First()); 
-                await SQL.AddDownloadState(hash, loaded);
                 loadCount += 1;
-            }
+            });
 
             return Ok(loadCount);
         }
@@ -82,7 +88,6 @@ namespace Wabbajack.BuildServer.Controllers
             {
                 await using var ins = entry.Open();
                 var iniString = Encoding.UTF8.GetString(await ins.ReadAllAsync());
-                Utils.Log(iniString);
                 var data = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(iniString.LoadIniString(), true));
                 
                 if (data == null)

Wabbajack.BuildServer/Controllers/ListValidation.cs

@@ -68,7 +68,10 @@ namespace Wabbajack.BuildServer.Controllers
                     var (_, result) = ValidateArchive(data, archive);
                     if (result == ArchiveStatus.InValid)
                     {
-                        return await TryToFix(data, archive);
+                        var fixResult = await TryToFix(data, archive);
+                        
+                        return fixResult;
+
                     }
 
                     return (archive, result);
@@ -105,7 +108,7 @@ namespace Wabbajack.BuildServer.Controllers
             });
 
             
-            var cacheOptions = new MemoryCacheEntryOptions().SetSlidingExpiration(TimeSpan.FromMinutes(1));
+            var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1));
             Cache.Set(ModListSummariesKey, results, cacheOptions);
             return results;
         }
@@ -142,6 +145,7 @@ namespace Wabbajack.BuildServer.Controllers
             var result = await _updater.GetAlternative(archive.Hash.ToHex());
             return result switch
             {
+                OkObjectResult ok => (archive, ArchiveStatus.Updated),
                 OkResult ok => (archive, ArchiveStatus.Updated),
                 AcceptedResult accept => (archive, ArchiveStatus.Updating),
                 _ => (archive, ArchiveStatus.InValid)

Wabbajack.BuildServer/Controllers/ModlistUpdater.cs

@@ -216,7 +216,7 @@ namespace Wabbajack.BuildServer.Controllers
         private async Task<Archive> FindNexusAlternative(NexusDownloader.State state, Hash srcHash)
         {
             var origSize = _settings.PathForArchive(srcHash).Size;
-            var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
+            var api = await NexusApiClient.Get(Request?.Headers["apikey"].FirstOrDefault());
             var allMods = await api.GetModFiles(state.Game, state.ModID);
             var archive = allMods.files.Where(m => !string.IsNullOrEmpty(m.category_name))
                 .OrderBy(s => Math.Abs((long)s.size - origSize))

Wabbajack.BuildServer/Controllers/NexusCache.cs

@@ -41,7 +41,6 @@ namespace Wabbajack.BuildServer.Controllers
         [Route("{GameName}/mods/{ModId}.json")]
         public async Task<ModInfo> GetModInfo(string GameName, long ModId)
         {
-            Utils.Log($"Nexus Mod Info {GameName} {ModId}");
             var game = GameRegistry.GetByFuzzyName(GameName).Game;
             var result = await SQL.GetNexusModInfoString(game, ModId);
             
@@ -69,8 +68,6 @@ namespace Wabbajack.BuildServer.Controllers
         [Route("{GameName}/mods/{ModId}/files.json")]
         public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, long ModId)
         {
-            Utils.Log($"Nexus Mod Files {GameName} {ModId}");
-
             var game = GameRegistry.GetByFuzzyName(GameName).Game;
             var result = await SQL.GetModFiles(game, ModId);
 

Wabbajack.BuildServer/GraphQL/Query.cs

@@ -8,6 +8,7 @@ namespace Wabbajack.BuildServer.GraphQL
     {
         public Query(SqlService sql)
         {
+            /*
             FieldAsync<ListGraphType<ModListStatusType>>("modLists",
                 arguments: new QueryArguments(new QueryArgument<ArchiveEnumFilterType>
                 {
@@ -28,7 +29,7 @@ namespace Wabbajack.BuildServer.GraphQL
                     }
 
                 });
-
+*/
             FieldAsync<ListGraphType<MetricResultType>>("dailyUniqueMetrics",
                 arguments: new QueryArguments(
                     new QueryArgument<MetricEnum> {Name = "metric_type", Description = "The grouping of metric data to query"}

Wabbajack.BuildServer/JobManager.cs

@@ -1,9 +1,11 @@
 using System;
 using System.Linq;
 using System.Reactive.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Nettle;
+using Wabbajack.BuildServer.BackendServices;
 using Wabbajack.BuildServer.Controllers;
 using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.BuildServer.Models;
@@ -77,6 +79,7 @@ namespace Wabbajack.BuildServer
             if (!Settings.JobScheduler) return;
 
             var task = RunNexusCacheLoop();
+            var listIngest = (new ListIngest(Sql, Settings)).RunLoop(CancellationToken.None);
             
             while (true)
             {
@@ -99,6 +102,7 @@ namespace Wabbajack.BuildServer
             }
         }
 
+
         private async Task KillOrphanedJobs()
         {
             try
@@ -126,7 +130,7 @@ namespace Wabbajack.BuildServer
             if (!Settings.RunFrontEndJobs && typeof(T).ImplementsInterface(typeof(IFrontEndJob))) return;
             try
             {
-                var jobs = (await Sql.GetUnfinishedJobs())
+                var jobs = (await Sql.GetAllJobs(span))
                     .Where(j => j.Payload is T)
                     .OrderByDescending(j => j.Created)
                     .Take(10);
@@ -134,7 +138,7 @@ namespace Wabbajack.BuildServer
                 foreach (var job in jobs)
                 {
                     if (job.Started == null || job.Ended == null) return;
-                    if (DateTime.Now - job.Ended < span) return;
+                    if (DateTime.UtcNow - job.Ended < span) return;
                 }
                 await Sql.EnqueueJob(new Job
                 {

Wabbajack.BuildServer/Models/Sql/SqlService.cs

@@ -167,7 +167,7 @@ namespace Wabbajack.BuildServer.Model.Models
                         AND DATEADD(DAY, number+1, dbo.MinMetricDate()) < dbo.MaxMetricDate()) as d
                         ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action
                         WHERE d.Action = @action
-                        AND d.Date >= DATEADD(month, -1, GETDATE())
+                        AND d.Date >= DATEADD(month, -1, GETUTCDATE())
                         group by d.Date, d.GroupingSubject, d.Action
                         ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action}))
                 .ToList();
@@ -184,7 +184,7 @@ namespace Wabbajack.BuildServer.Model.Models
         {
             await using var conn = await Open();
             await conn.ExecuteAsync(
-                @"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)",
+                @"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETUTCDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)",
                 new {
                     job.Priority,
                     PrimaryKeyString = job.Payload.PrimaryKeyString,
@@ -201,7 +201,7 @@ namespace Wabbajack.BuildServer.Model.Models
         {
             await using var conn = await Open();
             await conn.ExecuteAsync(
-                @"UPDATE dbo.Jobs SET Ended = GETDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id",
+                @"UPDATE dbo.Jobs SET Ended = GETUTCDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id",
                 new {
                     job.Id,
                     Success = job.Result.ResultType == JobResultType.Success,
@@ -221,7 +221,7 @@ namespace Wabbajack.BuildServer.Model.Models
         {
             await using var conn = await Open();
             var result = await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>(
-                @"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy 
+                @"UPDATE jobs SET Started = GETUTCDATE(), RunBy = @RunBy 
                         WHERE ID in (SELECT TOP(1) ID FROM Jobs 
                                        WHERE Started is NULL
                                        AND PrimaryKeyString NOT IN (SELECT PrimaryKeyString from jobs WHERE Started IS NOT NULL and Ended IS NULL)
@@ -229,14 +229,14 @@ namespace Wabbajack.BuildServer.Model.Models
                       SELECT TOP(1) Id, Started, Ended, Created, Payload, Priority FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC",
                 new {RunBy = Guid.NewGuid().ToString()});
             return result.Select(k =>
-            new Job {
+                new Job {
-                Id = k.Item1,
+                    Id = k.Item1,
-                Started = k.Item2,
+                    Started = k.Item2,
-                Ended = k.Item3,
+                    Ended = k.Item3,
-                Created = k.Item4,
+                    Created = k.Item4,
-                Payload = k.Item5,
+                    Payload = k.Item5,
-                Priority = (Job.JobPriority)k.Item6
+                    Priority = (Job.JobPriority)k.Item6
-            }).FirstOrDefault();
+                }).FirstOrDefault();
     }
         
         
@@ -244,8 +244,16 @@ namespace Wabbajack.BuildServer.Model.Models
         {
             await using var conn = await Open();
             var results =
-                await conn.QueryAsync<Job>("SELECT * from dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL ");
+                await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority FROM dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL ");
-            return results;
+            return results.Select(k =>
+                new Job {
+                    Id = k.Item1,
+                    Started = k.Item2,
+                    Ended = k.Item3,
+                    Created = k.Item4,
+                    Payload = k.Item5,
+                    Priority = (Job.JobPriority)k.Item6
+                });
         }
 
 
@@ -253,8 +261,16 @@ namespace Wabbajack.BuildServer.Model.Models
         {
             await using var conn = await Open();
             var results =
-                await conn.QueryAsync<Job>("SELECT * from dbo.Jobs WHERE Ended IS NULL ");
+                await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority from dbo.Jobs WHERE Ended IS NULL ");
-            return results;
+            return results.Select(k =>
+                new Job {
+                    Id = k.Item1,
+                    Started = k.Item2,
+                    Ended = k.Item3,
+                    Created = k.Item4,
+                    Payload = k.Item5,
+                    Priority = (Job.JobPriority)k.Item6
+                });
         }
 
         
@@ -353,7 +369,7 @@ namespace Wabbajack.BuildServer.Model.Models
         public async Task<IEnumerable<UploadedFile>> AllUploadedFiles()
         {
             await using var conn = await Open();
-            return await conn.QueryAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles ORDER BY UploadDate DESC");
+            return await conn.QueryAsync<UploadedFile>("SELECT Id, Name, Size, UploadedBy as Uploader, Hash, UploadDate, CDNName FROM dbo.UploadedFiles ORDER BY UploadDate DESC");
         }
         
         public async Task DeleteUploadedFile(Guid dupId)
@@ -853,5 +869,22 @@ namespace Wabbajack.BuildServer.Model.Models
         
 
         #endregion
+
+        public async Task<IEnumerable<Job>> GetAllJobs(TimeSpan from)
+        {
+            await using var conn = await Open();
+            var results =
+                await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority from dbo.Jobs WHERE Created >= @FromTime ",
+                    new {FromTime = DateTime.UtcNow - from});
+            return results.Select(k =>
+                new Job {
+                    Id = k.Item1,
+                    Started = k.Item2,
+                    Ended = k.Item3,
+                    Created = k.Item4,
+                    Payload = k.Item5,
+                    Priority = (Job.JobPriority)k.Item6
+                });
+        }
     }
 }

Wabbajack.Common/Json.cs

@@ -98,7 +98,7 @@ namespace Wabbajack.Common
             }
         }
 
-        private class RelativePathConverter : JsonConverter<RelativePath>
+        public class RelativePathConverter : JsonConverter<RelativePath>
         {
             public override void WriteJson(JsonWriter writer, RelativePath value, JsonSerializer serializer)
             {

Wabbajack.Common/Paths.cs

@@ -9,6 +9,7 @@ using System.Reflection;
 using System.Text;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
+using Newtonsoft.Json;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
 using File = Alphaleonis.Win32.Filesystem.File;
 using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
@@ -405,6 +406,7 @@ namespace Wabbajack.Common
         }
     }
 
+    [JsonConverter(typeof(Utils.RelativePathConverter))]
     public struct RelativePath : IPath, IEquatable<RelativePath>, IComparable<RelativePath>
     {
         private readonly string? _nullable_path;