Implement a primary key on jobs so that they only run one at a time

Wabbajack.BuildServer.Test/JobQueueTests.cs

@@ -41,6 +41,10 @@ namespace Wabbajack.BuildServer.Test
                 var found = await sqlService.GetJob();
                 Assert.NotNull(found);
                 Assert.Equal(pri, found.Priority);
+                found.Result = JobResult.Success();
+                
+                // Finish the job so the next can run
+                await sqlService.FinishJob(found);
             }
         }
 

Wabbajack.BuildServer.Test/sql/wabbajack_db.sql

@@ -204,6 +204,7 @@ GO
 CREATE TABLE [dbo].[Jobs](
                              [Id] [bigint] IDENTITY(1,1) NOT NULL,
                              [Priority] [int] NOT NULL,
+                             [PrimaryKeyString] [nvarchar](max) NULL,
                              [Started] [datetime] NULL,
                              [Ended] [datetime] NULL,
                              [Created] [datetime] NOT NULL,

Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs

@@ -30,6 +30,10 @@ namespace Wabbajack.BuildServer.Models.JobQueue
         public virtual bool UsesNexus { get; } = false;
 
         public abstract Task<JobResult> Execute(SqlService sql,AppSettings settings);
+        
+        protected abstract IEnumerable<object> PrimaryKey { get; }
+
+        public string PrimaryKeyString => string.Join("|", PrimaryKey.Cons(this.GetType().Name).Select(i => i.ToString()));
 
         static AJobPayload()
         {

Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using System.Linq;
 using Wabbajack.BuildServer.Model.Models;
@@ -38,6 +39,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
             return JobResult.Success();
         }
 
+        protected override IEnumerable<object> PrimaryKey => new object[0];
+
         private static async Task EnqueueFromList(SqlService sql, ModlistMetadata list, WorkQueue queue)
         {
             var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);

Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs

@@ -1,4 +1,5 @@
-using System.Linq;
+using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using Wabbajack.BuildServer.Models.JobQueue;
 using Wabbajack.Common;
@@ -65,5 +66,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
             }
             
         }
+
+        protected override IEnumerable<object> PrimaryKey => new object[0];
     }
 }

Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs

@@ -71,6 +71,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
             return JobResult.Success();
         }
 
+        protected override IEnumerable<object> PrimaryKey => new object[0];
+
         public static DateTime LastNexusSync { get; set; } = DateTime.Now;
         public static async Task<long> UpdateNexusCacheFast(SqlService sql)
         {

Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Net.Http;
 using System.Threading.Tasks;
@@ -64,5 +65,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
             return JobResult.Success();
 
         }
+
+        protected override IEnumerable<object> PrimaryKey => new object[0];
     }
 }

Wabbajack.BuildServer/Models/Jobs/IndexJob.cs

@@ -63,6 +63,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
             return JobResult.Success();
         }
 
+        protected override IEnumerable<object> PrimaryKey => Archive.State.PrimaryKey;
     }
     
 }

Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs

@@ -69,5 +69,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
             }
             return JobResult.Success();
         }
+
+        protected override IEnumerable<object> PrimaryKey => new object[0];
     }
 }

Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Wabbajack.BuildServer.Model.Models;
@@ -43,7 +44,9 @@ namespace Wabbajack.BuildServer.Models.Jobs
 
             return JobResult.Success();
         }
-        
+
+        protected override IEnumerable<object> PrimaryKey => new object[0];
+
         private async Task ValidateList(SqlService sql, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
         {
             var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);

Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Net;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
@@ -72,6 +73,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
             return JobResult.Success();
         }
 
+        protected override IEnumerable<object> PrimaryKey => new object[] {FileId};
+
         public class Progress : IProgress<FluentFTP.FtpProgress>
         {
             private RelativePath _name;

Wabbajack.BuildServer/Models/PatchArchive.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Net;
 using System.Threading.Tasks;
 using FluentFTP;
@@ -74,6 +75,8 @@ namespace Wabbajack.BuildServer.Models
             
         }
 
+        protected override IEnumerable<object> PrimaryKey => new object[] {Src, DestPK};
+
         public static AbsolutePath CdnPath(Hash srcHash, Hash destHash)
         {
             return $"updates/{srcHash.ToHex()}_{destHash.ToHex()}".RelativeTo(AbsolutePath.EntryPoint);

Wabbajack.BuildServer/Models/Sql/SqlService.cs

@@ -181,9 +181,10 @@ namespace Wabbajack.BuildServer.Model.Models
         {
             await using var conn = await Open();
             await conn.ExecuteAsync(
-                @"INSERT INTO dbo.Jobs (Created, Priority, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @Payload, @OnSuccess)",
+                @"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)",
                 new {
                     job.Priority,
+                    PrimaryKeyString = job.Payload.PrimaryKeyString,
                     Payload = job.Payload.ToJson(), 
                     OnSuccess = job.OnSuccess?.ToJson() ?? null});
         }
@@ -217,7 +218,11 @@ namespace Wabbajack.BuildServer.Model.Models
         {
             await using var conn = await Open();
             var result = await conn.QueryAsync<Job>(
-                @"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy WHERE ID in (SELECT TOP(1) ID FROM Jobs WHERE Started is NULL ORDER BY Priority DESC, Created);
+                @"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy 
+                        WHERE ID in (SELECT TOP(1) ID FROM Jobs 
+                                       WHERE Started is NULL
+                                       AND PrimaryKeyString NOT IN (SELECT PrimaryKeyString from jobs WHERE Started IS NOT NULL and Ended IS NULL)
+                                       ORDER BY Priority DESC, Created);
                       SELECT TOP(1) * FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC",
                 new {RunBy = Guid.NewGuid().ToString()});
             return result.FirstOrDefault();