diff --git a/Wabbajack.BuildServer.Test/JobQueueTests.cs b/Wabbajack.BuildServer.Test/JobQueueTests.cs index 4d3125ec..ab1ae72a 100644 --- a/Wabbajack.BuildServer.Test/JobQueueTests.cs +++ b/Wabbajack.BuildServer.Test/JobQueueTests.cs @@ -41,6 +41,10 @@ namespace Wabbajack.BuildServer.Test var found = await sqlService.GetJob(); Assert.NotNull(found); Assert.Equal(pri, found.Priority); + found.Result = JobResult.Success(); + + // Finish the job so the next can run + await sqlService.FinishJob(found); } } diff --git a/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql b/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql index 556563ec..c74c838b 100644 --- a/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql +++ b/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql @@ -204,6 +204,7 @@ GO CREATE TABLE [dbo].[Jobs]( [Id] [bigint] IDENTITY(1,1) NOT NULL, [Priority] [int] NOT NULL, + [PrimaryKeyString] [nvarchar](max) NULL, [Started] [datetime] NULL, [Ended] [datetime] NULL, [Created] [datetime] NOT NULL, diff --git a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs index a28f2048..46ae9e5c 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs @@ -30,6 +30,10 @@ namespace Wabbajack.BuildServer.Models.JobQueue public virtual bool UsesNexus { get; } = false; public abstract Task Execute(SqlService sql,AppSettings settings); + + protected abstract IEnumerable PrimaryKey { get; } + + public string PrimaryKeyString => string.Join("|", PrimaryKey.Cons(this.GetType().Name).Select(i => i.ToString())); static AJobPayload() { diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs index eba18c18..21e01093 100644 --- a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs +++ b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Threading.Tasks; using System.Linq; using Wabbajack.BuildServer.Model.Models; @@ -38,6 +39,8 @@ namespace Wabbajack.BuildServer.Models.Jobs return JobResult.Success(); } + protected override IEnumerable PrimaryKey => new object[0]; + private static async Task EnqueueFromList(SqlService sql, ModlistMetadata list, WorkQueue queue) { var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension); diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs index 7a393b4e..003094bf 100644 --- a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs +++ b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs @@ -1,4 +1,5 @@ -using System.Linq; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; @@ -65,5 +66,7 @@ namespace Wabbajack.BuildServer.Models.Jobs } } + + protected override IEnumerable PrimaryKey => new object[0]; } } diff --git a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs index 07e893cc..8e591126 100644 --- a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs @@ -71,6 +71,8 @@ namespace Wabbajack.BuildServer.Models.Jobs return JobResult.Success(); } + protected override IEnumerable PrimaryKey => new object[0]; + public static DateTime LastNexusSync { get; set; } = DateTime.Now; public static async Task UpdateNexusCacheFast(SqlService sql) { diff --git a/Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs b/Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs index 983d2d47..b61450f2 100644 --- a/Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs +++ b/Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Threading.Tasks; @@ -64,5 +65,7 @@ namespace Wabbajack.BuildServer.Models.Jobs return JobResult.Success(); } + + protected override IEnumerable PrimaryKey => new object[0]; } } diff --git a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs index 3d2d1592..9aca15f7 100644 --- a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs @@ -63,6 +63,7 @@ namespace Wabbajack.BuildServer.Models.Jobs return JobResult.Success(); } + protected override IEnumerable PrimaryKey => Archive.State.PrimaryKey; } } diff --git a/Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs b/Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs index 56cdcfa4..2ebbb278 100644 --- a/Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs +++ b/Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs @@ -69,5 +69,7 @@ namespace Wabbajack.BuildServer.Models.Jobs } return JobResult.Success(); } + + protected override IEnumerable PrimaryKey => new object[0]; } } diff --git a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs index 77af3e23..f2116ab0 100644 --- a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs +++ b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Wabbajack.BuildServer.Model.Models; @@ -43,7 +44,9 @@ namespace Wabbajack.BuildServer.Models.Jobs return JobResult.Success(); } - + + protected override IEnumerable PrimaryKey => new object[0]; + private async Task ValidateList(SqlService sql, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists) { var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension); diff --git a/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs b/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs index ff7d9619..1460a233 100644 --- a/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs +++ b/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Net; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; @@ -72,6 +73,8 @@ namespace Wabbajack.BuildServer.Models.Jobs return JobResult.Success(); } + protected override IEnumerable PrimaryKey => new object[] {FileId}; + public class Progress : IProgress { private RelativePath _name; diff --git a/Wabbajack.BuildServer/Models/PatchArchive.cs b/Wabbajack.BuildServer/Models/PatchArchive.cs index d3855623..a895a068 100644 --- a/Wabbajack.BuildServer/Models/PatchArchive.cs +++ b/Wabbajack.BuildServer/Models/PatchArchive.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Net; using System.Threading.Tasks; using FluentFTP; @@ -74,6 +75,8 @@ namespace Wabbajack.BuildServer.Models } + protected override IEnumerable PrimaryKey => new object[] {Src, DestPK}; + public static AbsolutePath CdnPath(Hash srcHash, Hash destHash) { return $"updates/{srcHash.ToHex()}_{destHash.ToHex()}".RelativeTo(AbsolutePath.EntryPoint); diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index 87a6be95..00351769 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -181,9 +181,10 @@ namespace Wabbajack.BuildServer.Model.Models { await using var conn = await Open(); await conn.ExecuteAsync( - @"INSERT INTO dbo.Jobs (Created, Priority, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @Payload, @OnSuccess)", + @"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)", new { job.Priority, + PrimaryKeyString = job.Payload.PrimaryKeyString, Payload = job.Payload.ToJson(), OnSuccess = job.OnSuccess?.ToJson() ?? null}); } @@ -217,7 +218,11 @@ namespace Wabbajack.BuildServer.Model.Models { await using var conn = await Open(); var result = await conn.QueryAsync( - @"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy WHERE ID in (SELECT TOP(1) ID FROM Jobs WHERE Started is NULL ORDER BY Priority DESC, Created); + @"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy + WHERE ID in (SELECT TOP(1) ID FROM Jobs + WHERE Started is NULL + AND PrimaryKeyString NOT IN (SELECT PrimaryKeyString from jobs WHERE Started IS NOT NULL and Ended IS NULL) + ORDER BY Priority DESC, Created); SELECT TOP(1) * FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC", new {RunBy = Guid.NewGuid().ToString()}); return result.FirstOrDefault();