Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement a primary key on jobs so that they only run one at a time

Browse Source
This commit is contained in:
Timothy Baldridge 2020-04-11 07:59:15 -06:00
parent ac7b4ed3ac
commit c02c1cd693
13 changed files with 41 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Wabbajack.BuildServer.Test/JobQueueTests.cs
View File

@ -41,6 +41,10 @@ namespace Wabbajack.BuildServer.Test
var found = await sqlService.GetJob();
Assert.NotNull(found);
Assert.Equal(pri, found.Priority);
found.Result = JobResult.Success();
// Finish the job so the next can run
await sqlService.FinishJob(found);
}
}

1
Wabbajack.BuildServer.Test/sql/wabbajack_db.sql
View File

@ -204,6 +204,7 @@ GO
CREATE TABLE [dbo].[Jobs](
[Id] [bigint] IDENTITY(1,1) NOT NULL,
[Priority] [int] NOT NULL,
[PrimaryKeyString] [nvarchar](max) NULL,
[Started] [datetime] NULL,
[Ended] [datetime] NULL,
[Created] [datetime] NOT NULL,

4
Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs
View File

@ -30,6 +30,10 @@ namespace Wabbajack.BuildServer.Models.JobQueue
public virtual bool UsesNexus { get; } = false;
public abstract Task<JobResult> Execute(SqlService sql,AppSettings settings);
protected abstract IEnumerable<object> PrimaryKey { get; }
public string PrimaryKeyString => string.Join("|", PrimaryKey.Cons(this.GetType().Name).Select(i => i.ToString()));
static AJobPayload()
{

3
Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using System.Linq;
using Wabbajack.BuildServer.Model.Models;
@ -38,6 +39,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => new object[0];
private static async Task EnqueueFromList(SqlService sql, ModlistMetadata list, WorkQueue queue)
{
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);

5
Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs
View File

@ -1,4 +1,5 @@
using System.Linq;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
@ -65,5 +66,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
}
}
protected override IEnumerable<object> PrimaryKey => new object[0];
}
}

2
Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs
View File

@ -71,6 +71,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => new object[0];
public static DateTime LastNexusSync { get; set; } = DateTime.Now;
public static async Task<long> UpdateNexusCacheFast(SqlService sql)
{

3
Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
@ -64,5 +65,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => new object[0];
}
}

1
Wabbajack.BuildServer/Models/Jobs/IndexJob.cs
View File

@ -63,6 +63,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => Archive.State.PrimaryKey;
}
}

2
Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs
View File

@ -69,5 +69,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
}
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => new object[0];
}
}

5
Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Model.Models;
@ -43,7 +44,9 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => new object[0];
private async Task ValidateList(SqlService sql, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
{
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);

3
Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
@ -72,6 +73,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => new object[] {FileId};
public class Progress : IProgress<FluentFTP.FtpProgress>
{
private RelativePath _name;

3
Wabbajack.BuildServer/Models/PatchArchive.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
@ -74,6 +75,8 @@ namespace Wabbajack.BuildServer.Models
}
protected override IEnumerable<object> PrimaryKey => new object[] {Src, DestPK};
public static AbsolutePath CdnPath(Hash srcHash, Hash destHash)
{
return $"updates/{srcHash.ToHex()}_{destHash.ToHex()}".RelativeTo(AbsolutePath.EntryPoint);

9
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -181,9 +181,10 @@ namespace Wabbajack.BuildServer.Model.Models
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"INSERT INTO dbo.Jobs (Created, Priority, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @Payload, @OnSuccess)",
@"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)",
new {
job.Priority,
PrimaryKeyString = job.Payload.PrimaryKeyString,
Payload = job.Payload.ToJson(),
OnSuccess = job.OnSuccess?.ToJson() ?? null});
}
@ -217,7 +218,11 @@ namespace Wabbajack.BuildServer.Model.Models
{
await using var conn = await Open();
var result = await conn.QueryAsync<Job>(
@"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy WHERE ID in (SELECT TOP(1) ID FROM Jobs WHERE Started is NULL ORDER BY Priority DESC, Created);
@"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy
WHERE ID in (SELECT TOP(1) ID FROM Jobs
WHERE Started is NULL
AND PrimaryKeyString NOT IN (SELECT PrimaryKeyString from jobs WHERE Started IS NOT NULL and Ended IS NULL)
ORDER BY Priority DESC, Created);
SELECT TOP(1) * FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC",
new {RunBy = Guid.NewGuid().ToString()});
return result.FirstOrDefault();