diff --git a/Wabbajack.App.Test/BasicUITests.cs b/Wabbajack.App.Test/BasicUITests.cs index 99ade169..1b559a8a 100644 --- a/Wabbajack.App.Test/BasicUITests.cs +++ b/Wabbajack.App.Test/BasicUITests.cs @@ -7,7 +7,7 @@ namespace Wabbajack.App.Test { public class BasicUITests { - /* + [StaFact] public async Task CanCompileASimpleModlist() { @@ -20,6 +20,6 @@ namespace Wabbajack.App.Test window.Close(); Assert.True(true); } - */ + } } diff --git a/Wabbajack.BuildServer.Test/ABuildServerTest.cs b/Wabbajack.BuildServer.Test/ABuildServerTest.cs new file mode 100644 index 00000000..8be5a92a --- /dev/null +++ b/Wabbajack.BuildServer.Test/ABuildServerTest.cs @@ -0,0 +1,130 @@ +using System; +using System.Collections.Generic; +using System.Data.SqlClient; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Wabbajack.BuildServer.Model.Models; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.BuildServer.Test +{ + public class ABuildServerTest : IAsyncLifetime + { + private static string CONN_STR = @"Data Source=.\SQLEXPRESS;Integrated Security=True;"; + private AppSettings _appSettings; + protected SqlService _sqlService; + private string DBName { get; } + + public ABuildServerTest(ITestOutputHelper helper) + { + TestContext = helper; + DBName = "test_db" + Guid.NewGuid().ToString().Replace("-", "_"); + _appSettings = MakeAppSettings(); + _sqlService = new SqlService(_appSettings); + } + + private AppSettings MakeAppSettings() + { + return new AppSettings + { + SqlConnection = CONN_STR + $"Initial Catalog={DBName}" + }; + } + + public ITestOutputHelper TestContext { get;} + + public async Task InitializeAsync() + { + await CreateSchema(); + } + + private async Task CreateSchema() + { + TestContext.WriteLine("Creating Database"); + //var conn = new SqlConnection("Data Source=localhost,1433;User ID=test;Password=test;MultipleActiveResultSets=true"); + await using var conn = new SqlConnection(CONN_STR); + + await conn.OpenAsync(); + //await new SqlCommand($"CREATE DATABASE {DBName};", conn).ExecuteNonQueryAsync(); + + await using var schemaStream = Assembly.GetExecutingAssembly().GetManifestResourceStream("Wabbajack.BuildServer.Test.sql.wabbajack_db.sql"); + await using var ms = new MemoryStream(); + await schemaStream.CopyToAsync(ms); + var schemaString = Encoding.UTF8.GetString(ms.ToArray()).Replace("wabbajack_prod", $"{DBName}"); + + foreach (var statement in SplitSqlStatements(schemaString)) + { + await new SqlCommand(statement, conn).ExecuteNonQueryAsync(); + + } + } + + private static IEnumerable SplitSqlStatements(string sqlScript) + { + // Split by "GO" statements + var statements = Regex.Split( + sqlScript, + @"^[\t \r\n]*GO[\t \r\n]*\d*[\t ]*(?:--.*)?$", + RegexOptions.Multiline | + RegexOptions.IgnorePatternWhitespace | + RegexOptions.IgnoreCase); + + // Remove empties, trim, and return + return statements + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Select(x => x.Trim(' ', '\r', '\n')); + } + + async Task IAsyncLifetime.DisposeAsync() + { + TestContext.WriteLine("Deleting Database"); + await using var conn = new SqlConnection(CONN_STR); + + await conn.OpenAsync(); + await KillAll(conn); + await new SqlCommand($"DROP DATABASE {DBName};", conn).ExecuteNonQueryAsync(); + } + + private async Task KillAll(SqlConnection conn) + { + await new SqlCommand($@" + DECLARE @Spid INT + DECLARE @ExecSQL VARCHAR(255) + + DECLARE KillCursor CURSOR LOCAL STATIC READ_ONLY FORWARD_ONLY + FOR + SELECT DISTINCT SPID + FROM MASTER..SysProcesses + WHERE DBID = DB_ID('{DBName}') + + OPEN KillCursor + + -- Grab the first SPID + FETCH NEXT + FROM KillCursor + INTO @Spid + + WHILE @@FETCH_STATUS = 0 + BEGIN + SET @ExecSQL = 'KILL ' + CAST(@Spid AS VARCHAR(50)) + + EXEC (@ExecSQL) + + -- Pull the next SPID + FETCH NEXT + FROM KillCursor + INTO @Spid + END + + CLOSE KillCursor + + DEALLOCATE KillCursor", conn).ExecuteNonQueryAsync(); + } + + } +} diff --git a/Wabbajack.BuildServer.Test/JobQueueTests.cs b/Wabbajack.BuildServer.Test/JobQueueTests.cs new file mode 100644 index 00000000..0593d1dd --- /dev/null +++ b/Wabbajack.BuildServer.Test/JobQueueTests.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Wabbajack.BuildServer.Model.Models; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.BuildServer.Models.Jobs; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.BuildServer.Test +{ + public class BasicTest : ABuildServerTest + { + [Fact] + public async Task CanEneuqueAndGetJobs() + { + var job = new Job {Payload = new GetNexusUpdatesJob()}; + await _sqlService.EnqueueJob(job); + var found = await _sqlService.GetJob(); + Assert.NotNull(found); + Assert.IsAssignableFrom(found.Payload); + } + + [Fact] + public async Task PriorityMatters() + { + var priority = new List + { + Job.JobPriority.Normal, Job.JobPriority.High, Job.JobPriority.Low + }; + foreach (var pri in priority) + await _sqlService.EnqueueJob(new Job {Payload = new GetNexusUpdatesJob(), Priority = pri}); + + foreach (var pri in priority.OrderByDescending(p => (int)p)) + { + var found = await _sqlService.GetJob(); + Assert.NotNull(found); + Assert.Equal(pri, found.Priority); + } + } + + public BasicTest(ITestOutputHelper helper) : base(helper) + { + + + } + } +} diff --git a/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj b/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj new file mode 100644 index 00000000..c9452831 --- /dev/null +++ b/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj @@ -0,0 +1,25 @@ + + + + netcoreapp3.1 + + false + + + + + + + + + + + + + + + + + + + diff --git a/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql b/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql new file mode 100644 index 00000000..1ba17616 --- /dev/null +++ b/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql @@ -0,0 +1,406 @@ +USE [master] +GO +/****** Object: Database [wabbajack_prod] Script Date: 3/28/2020 4:58:58 PM ******/ +CREATE DATABASE [wabbajack_prod] + CONTAINMENT = NONE + WITH CATALOG_COLLATION = DATABASE_DEFAULT +GO +ALTER DATABASE [wabbajack_prod] SET COMPATIBILITY_LEVEL = 150 +GO +IF (1 = FULLTEXTSERVICEPROPERTY('IsFullTextInstalled')) + begin + EXEC [wabbajack_prod].[dbo].[sp_fulltext_database] @action = 'enable' + end +GO +ALTER DATABASE [wabbajack_prod] SET ANSI_NULL_DEFAULT OFF +GO +ALTER DATABASE [wabbajack_prod] SET ANSI_NULLS OFF +GO +ALTER DATABASE [wabbajack_prod] SET ANSI_PADDING OFF +GO +ALTER DATABASE [wabbajack_prod] SET ANSI_WARNINGS OFF +GO +ALTER DATABASE [wabbajack_prod] SET ARITHABORT OFF +GO +ALTER DATABASE [wabbajack_prod] SET AUTO_CLOSE OFF +GO +ALTER DATABASE [wabbajack_prod] SET AUTO_SHRINK OFF +GO +ALTER DATABASE [wabbajack_prod] SET AUTO_UPDATE_STATISTICS ON +GO +ALTER DATABASE [wabbajack_prod] SET CURSOR_CLOSE_ON_COMMIT OFF +GO +ALTER DATABASE [wabbajack_prod] SET CURSOR_DEFAULT GLOBAL +GO +ALTER DATABASE [wabbajack_prod] SET CONCAT_NULL_YIELDS_NULL OFF +GO +ALTER DATABASE [wabbajack_prod] SET NUMERIC_ROUNDABORT OFF +GO +ALTER DATABASE [wabbajack_prod] SET QUOTED_IDENTIFIER OFF +GO +ALTER DATABASE [wabbajack_prod] SET RECURSIVE_TRIGGERS OFF +GO +ALTER DATABASE [wabbajack_prod] SET DISABLE_BROKER +GO +ALTER DATABASE [wabbajack_prod] SET AUTO_UPDATE_STATISTICS_ASYNC OFF +GO +ALTER DATABASE [wabbajack_prod] SET DATE_CORRELATION_OPTIMIZATION OFF +GO +ALTER DATABASE [wabbajack_prod] SET TRUSTWORTHY OFF +GO +ALTER DATABASE [wabbajack_prod] SET ALLOW_SNAPSHOT_ISOLATION OFF +GO +ALTER DATABASE [wabbajack_prod] SET PARAMETERIZATION SIMPLE +GO +ALTER DATABASE [wabbajack_prod] SET READ_COMMITTED_SNAPSHOT OFF +GO +ALTER DATABASE [wabbajack_prod] SET HONOR_BROKER_PRIORITY OFF +GO +ALTER DATABASE [wabbajack_prod] SET RECOVERY FULL +GO +ALTER DATABASE [wabbajack_prod] SET MULTI_USER +GO +ALTER DATABASE [wabbajack_prod] SET PAGE_VERIFY CHECKSUM +GO +ALTER DATABASE [wabbajack_prod] SET DB_CHAINING OFF +GO +ALTER DATABASE [wabbajack_prod] SET FILESTREAM( NON_TRANSACTED_ACCESS = OFF ) +GO +ALTER DATABASE [wabbajack_prod] SET TARGET_RECOVERY_TIME = 60 SECONDS +GO +ALTER DATABASE [wabbajack_prod] SET DELAYED_DURABILITY = DISABLED +GO +EXEC sys.sp_db_vardecimal_storage_format N'wabbajack_prod', N'ON' +GO +ALTER DATABASE [wabbajack_prod] SET QUERY_STORE = OFF +GO +USE [wabbajack_prod] +GO +/****** Object: Schema [test] Script Date: 3/28/2020 4:58:59 PM ******/ +CREATE SCHEMA [test] +GO +/****** Object: UserDefinedTableType [dbo].[ArchiveContentType] Script Date: 3/28/2020 4:58:59 PM ******/ +CREATE TYPE [dbo].[ArchiveContentType] AS TABLE( + [Parent] [bigint] NOT NULL, + [Child] [bigint] NOT NULL, + [Path] [nvarchar](max) NOT NULL + ) +GO +/****** Object: UserDefinedTableType [dbo].[IndexedFileType] Script Date: 3/28/2020 4:58:59 PM ******/ +CREATE TYPE [dbo].[IndexedFileType] AS TABLE( + [Hash] [bigint] NOT NULL, + [Sha256] [binary](32) NOT NULL, + [Sha1] [binary](20) NOT NULL, + [Md5] [binary](16) NOT NULL, + [Crc32] [int] NOT NULL, + [Size] [bigint] NOT NULL + ) +GO +/****** Object: UserDefinedFunction [dbo].[Base64ToLong] Script Date: 3/28/2020 4:58:59 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +-- ============================================= +-- Author: +-- Create date: +-- Description: +-- ============================================= +CREATE FUNCTION [dbo].[Base64ToLong] +( + -- Add the parameters for the function here + @Input varchar +) + RETURNS bigint +AS +BEGIN + -- Declare the return variable here + DECLARE @ResultVar bigint + + -- Add the T-SQL statements to compute the return value here + SELECT @ResultVar = CAST('string' as varbinary(max)) FOR XML PATH(''), BINARY BASE64 + + -- Return the result of the function + RETURN @ResultVar + +END +GO +/****** Object: UserDefinedFunction [dbo].[MaxMetricDate] Script Date: 3/28/2020 4:58:59 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO + +-- ============================================= +-- Author: +-- Create date: +-- Description: +-- ============================================= +CREATE FUNCTION [dbo].[MaxMetricDate] +( +) + RETURNS date +AS +BEGIN + -- Declare the return variable here + DECLARE @Result date + + -- Add the T-SQL statements to compute the return value here + SELECT @Result = max(Timestamp) from dbo.Metrics where MetricsKey is not null + + -- Return the result of the function + RETURN @Result + +END +GO +/****** Object: UserDefinedFunction [dbo].[MinMetricDate] Script Date: 3/28/2020 4:58:59 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO + +-- ============================================= +-- Author: +-- Create date: +-- Description: +-- ============================================= +CREATE FUNCTION [dbo].[MinMetricDate] +( +) + RETURNS date +AS +BEGIN + -- Declare the return variable here + DECLARE @Result date + + -- Add the T-SQL statements to compute the return value here + SELECT @Result = min(Timestamp) from dbo.Metrics WHERE MetricsKey is not null + + -- Return the result of the function + RETURN @Result + +END +GO +/****** Object: Table [dbo].[IndexedFile] Script Date: 3/28/2020 4:58:59 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[IndexedFile]( + [Hash] [bigint] NOT NULL, + [Sha256] [binary](32) NOT NULL, + [Sha1] [binary](20) NOT NULL, + [Md5] [binary](16) NOT NULL, + [Crc32] [int] NOT NULL, + [Size] [bigint] NOT NULL, + CONSTRAINT [PK_IndexedFile] PRIMARY KEY CLUSTERED + ( + [Hash] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO + +/****** Object: Table [dbo].[Jobs] ******/ +CREATE TABLE [dbo].[Jobs]( + [Id] [bigint] IDENTITY(1,1) NOT NULL, + [Priority] [int] NOT NULL, + [Started] [datetime] NULL, + [Ended] [datetime] NULL, + [Created] [datetime] NOT NULL, + [Success] [tinyint] NULL, + [ResultContent] [nvarchar](max) NULL, + [Payload] [nvarchar](max) NULL, + [OnSuccess] [nvarchar](max) NULL, + [RunBy] [uniqueidentifier] NULL +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO + + + +/****** Object: Table [dbo].[ArchiveContent] Script Date: 3/28/2020 4:58:59 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[ArchiveContent]( + [Parent] [bigint] NOT NULL, + [Child] [bigint] NOT NULL, + [Path] [nvarchar](max) NULL, + [PathHash] AS (CONVERT([binary](32),hashbytes('SHA2_256',[Path]))) PERSISTED NOT NULL +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[AllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[AllFilesInArchive]( + [TopParent] [bigint] NOT NULL, + [Child] [bigint] NOT NULL, + CONSTRAINT [PK_AllFilesInArchive] PRIMARY KEY CLUSTERED + ( + [TopParent] ASC, + [Child] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO +/****** Object: View [dbo].[AllArchiveContent] Script Date: 3/28/2020 4:58:59 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO + + +CREATE VIEW [dbo].[AllArchiveContent] + WITH SCHEMABINDING +AS +SELECT af.TopParent, ac.Parent, af.Child, ac.Path, idx.Size +FROM + dbo.AllFilesInArchive af + LEFT JOIN dbo.ArchiveContent ac on af.Child = ac.Child + LEFT JOIN dbo.IndexedFile idx on af.Child = idx.Hash +GO +/****** Object: Table [dbo].[Metrics] Script Date: 3/28/2020 4:58:59 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[Metrics]( + [Id] [bigint] IDENTITY(1,1) NOT NULL, + [Timestamp] [datetime] NOT NULL, + [Action] [varchar](64) NOT NULL, + [Subject] [varchar](max) NOT NULL, + [MetricsKey] [varchar](64) NULL, + [GroupingSubject] AS (substring([Subject],(0),case when patindex('%[0-9].%',[Subject])=(0) then len([Subject])+(1) else patindex('%[0-9].%',[Subject]) end)), + CONSTRAINT [PK_Metrics] PRIMARY KEY CLUSTERED + ( + [Id] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Index [IX_Child] Script Date: 3/28/2020 4:58:59 PM ******/ +CREATE NONCLUSTERED INDEX [IX_Child] ON [dbo].[AllFilesInArchive] + ( + [Child] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +GO +/****** Object: Index [IX_ArchiveContent_Child] Script Date: 3/28/2020 4:58:59 PM ******/ +CREATE NONCLUSTERED INDEX [IX_ArchiveContent_Child] ON [dbo].[ArchiveContent] + ( + [Child] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +GO +SET ARITHABORT ON +SET CONCAT_NULL_YIELDS_NULL ON +SET QUOTED_IDENTIFIER ON +SET ANSI_NULLS ON +SET ANSI_PADDING ON +SET ANSI_WARNINGS ON +SET NUMERIC_ROUNDABORT OFF +GO +/****** Object: Index [PK_ArchiveContent] Script Date: 3/28/2020 4:58:59 PM ******/ +CREATE UNIQUE NONCLUSTERED INDEX [PK_ArchiveContent] ON [dbo].[ArchiveContent] + ( + [Parent] ASC, + [PathHash] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +GO +SET ANSI_PADDING ON +GO +/****** Object: Index [IX_IndexedFile_By_SHA256] Script Date: 3/28/2020 4:58:59 PM ******/ +CREATE UNIQUE NONCLUSTERED INDEX [IX_IndexedFile_By_SHA256] ON [dbo].[IndexedFile] + ( + [Sha256] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +GO +/****** Object: StoredProcedure [dbo].[MergeAllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +-- ============================================= +-- Author: +-- Create date: +-- Description: +-- ============================================= +CREATE PROCEDURE [dbo].[MergeAllFilesInArchive] +AS +BEGIN + -- SET NOCOUNT ON added to prevent extra result sets from + -- interfering with SELECT statements. + SET NOCOUNT ON; + + MERGE dbo.AllFilesInArchive t USING ( + SELECT DISTINCT TopParent, unpvt.Child + FROM + (SELECT a3.Parent AS P3, a2.Parent as P2, a1.Parent P1, a0.Parent P0, a0.Parent as Parent, a0.Child FROM + dbo.ArChiveContent a0 + LEFT JOIN dbo.ArChiveContent a1 ON a0.Parent = a1.Child + LEFT JOIN dbo.ArChiveContent a2 ON a1.Parent = a2.Child + LEFT JOIN dbo.ArChiveContent a3 ON a2.Parent = a3.Child) p + UNPIVOT + (TopParent For C IN (p.P3, p.P2, p.P1, p.P0)) as unpvt + LEFT JOIN dbo.IndexedFile idf on unpvt.Child = idf.Hash + WHERE TopParent is not null) s + ON t.TopParent = s.TopParent AND t.Child = s.Child + WHEN NOT MATCHED + THEN INSERT (TopParent, Child) VALUES (s.TopParent, s.Child); +END +GO +/****** Object: StoredProcedure [dbo].[MergeIndexedFiles] Script Date: 3/28/2020 4:58:59 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +-- Description: +-- ============================================= +CREATE PROCEDURE [dbo].[MergeIndexedFiles] + -- Add the parameters for the stored procedure here + @Files dbo.IndexedFileType READONLY, + @Contents dbo.ArchiveContentType READONLY +AS +BEGIN + -- SET NOCOUNT ON added to prevent extra result sets from + -- interfering with SELECT statements. + SET NOCOUNT ON; + BEGIN TRANSACTION; + + MERGE dbo.IndexedFile AS TARGET + USING (SELECT DISTINCT * FROM @Files) as SOURCE + ON (TARGET.Hash = SOURCE.HASH) + WHEN NOT MATCHED BY TARGET + THEN INSERT (Hash, Sha256, Sha1, Md5, Crc32, Size) + VALUES (Source.Hash, Source.Sha256, Source.Sha1, Source.Md5, Source.Crc32, Source.Size); + + MERGE dbo.ArchiveContent AS TARGET + USING (SELECT DISTINCT * FROM @Contents) as SOURCE + ON (TARGET.Parent = SOURCE.Parent AND TARGET.PathHash = CAST(HASHBYTES('SHA2_256', SOURCE.Path) as binary(32))) + WHEN NOT MATCHED + THEN INSERT (Parent, Child, Path) + VALUES (Source.Parent, Source.Child, Source.Path); + + MERGE dbo.AllFilesInArchive t USING ( + SELECT DISTINCT TopParent, unpvt.Child + FROM + (SELECT a3.Parent AS P3, a2.Parent as P2, a1.Parent P1, a0.Parent P0, a0.Parent as Parent, a0.Child FROM + dbo.ArChiveContent a0 + LEFT JOIN dbo.ArChiveContent a1 ON a0.Parent = a1.Child + LEFT JOIN dbo.ArChiveContent a2 ON a1.Parent = a2.Child + LEFT JOIN dbo.ArChiveContent a3 ON a2.Parent = a3.Child) p + UNPIVOT + (TopParent For C IN (p.P3, p.P2, p.P1, p.P0)) as unpvt + LEFT JOIN dbo.IndexedFile idf on unpvt.Child = idf.Hash + WHERE TopParent is not null + AND Child in (SELECT DISTINCT Hash FROM @Files)) s + ON t.TopParent = s.TopParent AND t.Child = s.Child + WHEN NOT MATCHED + THEN INSERT (TopParent, Child) VALUES (s.TopParent, s.Child); + + COMMIT; + +END +GO +USE [master] +GO +ALTER DATABASE [wabbajack_prod] SET READ_WRITE +GO diff --git a/Wabbajack.BuildServer/AppSettings.cs b/Wabbajack.BuildServer/AppSettings.cs index e584c725..c1734b36 100644 --- a/Wabbajack.BuildServer/AppSettings.cs +++ b/Wabbajack.BuildServer/AppSettings.cs @@ -9,6 +9,11 @@ namespace Wabbajack.BuildServer { config.Bind("WabbajackSettings", this); } + + public AppSettings() + { + + } public AbsolutePath DownloadDir { get; set; } public AbsolutePath ArchiveDir { get; set; } diff --git a/Wabbajack.BuildServer/Controllers/Jobs.cs b/Wabbajack.BuildServer/Controllers/Jobs.cs index 82763139..1a8fa84a 100644 --- a/Wabbajack.BuildServer/Controllers/Jobs.cs +++ b/Wabbajack.BuildServer/Controllers/Jobs.cs @@ -33,7 +33,7 @@ namespace Wabbajack.BuildServer.Controllers [HttpGet] [Route("enqueue_job/{JobName}")] - public async Task EnqueueJob(string JobName) + public async Task EnqueueJob(string JobName) { var jobtype = AJobPayload.NameToType[JobName]; var job = new Job{Priority = Job.JobPriority.High, Payload = (AJobPayload)jobtype.GetConstructor(new Type[0]).Invoke(new object?[0])}; diff --git a/Wabbajack.BuildServer/GraphQL/Query.cs b/Wabbajack.BuildServer/GraphQL/Query.cs index 5ff94f4c..4d336e4d 100644 --- a/Wabbajack.BuildServer/GraphQL/Query.cs +++ b/Wabbajack.BuildServer/GraphQL/Query.cs @@ -49,10 +49,10 @@ namespace Wabbajack.BuildServer.GraphQL FieldAsync>("job", arguments: new QueryArguments( - new QueryArgument {Name = "id", Description = "Id of the Job"}), + new QueryArgument {Name = "id", Description = "Id of the Job"}), resolve: async context => { - var id = context.GetArgument("id"); + var id = context.GetArgument("id"); var data = await db.Jobs.AsQueryable().Where(j => j.Id == id).ToListAsync(); return data; }); diff --git a/Wabbajack.BuildServer/Models/JobQueue/Job.cs b/Wabbajack.BuildServer/Models/JobQueue/Job.cs index 39c896fc..4c013d2d 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/Job.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/Job.cs @@ -19,24 +19,17 @@ namespace Wabbajack.BuildServer.Models.JobQueue High, } - [BsonId] public String Id { get; set; } = Guid.NewGuid().ToString(); + public long Id { get; set; } public DateTime? Started { get; set; } public DateTime? Ended { get; set; } public DateTime Created { get; set; } = DateTime.Now; public JobPriority Priority { get; set; } = JobPriority.Normal; - public JobResult Result { get; set; } public bool RequiresNexus { get; set; } = true; public AJobPayload Payload { get; set; } public Job OnSuccess { get; set; } - public static async Task Enqueue(DBContext db, Job job) - { - await db.Jobs.InsertOneAsync(job); - return job.Id; - } - public static async Task GetNext(DBContext db) { var filter = new BsonDocument diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index 34818b2e..51043f60 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -9,6 +9,7 @@ using Dapper; using Microsoft.Extensions.Configuration; using Wabbajack.BuildServer.Model.Models.Results; using Wabbajack.BuildServer.Models; +using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; using Wabbajack.VirtualFileSystem; @@ -164,6 +165,87 @@ namespace Wabbajack.BuildServer.Model.Models ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action})) .ToList(); } + + + #region JobRoutines + /// + /// Enqueue a Job into the Job queue to be run at a later time + /// + /// + /// + public async Task EnqueueJob(Job job) + { + await using var conn = await Open(); + await conn.ExecuteAsync( + @"INSERT INTO dbo.Jobs (Created, Priority, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @Payload, @OnSuccess)", + new { + Priority = job.Priority, + Payload = job.Payload.ToJSON(), + OnSuccess = job.OnSuccess?.ToJSON() ?? null}); + } + + /// + /// Enqueue a Job into the Job queue to be run at a later time + /// + /// + /// + public async Task FinishJob(Job job) + { + await using var conn = await Open(); + await conn.ExecuteAsync( + @"UPDATE dbo.Jobs SET Finshed = GETDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id", + new { + Id = job.Id, + Success = job.Result.ResultType == JobResultType.Success, + ResultPayload = job.Result.ToJSON() + + }); + + if (job.OnSuccess != null) + await EnqueueJob(job.OnSuccess); + } + + + /// + /// Get a Job from the Job queue to run. + /// + /// + public async Task GetJob() + { + await using var conn = await Open(); + var result = await conn.QueryAsync( + @"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy WHERE ID in (SELECT TOP(1) ID FROM Jobs WHERE Started is NULL ORDER BY Priority DESC, Created); + SELECT TOP(1) * FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC", + new {RunBy = Guid.NewGuid().ToString()}); + return result.FirstOrDefault(); + } + + + #endregion + + + #region TypeMappers + + static SqlService() + { + SqlMapper.AddTypeHandler(new PayloadMapper()); + } + + public class PayloadMapper : SqlMapper.TypeHandler + { + public override void SetValue(IDbDataParameter parameter, AJobPayload value) + { + parameter.Value = value.ToJSON(); + } + + public override AJobPayload Parse(object value) + { + return Utils.FromJSONString((string)value); + } + } + + + #endregion } } diff --git a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj index 7ce9b762..f0e6bf9e 100644 --- a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj +++ b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj @@ -80,6 +80,7 @@ + diff --git a/Wabbajack.BuildServer/sql/wabbajack_db.sql b/Wabbajack.BuildServer/sql/wabbajack_db.sql deleted file mode 100644 index cc1e84f0..00000000 Binary files a/Wabbajack.BuildServer/sql/wabbajack_db.sql and /dev/null differ diff --git a/Wabbajack.sln b/Wabbajack.sln index cf50ad1c..745a4294 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -42,6 +42,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Common.Test", "Wa EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.App.Test", "Wabbajack.App.Test\Wabbajack.App.Test.csproj", "{44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.BuildServer.Test", "Wabbajack.BuildServer.Test\Wabbajack.BuildServer.Test.csproj", "{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -149,6 +151,14 @@ Global {44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}.Release|Any CPU.Build.0 = Release|Any CPU {44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}.Release|x64.ActiveCfg = Release|Any CPU {44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}.Release|x64.Build.0 = Release|Any CPU + {160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Debug|Any CPU.Build.0 = Debug|Any CPU + {160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Debug|x64.ActiveCfg = Debug|Any CPU + {160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Debug|x64.Build.0 = Debug|Any CPU + {160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|Any CPU.ActiveCfg = Release|Any CPU + {160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|Any CPU.Build.0 = Release|Any CPU + {160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|x64.ActiveCfg = Release|Any CPU + {160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE