Adding tests for the build server, changed the Queue Service over to SQL

Wabbajack.App.Test/BasicUITests.cs

@@ -7,7 +7,7 @@ namespace Wabbajack.App.Test
 {
     public class BasicUITests
     {
-        /*
+        
         [StaFact]
         public async Task CanCompileASimpleModlist()
         {
@@ -20,6 +20,6 @@ namespace Wabbajack.App.Test
             window.Close();
             Assert.True(true);
         } 
-        */
+        
     }
 }

Wabbajack.BuildServer.Test/ABuildServerTest.cs

@@ -0,0 +1,130 @@
+using System;
+using System.Collections.Generic;
+using System.Data.SqlClient;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using Wabbajack.BuildServer.Model.Models;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Wabbajack.BuildServer.Test
+{
+    public class ABuildServerTest : IAsyncLifetime
+    {
+        private static string CONN_STR = @"Data Source=.\SQLEXPRESS;Integrated Security=True;";
+        private AppSettings _appSettings;
+        protected SqlService _sqlService;
+        private string DBName { get; }
+
+        public ABuildServerTest(ITestOutputHelper helper)
+        {
+            TestContext = helper;
+            DBName = "test_db" + Guid.NewGuid().ToString().Replace("-", "_");
+            _appSettings = MakeAppSettings();
+            _sqlService = new SqlService(_appSettings);
+        }
+
+        private AppSettings MakeAppSettings()
+        {
+            return new AppSettings
+            {
+                SqlConnection = CONN_STR + $"Initial Catalog={DBName}"
+            };
+        }
+
+        public ITestOutputHelper TestContext { get;}
+
+        public async Task InitializeAsync()
+        {
+            await CreateSchema();
+        }
+        
+        private async Task CreateSchema()
+        {
+            TestContext.WriteLine("Creating Database");
+            //var conn = new SqlConnection("Data Source=localhost,1433;User ID=test;Password=test;MultipleActiveResultSets=true");
+            await using var conn = new SqlConnection(CONN_STR);
+
+            await conn.OpenAsync();
+            //await new SqlCommand($"CREATE DATABASE {DBName};", conn).ExecuteNonQueryAsync();
+
+            await using var schemaStream = Assembly.GetExecutingAssembly().GetManifestResourceStream("Wabbajack.BuildServer.Test.sql.wabbajack_db.sql");
+            await using var ms = new MemoryStream();
+            await schemaStream.CopyToAsync(ms);
+            var schemaString = Encoding.UTF8.GetString(ms.ToArray()).Replace("wabbajack_prod", $"{DBName}");
+            
+            foreach (var statement in SplitSqlStatements(schemaString))
+            {
+                await new SqlCommand(statement, conn).ExecuteNonQueryAsync();
+                
+            }
+        }
+
+        private static IEnumerable<string> SplitSqlStatements(string sqlScript)
+        {
+            // Split by "GO" statements
+            var statements = Regex.Split(
+                sqlScript,
+                @"^[\t \r\n]*GO[\t \r\n]*\d*[\t ]*(?:--.*)?$",
+                RegexOptions.Multiline |
+                RegexOptions.IgnorePatternWhitespace |
+                RegexOptions.IgnoreCase);
+
+            // Remove empties, trim, and return
+            return statements
+                .Where(x => !string.IsNullOrWhiteSpace(x))
+                .Select(x => x.Trim(' ', '\r', '\n'));
+        }
+
+        async Task IAsyncLifetime.DisposeAsync()
+        {
+            TestContext.WriteLine("Deleting Database");
+            await using  var conn = new SqlConnection(CONN_STR);
+
+            await conn.OpenAsync();
+            await KillAll(conn);
+            await new SqlCommand($"DROP DATABASE {DBName};", conn).ExecuteNonQueryAsync();
+        }
+
+        private async Task KillAll(SqlConnection conn)
+        {
+            await new SqlCommand($@"
+                        DECLARE	@Spid INT
+                        DECLARE	@ExecSQL VARCHAR(255)
+                         
+                        DECLARE	KillCursor CURSOR LOCAL STATIC READ_ONLY FORWARD_ONLY
+                        FOR
+                        SELECT	DISTINCT SPID
+                        FROM	MASTER..SysProcesses
+                        WHERE	DBID = DB_ID('{DBName}')
+                         
+                        OPEN	KillCursor
+                         
+                        -- Grab the first SPID
+                        FETCH	NEXT
+                        FROM	KillCursor
+                        INTO	@Spid
+                         
+                        WHILE	@@FETCH_STATUS = 0
+	                        BEGIN
+		                        SET		@ExecSQL = 'KILL ' + CAST(@Spid AS VARCHAR(50))
+                         
+		                        EXEC	(@ExecSQL)
+                         
+		                        -- Pull the next SPID
+                                FETCH	NEXT 
+		                        FROM	KillCursor 
+		                        INTO	@Spid  
+	                        END
+                         
+                        CLOSE	KillCursor
+                         
+                        DEALLOCATE	KillCursor", conn).ExecuteNonQueryAsync();
+        }
+
+    }
+}

Wabbajack.BuildServer.Test/JobQueueTests.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Wabbajack.BuildServer.Model.Models;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.BuildServer.Models.Jobs;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Wabbajack.BuildServer.Test
+{
+    public class BasicTest : ABuildServerTest
+    {
+        [Fact]
+        public async Task CanEneuqueAndGetJobs()
+        {
+            var job = new Job {Payload = new GetNexusUpdatesJob()};
+            await _sqlService.EnqueueJob(job);
+            var found = await _sqlService.GetJob();
+            Assert.NotNull(found);
+            Assert.IsAssignableFrom<GetNexusUpdatesJob>(found.Payload);
+        }
+        
+        [Fact]
+        public async Task PriorityMatters()
+        {
+            var priority = new List<Job.JobPriority>
+            {
+                Job.JobPriority.Normal, Job.JobPriority.High, Job.JobPriority.Low
+            };
+            foreach (var pri in priority) 
+                await _sqlService.EnqueueJob(new Job {Payload = new GetNexusUpdatesJob(), Priority = pri});
+
+            foreach (var pri in priority.OrderByDescending(p => (int)p))
+            {
+                var found = await _sqlService.GetJob();
+                Assert.NotNull(found);
+                Assert.Equal(pri, found.Priority);
+            }
+        }
+
+        public BasicTest(ITestOutputHelper helper) : base(helper)
+        {
+            
+            
+        }
+    }
+}

Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>netcoreapp3.1</TargetFramework>
+
+        <IsPackable>false</IsPackable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />
+        <PackageReference Include="xunit" Version="2.4.0" />
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.0" />
+        <PackageReference Include="coverlet.collector" Version="1.0.1" />
+        <PackageReference Include="System.Data.SqlClient" Version="4.8.1" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <EmbeddedResource Include="sql\wabbajack_db.sql" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Wabbajack.BuildServer\Wabbajack.BuildServer.csproj" />
+    </ItemGroup>
+
+</Project>

Wabbajack.BuildServer.Test/sql/wabbajack_db.sql

@@ -0,0 +1,406 @@
+USE [master]
+GO
+/****** Object:  Database [wabbajack_prod]    Script Date: 3/28/2020 4:58:58 PM ******/
+CREATE DATABASE [wabbajack_prod]
+    CONTAINMENT = NONE
+    WITH CATALOG_COLLATION = DATABASE_DEFAULT
+GO
+ALTER DATABASE [wabbajack_prod] SET COMPATIBILITY_LEVEL = 150
+GO
+IF (1 = FULLTEXTSERVICEPROPERTY('IsFullTextInstalled'))
+    begin
+        EXEC [wabbajack_prod].[dbo].[sp_fulltext_database] @action = 'enable'
+    end
+GO
+ALTER DATABASE [wabbajack_prod] SET ANSI_NULL_DEFAULT OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET ANSI_NULLS OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET ANSI_PADDING OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET ANSI_WARNINGS OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET ARITHABORT OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET AUTO_CLOSE OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET AUTO_SHRINK OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET AUTO_UPDATE_STATISTICS ON
+GO
+ALTER DATABASE [wabbajack_prod] SET CURSOR_CLOSE_ON_COMMIT OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET CURSOR_DEFAULT  GLOBAL
+GO
+ALTER DATABASE [wabbajack_prod] SET CONCAT_NULL_YIELDS_NULL OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET NUMERIC_ROUNDABORT OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET QUOTED_IDENTIFIER OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET RECURSIVE_TRIGGERS OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET  DISABLE_BROKER
+GO
+ALTER DATABASE [wabbajack_prod] SET AUTO_UPDATE_STATISTICS_ASYNC OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET DATE_CORRELATION_OPTIMIZATION OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET TRUSTWORTHY OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET ALLOW_SNAPSHOT_ISOLATION OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET PARAMETERIZATION SIMPLE
+GO
+ALTER DATABASE [wabbajack_prod] SET READ_COMMITTED_SNAPSHOT OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET HONOR_BROKER_PRIORITY OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET RECOVERY FULL
+GO
+ALTER DATABASE [wabbajack_prod] SET  MULTI_USER
+GO
+ALTER DATABASE [wabbajack_prod] SET PAGE_VERIFY CHECKSUM
+GO
+ALTER DATABASE [wabbajack_prod] SET DB_CHAINING OFF
+GO
+ALTER DATABASE [wabbajack_prod] SET FILESTREAM( NON_TRANSACTED_ACCESS = OFF )
+GO
+ALTER DATABASE [wabbajack_prod] SET TARGET_RECOVERY_TIME = 60 SECONDS
+GO
+ALTER DATABASE [wabbajack_prod] SET DELAYED_DURABILITY = DISABLED
+GO
+EXEC sys.sp_db_vardecimal_storage_format N'wabbajack_prod', N'ON'
+GO
+ALTER DATABASE [wabbajack_prod] SET QUERY_STORE = OFF
+GO
+USE [wabbajack_prod]
+GO
+/****** Object:  Schema [test]    Script Date: 3/28/2020 4:58:59 PM ******/
+CREATE SCHEMA [test]
+GO
+/****** Object:  UserDefinedTableType [dbo].[ArchiveContentType]    Script Date: 3/28/2020 4:58:59 PM ******/
+CREATE TYPE [dbo].[ArchiveContentType] AS TABLE(
+                                                   [Parent] [bigint] NOT NULL,
+                                                   [Child] [bigint] NOT NULL,
+                                                   [Path] [nvarchar](max) NOT NULL
+                                               )
+GO
+/****** Object:  UserDefinedTableType [dbo].[IndexedFileType]    Script Date: 3/28/2020 4:58:59 PM ******/
+CREATE TYPE [dbo].[IndexedFileType] AS TABLE(
+                                                [Hash] [bigint] NOT NULL,
+                                                [Sha256] [binary](32) NOT NULL,
+                                                [Sha1] [binary](20) NOT NULL,
+                                                [Md5] [binary](16) NOT NULL,
+                                                [Crc32] [int] NOT NULL,
+                                                [Size] [bigint] NOT NULL
+                                            )
+GO
+/****** Object:  UserDefinedFunction [dbo].[Base64ToLong]    Script Date: 3/28/2020 4:58:59 PM ******/
+SET ANSI_NULLS ON
+GO
+SET QUOTED_IDENTIFIER ON
+GO
+-- =============================================
+-- Author:		<Author,,Name>
+-- Create date: <Create Date, ,>
+-- Description:	<Description, ,>
+-- =============================================
+CREATE FUNCTION [dbo].[Base64ToLong]
+(
+    -- Add the parameters for the function here
+    @Input varchar
+)
+    RETURNS bigint
+AS
+BEGIN
+    -- Declare the return variable here
+    DECLARE @ResultVar bigint
+
+    -- Add the T-SQL statements to compute the return value here
+    SELECT @ResultVar = CAST('string' as varbinary(max)) FOR XML PATH(''), BINARY BASE64
+
+    -- Return the result of the function
+    RETURN @ResultVar
+
+END
+GO
+/****** Object:  UserDefinedFunction [dbo].[MaxMetricDate]    Script Date: 3/28/2020 4:58:59 PM ******/
+SET ANSI_NULLS ON
+GO
+SET QUOTED_IDENTIFIER ON
+GO
+
+-- =============================================
+-- Author:		<Author,,Name>
+-- Create date: <Create Date, ,>
+-- Description:	<Description, ,>
+-- =============================================
+CREATE FUNCTION [dbo].[MaxMetricDate]
+(
+)
+    RETURNS date
+AS
+BEGIN
+    -- Declare the return variable here
+    DECLARE @Result date
+
+    -- Add the T-SQL statements to compute the return value here
+    SELECT @Result = max(Timestamp) from dbo.Metrics where MetricsKey is not null
+
+    -- Return the result of the function
+    RETURN @Result
+
+END
+GO
+/****** Object:  UserDefinedFunction [dbo].[MinMetricDate]    Script Date: 3/28/2020 4:58:59 PM ******/
+SET ANSI_NULLS ON
+GO
+SET QUOTED_IDENTIFIER ON
+GO
+
+-- =============================================
+-- Author:		<Author,,Name>
+-- Create date: <Create Date, ,>
+-- Description:	<Description, ,>
+-- =============================================
+CREATE FUNCTION [dbo].[MinMetricDate]
+(
+)
+    RETURNS date
+AS
+BEGIN
+    -- Declare the return variable here
+    DECLARE @Result date
+
+    -- Add the T-SQL statements to compute the return value here
+    SELECT @Result = min(Timestamp) from dbo.Metrics WHERE MetricsKey is not null
+
+    -- Return the result of the function
+    RETURN @Result
+
+END
+GO
+/****** Object:  Table [dbo].[IndexedFile]    Script Date: 3/28/2020 4:58:59 PM ******/
+SET ANSI_NULLS ON
+GO
+SET QUOTED_IDENTIFIER ON
+GO
+CREATE TABLE [dbo].[IndexedFile](
+                                    [Hash] [bigint] NOT NULL,
+                                    [Sha256] [binary](32) NOT NULL,
+                                    [Sha1] [binary](20) NOT NULL,
+                                    [Md5] [binary](16) NOT NULL,
+                                    [Crc32] [int] NOT NULL,
+                                    [Size] [bigint] NOT NULL,
+                                    CONSTRAINT [PK_IndexedFile] PRIMARY KEY CLUSTERED
+                                        (
+                                         [Hash] ASC
+                                            )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+) ON [PRIMARY]
+GO
+
+/****** Object:  Table [dbo].[Jobs] ******/
+CREATE TABLE [dbo].[Jobs](
+                             [Id] [bigint] IDENTITY(1,1) NOT NULL,
+                             [Priority] [int] NOT NULL,
+                             [Started] [datetime] NULL,
+                             [Ended] [datetime] NULL,
+                             [Created] [datetime] NOT NULL,
+                             [Success] [tinyint] NULL,
+                             [ResultContent] [nvarchar](max) NULL,
+                             [Payload] [nvarchar](max) NULL,
+                             [OnSuccess] [nvarchar](max) NULL,
+                             [RunBy] [uniqueidentifier] NULL
+) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
+GO
+
+
+
+/****** Object:  Table [dbo].[ArchiveContent]    Script Date: 3/28/2020 4:58:59 PM ******/
+SET ANSI_NULLS ON
+GO
+SET QUOTED_IDENTIFIER ON
+GO
+CREATE TABLE [dbo].[ArchiveContent](
+                                       [Parent] [bigint] NOT NULL,
+                                       [Child] [bigint] NOT NULL,
+                                       [Path] [nvarchar](max) NULL,
+                                       [PathHash]  AS (CONVERT([binary](32),hashbytes('SHA2_256',[Path]))) PERSISTED NOT NULL
+) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
+GO
+/****** Object:  Table [dbo].[AllFilesInArchive]    Script Date: 3/28/2020 4:58:59 PM ******/
+SET ANSI_NULLS ON
+GO
+SET QUOTED_IDENTIFIER ON
+GO
+CREATE TABLE [dbo].[AllFilesInArchive](
+                                          [TopParent] [bigint] NOT NULL,
+                                          [Child] [bigint] NOT NULL,
+                                          CONSTRAINT [PK_AllFilesInArchive] PRIMARY KEY CLUSTERED
+                                              (
+                                               [TopParent] ASC,
+                                               [Child] ASC
+                                                  )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+) ON [PRIMARY]
+GO
+/****** Object:  View [dbo].[AllArchiveContent]    Script Date: 3/28/2020 4:58:59 PM ******/
+SET ANSI_NULLS ON
+GO
+SET QUOTED_IDENTIFIER ON
+GO
+
+
+CREATE VIEW [dbo].[AllArchiveContent]
+            WITH SCHEMABINDING
+AS
+SELECT af.TopParent, ac.Parent, af.Child, ac.Path, idx.Size
+FROM
+    dbo.AllFilesInArchive af
+        LEFT JOIN dbo.ArchiveContent ac on af.Child = ac.Child
+        LEFT JOIN dbo.IndexedFile idx on af.Child = idx.Hash
+GO
+/****** Object:  Table [dbo].[Metrics]    Script Date: 3/28/2020 4:58:59 PM ******/
+SET ANSI_NULLS ON
+GO
+SET QUOTED_IDENTIFIER ON
+GO
+CREATE TABLE [dbo].[Metrics](
+                                [Id] [bigint] IDENTITY(1,1) NOT NULL,
+                                [Timestamp] [datetime] NOT NULL,
+                                [Action] [varchar](64) NOT NULL,
+                                [Subject] [varchar](max) NOT NULL,
+                                [MetricsKey] [varchar](64) NULL,
+                                [GroupingSubject]  AS (substring([Subject],(0),case when patindex('%[0-9].%',[Subject])=(0) then len([Subject])+(1) else patindex('%[0-9].%',[Subject]) end)),
+                                CONSTRAINT [PK_Metrics] PRIMARY KEY CLUSTERED
+                                    (
+                                     [Id] ASC
+                                        )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
+GO
+/****** Object:  Index [IX_Child]    Script Date: 3/28/2020 4:58:59 PM ******/
+CREATE NONCLUSTERED INDEX [IX_Child] ON [dbo].[AllFilesInArchive]
+    (
+     [Child] ASC
+        )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+GO
+/****** Object:  Index [IX_ArchiveContent_Child]    Script Date: 3/28/2020 4:58:59 PM ******/
+CREATE NONCLUSTERED INDEX [IX_ArchiveContent_Child] ON [dbo].[ArchiveContent]
+    (
+     [Child] ASC
+        )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+GO
+SET ARITHABORT ON
+SET CONCAT_NULL_YIELDS_NULL ON
+SET QUOTED_IDENTIFIER ON
+SET ANSI_NULLS ON
+SET ANSI_PADDING ON
+SET ANSI_WARNINGS ON
+SET NUMERIC_ROUNDABORT OFF
+GO
+/****** Object:  Index [PK_ArchiveContent]    Script Date: 3/28/2020 4:58:59 PM ******/
+CREATE UNIQUE NONCLUSTERED INDEX [PK_ArchiveContent] ON [dbo].[ArchiveContent]
+    (
+     [Parent] ASC,
+     [PathHash] ASC
+        )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+GO
+SET ANSI_PADDING ON
+GO
+/****** Object:  Index [IX_IndexedFile_By_SHA256]    Script Date: 3/28/2020 4:58:59 PM ******/
+CREATE UNIQUE NONCLUSTERED INDEX [IX_IndexedFile_By_SHA256] ON [dbo].[IndexedFile]
+    (
+     [Sha256] ASC
+        )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+GO
+/****** Object:  StoredProcedure [dbo].[MergeAllFilesInArchive]    Script Date: 3/28/2020 4:58:59 PM ******/
+SET ANSI_NULLS ON
+GO
+SET QUOTED_IDENTIFIER ON
+GO
+-- =============================================
+-- Author:		<Author,,Name>
+-- Create date: <Create Date,,>
+-- Description:	<Description,,>
+-- =============================================
+CREATE PROCEDURE [dbo].[MergeAllFilesInArchive]
+AS
+BEGIN
+    -- SET NOCOUNT ON added to prevent extra result sets from
+    -- interfering with SELECT statements.
+    SET NOCOUNT ON;
+
+    MERGE dbo.AllFilesInArchive t USING (
+        SELECT DISTINCT TopParent, unpvt.Child
+        FROM
+            (SELECT a3.Parent AS P3, a2.Parent as P2, a1.Parent P1, a0.Parent P0, a0.Parent as Parent, a0.Child FROM
+                dbo.ArChiveContent a0
+                    LEFT JOIN dbo.ArChiveContent a1 ON a0.Parent = a1.Child
+                    LEFT JOIN dbo.ArChiveContent a2 ON a1.Parent = a2.Child
+                    LEFT JOIN dbo.ArChiveContent a3 ON a2.Parent = a3.Child) p
+                UNPIVOT
+                (TopParent For C IN (p.P3, p.P2, p.P1, p.P0)) as unpvt
+                LEFT JOIN dbo.IndexedFile idf on unpvt.Child = idf.Hash
+        WHERE TopParent is not null) s
+    ON t.TopParent = s.TopParent AND t.Child = s.Child
+    WHEN NOT MATCHED
+        THEN INSERT (TopParent, Child) VALUES (s.TopParent, s.Child);
+END
+GO
+/****** Object:  StoredProcedure [dbo].[MergeIndexedFiles]    Script Date: 3/28/2020 4:58:59 PM ******/
+SET ANSI_NULLS ON
+GO
+SET QUOTED_IDENTIFIER ON
+GO
+-- Description:	<Description,,>
+-- =============================================
+CREATE PROCEDURE [dbo].[MergeIndexedFiles]
+    -- Add the parameters for the stored procedure here
+    @Files dbo.IndexedFileType READONLY,
+    @Contents dbo.ArchiveContentType READONLY
+AS
+BEGIN
+    -- SET NOCOUNT ON added to prevent extra result sets from
+    -- interfering with SELECT statements.
+    SET NOCOUNT ON;
+    BEGIN TRANSACTION;
+
+    MERGE dbo.IndexedFile AS TARGET
+    USING (SELECT DISTINCT * FROM @Files) as SOURCE
+    ON (TARGET.Hash = SOURCE.HASH)
+    WHEN NOT MATCHED BY TARGET
+        THEN INSERT (Hash, Sha256, Sha1, Md5, Crc32, Size)
+             VALUES (Source.Hash, Source.Sha256, Source.Sha1, Source.Md5, Source.Crc32, Source.Size);
+
+    MERGE dbo.ArchiveContent AS TARGET
+    USING (SELECT DISTINCT * FROM @Contents) as SOURCE
+    ON (TARGET.Parent = SOURCE.Parent AND TARGET.PathHash = CAST(HASHBYTES('SHA2_256', SOURCE.Path) as binary(32)))
+    WHEN NOT MATCHED
+        THEN INSERT (Parent, Child, Path)
+             VALUES (Source.Parent, Source.Child, Source.Path);
+
+    MERGE dbo.AllFilesInArchive t USING (
+        SELECT DISTINCT TopParent, unpvt.Child
+        FROM
+            (SELECT a3.Parent AS P3, a2.Parent as P2, a1.Parent P1, a0.Parent P0, a0.Parent as Parent, a0.Child FROM
+                dbo.ArChiveContent a0
+                    LEFT JOIN dbo.ArChiveContent a1 ON a0.Parent = a1.Child
+                    LEFT JOIN dbo.ArChiveContent a2 ON a1.Parent = a2.Child
+                    LEFT JOIN dbo.ArChiveContent a3 ON a2.Parent = a3.Child) p
+                UNPIVOT
+                (TopParent For C IN (p.P3, p.P2, p.P1, p.P0)) as unpvt
+                LEFT JOIN dbo.IndexedFile idf on unpvt.Child = idf.Hash
+        WHERE TopParent is not null
+          AND Child in (SELECT DISTINCT Hash FROM @Files)) s
+    ON t.TopParent = s.TopParent AND t.Child = s.Child
+    WHEN NOT MATCHED
+        THEN INSERT (TopParent, Child) VALUES (s.TopParent, s.Child);
+
+    COMMIT;
+
+END
+GO
+USE [master]
+GO
+ALTER DATABASE [wabbajack_prod] SET  READ_WRITE
+GO

Wabbajack.BuildServer/AppSettings.cs

@@ -9,6 +9,11 @@ namespace Wabbajack.BuildServer
         {
             config.Bind("WabbajackSettings", this);
         }
+
+        public AppSettings()
+        {
+            
+        }
         
         public AbsolutePath DownloadDir { get; set; }
         public AbsolutePath ArchiveDir { get; set; }

Wabbajack.BuildServer/Controllers/Jobs.cs

@@ -33,7 +33,7 @@ namespace Wabbajack.BuildServer.Controllers
 
         [HttpGet]
         [Route("enqueue_job/{JobName}")]
-        public async Task<string> EnqueueJob(string JobName)
+        public async Task<long> EnqueueJob(string JobName)
         {
             var jobtype = AJobPayload.NameToType[JobName];
             var job = new Job{Priority = Job.JobPriority.High, Payload = (AJobPayload)jobtype.GetConstructor(new Type[0]).Invoke(new object?[0])};

Wabbajack.BuildServer/GraphQL/Query.cs

@@ -49,10 +49,10 @@ namespace Wabbajack.BuildServer.GraphQL
             
             FieldAsync<ListGraphType<JobType>>("job",
                 arguments: new QueryArguments(
-                    new QueryArgument<IdGraphType> {Name = "id", Description = "Id of the Job"}),
+                    new QueryArgument<IntGraphType> {Name = "id", Description = "Id of the Job"}),
                 resolve: async context =>
                 {
-                    var id = context.GetArgument<string>("id");
+                    var id = context.GetArgument<long>("id");
                     var data = await db.Jobs.AsQueryable().Where(j => j.Id == id).ToListAsync();
                     return data;
                 });

Wabbajack.BuildServer/Models/JobQueue/Job.cs

@@ -19,24 +19,17 @@ namespace Wabbajack.BuildServer.Models.JobQueue
             High,
         }
 
-        [BsonId] public String Id { get; set; } = Guid.NewGuid().ToString();
+        public long Id { get; set; }
         public DateTime? Started { get; set; }
         public DateTime? Ended { get; set; }
         public DateTime Created { get; set; } = DateTime.Now;
         public JobPriority Priority { get; set; } = JobPriority.Normal;
-
         public JobResult Result { get; set; }
         public bool RequiresNexus { get; set; } = true;
         public AJobPayload Payload { get; set; }
         
         public Job OnSuccess { get; set; }
 
-        public static async Task<String> Enqueue(DBContext db, Job job)
-        {
-            await db.Jobs.InsertOneAsync(job);
-            return job.Id;
-        }
-
         public static async Task<Job> GetNext(DBContext db)
         {
             var filter = new BsonDocument

Wabbajack.BuildServer/Models/Sql/SqlService.cs

@@ -9,6 +9,7 @@ using Dapper;
 using Microsoft.Extensions.Configuration;
 using Wabbajack.BuildServer.Model.Models.Results;
 using Wabbajack.BuildServer.Models;
+using Wabbajack.BuildServer.Models.JobQueue;
 using Wabbajack.Common;
 using Wabbajack.VirtualFileSystem;
 
@@ -164,6 +165,87 @@ namespace Wabbajack.BuildServer.Model.Models
                         ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action}))
                 .ToList();
         }
+        
+        
+        #region JobRoutines
 
+        /// <summary>
+        /// Enqueue a Job into the Job queue to be run at a later time
+        /// </summary>
+        /// <param name="job"></param>
+        /// <returns></returns>
+        public async Task EnqueueJob(Job job)
+        {
+            await using var conn = await Open();
+            await conn.ExecuteAsync(
+                @"INSERT INTO dbo.Jobs (Created, Priority, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @Payload, @OnSuccess)",
+                new {
+                    Priority = job.Priority,
+                    Payload = job.Payload.ToJSON(), 
+                    OnSuccess = job.OnSuccess?.ToJSON() ?? null});
+        }
+        
+        /// <summary>
+        /// Enqueue a Job into the Job queue to be run at a later time
+        /// </summary>
+        /// <param name="job"></param>
+        /// <returns></returns>
+        public async Task FinishJob(Job job)
+        {
+            await using var conn = await Open();
+            await conn.ExecuteAsync(
+                @"UPDATE dbo.Jobs SET Finshed = GETDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id",
+                new { 
+                    Id = job.Id,
+                    Success = job.Result.ResultType == JobResultType.Success,
+                    ResultPayload = job.Result.ToJSON()
+                    
+                });
+            
+            if (job.OnSuccess != null)
+                await EnqueueJob(job.OnSuccess);
+        }
+
+        
+        /// <summary>
+        /// Get a Job from the Job queue to run. 
+        /// </summary>
+        /// <returns></returns>
+        public async Task<Job> GetJob()
+        {
+            await using var conn = await Open();
+            var result = await conn.QueryAsync<Job>(
+                @"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy WHERE ID in (SELECT TOP(1) ID FROM Jobs WHERE Started is NULL ORDER BY Priority DESC, Created);
+                      SELECT TOP(1) * FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC",
+                new {RunBy = Guid.NewGuid().ToString()});
+            return result.FirstOrDefault();
+        }
+
+        
+        #endregion
+
+
+        #region TypeMappers
+
+        static SqlService()
+        {
+            SqlMapper.AddTypeHandler(new PayloadMapper());
+        }
+
+        public class PayloadMapper : SqlMapper.TypeHandler<AJobPayload>
+        {
+            public override void SetValue(IDbDataParameter parameter, AJobPayload value)
+            {
+                parameter.Value = value.ToJSON();
+            }
+
+            public override AJobPayload Parse(object value)
+            {
+                return Utils.FromJSONString<AJobPayload>((string)value);
+            }
+        }
+        
+
+        #endregion
     }
 }

Wabbajack.BuildServer/Wabbajack.BuildServer.csproj

@@ -80,6 +80,7 @@
 
     <ItemGroup>
       <EmbeddedResource Remove="swiftshader\**" />
+      <None Remove="sql\wabbajack_db.sql" />
     </ItemGroup>
 
     <ItemGroup>

Wabbajack.sln

@@ -42,6 +42,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Common.Test", "Wa
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.App.Test", "Wabbajack.App.Test\Wabbajack.App.Test.csproj", "{44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.BuildServer.Test", "Wabbajack.BuildServer.Test\Wabbajack.BuildServer.Test.csproj", "{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -149,6 +151,14 @@ Global
 		{44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}.Release|Any CPU.Build.0 = Release|Any CPU
 		{44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}.Release|x64.ActiveCfg = Release|Any CPU
 		{44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}.Release|x64.Build.0 = Release|Any CPU
+		{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Debug|x64.Build.0 = Debug|Any CPU
+		{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|Any CPU.Build.0 = Release|Any CPU
+		{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|x64.ActiveCfg = Release|Any CPU
+		{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE