diff --git a/Wabbajack.Common/CircuitBreaker/WithAutoRetry.cs b/Wabbajack.Common/CircuitBreaker/WithAutoRetry.cs index a6c5587e..d99aa90b 100644 --- a/Wabbajack.Common/CircuitBreaker/WithAutoRetry.cs +++ b/Wabbajack.Common/CircuitBreaker/WithAutoRetry.cs @@ -58,6 +58,54 @@ namespace Wabbajack.Common } } + + public static async ValueTask WithAutoRetryAllAsync(Func f, TimeSpan? delay = null, int? multipler = null, int? maxRetries = null) + { + int retries = 0; + delay ??= DEFAULT_DELAY; + multipler ??= DEFAULT_DELAY_MULTIPLIER; + maxRetries ??= DEFAULT_RETRIES; + + TOP: + try + { + await f(); + } + catch (Exception ex) + { + retries += 1; + if (retries > maxRetries) + throw; + Utils.Log($"(Retry {retries} of {maxRetries}), got exception {ex.Message}, waiting {delay!.Value.TotalMilliseconds}ms"); + await Task.Delay(delay.Value); + delay = delay * multipler; + goto TOP; + } + } + public static async ValueTask WithAutoRetryAllAsync(Func> f, TimeSpan? delay = null, int? multipler = null, int? maxRetries = null) + { + int retries = 0; + delay ??= DEFAULT_DELAY; + multipler ??= DEFAULT_DELAY_MULTIPLIER; + maxRetries ??= DEFAULT_RETRIES; + + TOP: + try + { + return await f(); + } + catch (Exception ex) + { + retries += 1; + if (retries > maxRetries) + throw; + Utils.Log($"(Retry {retries} of {maxRetries}), got exception {ex.Message}, waiting {delay!.Value.TotalMilliseconds}ms"); + await Task.Delay(delay.Value); + delay = delay * multipler; + goto TOP; + } + } + public static void WithAutoRetry(Action f, TimeSpan? delay = null, int? multipler = null, int? maxRetries = null) where TE : Exception { int retries = 0; diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 3fcac193..62e4a975 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -159,8 +159,6 @@ namespace Wabbajack.Common public static Uri WabbajackOrg = new Uri("https://www.wabbajack.org/"); public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2; - - public static Uri WabbajackMirror = new Uri("https://wabbajack-mirror.b-cdn.net"); } } diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index e6b91b05..2c02acf3 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -125,7 +125,7 @@ namespace Wabbajack.Lib.Downloaders Utils.Log($"Trying to find solution to broken download for {archive.Name}"); var result = await FindUpgrade(archive); - if (result == default) + if (result == default ) { result = await AbstractDownloadState.ServerFindUpgrade(archive); if (result == default) diff --git a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs index 082fafef..e0773b3a 100644 --- a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs +++ b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs @@ -26,7 +26,7 @@ namespace Wabbajack.Lib.Downloaders {"wabbajacktest.b-cdn.net", "test-files.wabbajack.org"} }; - + public string[]? Mirrors; public long TotalRetries; @@ -70,6 +70,11 @@ namespace Wabbajack.Lib.Downloaders return true; } + public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func> downloadResolver) + { + return default; + } + public override async Task Download(Archive a, AbsolutePath destination) { destination.Parent.CreateDirectory(); diff --git a/Wabbajack.Server.Test/ABuildServerSystemTest.cs b/Wabbajack.Server.Test/ABuildServerSystemTest.cs index e1989af9..44e8f014 100644 --- a/Wabbajack.Server.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.Server.Test/ABuildServerSystemTest.cs @@ -62,7 +62,7 @@ namespace Wabbajack.BuildServer.Test _token = new CancellationTokenSource(); _task = _host.RunAsync(_token.Token); Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080"); - Consts.WabbajackMirror = new Uri("https://wabbajack-test.b-cdn.net"); + Consts.TestMode = true; await "ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllTextAsync( "GoogleIDs:\nAllowedPrefixes:\n - http://localhost"); diff --git a/Wabbajack.Server.Test/AuthoredFilesTests.cs b/Wabbajack.Server.Test/AuthoredFilesTests.cs index 3f4eb15f..4c6514f2 100644 --- a/Wabbajack.Server.Test/AuthoredFilesTests.cs +++ b/Wabbajack.Server.Test/AuthoredFilesTests.cs @@ -83,26 +83,5 @@ namespace Wabbajack.BuildServer.Test Assert.Empty(toDelete.SQLDelete); } - - [Fact] - public async Task ServerGetsEdgeServerInfo() - { - var service = Fixture.GetService(); - Assert.True(await service.Execute() > 0); - Assert.NotEmpty(service.Mirrors); - Assert.True(DateTime.UtcNow - service.LastUpdate < TimeSpan.FromMinutes(1)); - - var servers = await ClientAPI.GetCDNMirrorList(); - Assert.Equal(service.Mirrors, servers); - - var state = new WabbajackCDNDownloader.State(new Uri("https://wabbajack.b-cdn.net/this_file_doesn_t_exist")); - await DownloadDispatcher.PrepareAll(new[] {state}); - await using var tmp = new TempFile(); - - await Assert.ThrowsAsync(async () => await state.Download(new Archive(state) {Name = "test"}, tmp.Path)); - var downloader = DownloadDispatcher.GetInstance(); - Assert.Null(downloader.Mirrors); // Now works through a host remap - } - } } diff --git a/Wabbajack.Server.Test/MirroredFilesTests.cs b/Wabbajack.Server.Test/MirroredFilesTests.cs index e6986776..ace914fa 100644 --- a/Wabbajack.Server.Test/MirroredFilesTests.cs +++ b/Wabbajack.Server.Test/MirroredFilesTests.cs @@ -39,6 +39,7 @@ namespace Wabbajack.Server.Test }); var uploader = Fixture.GetService(); + uploader.ActiveFileSyncEnabled = false; Assert.Equal(1, await uploader.Execute()); @@ -51,6 +52,23 @@ namespace Wabbajack.Server.Test await using var file2 = new TempFile(); await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file2.Path); Assert.Equal(dataHash!.Value, await file2.Path.FileHashAsync()); + + var onServer = await uploader.GetHashesOnCDN(); + Assert.Contains(dataHash.Value, onServer); + + await uploader.DeleteOldMirrorFiles(); + + // Still in SQL so it will still exist + await using var file3 = new TempFile(); + await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file3.Path); + Assert.Equal(dataHash!.Value, await file3.Path.FileHashAsync()); + + // Enabling the sync should kill off the unattached file + uploader.ActiveFileSyncEnabled = true; + Assert.Equal(0, await uploader.Execute()); + + var onServer2 = await uploader.GetHashesOnCDN(); + Assert.DoesNotContain(dataHash.Value, onServer2); } [Fact] diff --git a/Wabbajack.Server.Test/ModListValidationTests.cs b/Wabbajack.Server.Test/ModListValidationTests.cs index c2212403..36ee449b 100644 --- a/Wabbajack.Server.Test/ModListValidationTests.cs +++ b/Wabbajack.Server.Test/ModListValidationTests.cs @@ -141,7 +141,7 @@ namespace Wabbajack.BuildServer.Test data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list"); Assert.NotNull(data); - Assert.Equal(0, data.ValidationSummary.Failed); + Assert.Equal(1, data.ValidationSummary.Failed); Assert.Equal(0, data.ValidationSummary.Passed); Assert.Equal(1, data.ValidationSummary.Updating); diff --git a/Wabbajack.Server.Test/ModlistUpdater.cs b/Wabbajack.Server.Test/ModlistUpdater.cs index d180a5d0..b8e1fbe9 100644 --- a/Wabbajack.Server.Test/ModlistUpdater.cs +++ b/Wabbajack.Server.Test/ModlistUpdater.cs @@ -76,7 +76,7 @@ namespace Wabbajack.Server.Test await Assert.ThrowsAsync(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); Assert.True(await patcher.Execute() > 1); - Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); + Assert.Equal(new Uri("https://test-files.wabbajack.org/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); Assert.Equal("Purged", await AuthorAPI.NoPatch(oldArchive.Hash, "Testing NoPatch")); diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index 12231485..bc2f2e8a 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -1,17 +1,17 @@ USE [master] GO -/****** Object: Database [wabbajack_prod] Script Date: 12/29/2020 8:55:03 PM ******/ +/****** Object: Database [wabbajack_prod] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE DATABASE [wabbajack_prod] CONTAINMENT = NONE - WITH CATALOG_COLLATION = DATABASE_DEFAULT + GO ALTER DATABASE [wabbajack_prod] SET COMPATIBILITY_LEVEL = 150 GO IF (1 = FULLTEXTSERVICEPROPERTY('IsFullTextInstalled')) begin -EXEC [wabbajack_prod].[dbo].[sp_fulltext_database] @action = 'enable' + EXEC [wabbajack_prod].[dbo].[sp_fulltext_database] @action = 'enable' end -GO + GO ALTER DATABASE [wabbajack_prod] SET ANSI_NULL_DEFAULT OFF GO ALTER DATABASE [wabbajack_prod] SET ANSI_NULLS OFF @@ -74,38 +74,34 @@ ALTER DATABASE [wabbajack_prod] SET QUERY_STORE = OFF GO USE [wabbajack_prod] GO -/****** Object: User [wabbajack] Script Date: 12/29/2020 8:55:03 PM ******/ +/****** Object: User [wabbajack] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE USER [wabbajack] WITHOUT LOGIN WITH DEFAULT_SCHEMA=[dbo] GO -ALTER ROLE [db_datareader] ADD MEMBER [wabbajack] -GO -ALTER ROLE [db_datawriter] ADD MEMBER [wabbajack] -GO -/****** Object: Schema [test] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Schema [test] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE SCHEMA [test] - GO -/****** Object: UserDefinedTableType [dbo].[ArchiveContentType] Script Date: 12/29/2020 8:55:04 PM ******/ +GO +/****** Object: UserDefinedTableType [dbo].[ArchiveContentType] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE TYPE [dbo].[ArchiveContentType] AS TABLE( - [Parent] [bigint] NOT NULL, - [Child] [bigint] NOT NULL, - [Path] [nvarchar](max) NOT NULL - ) - GO -/****** Object: UserDefinedTableType [dbo].[IndexedFileType] Script Date: 12/29/2020 8:55:04 PM ******/ + [Parent] [bigint] NOT NULL, + [Child] [bigint] NOT NULL, + [Path] [nvarchar](max) NOT NULL +) +GO +/****** Object: UserDefinedTableType [dbo].[IndexedFileType] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE TYPE [dbo].[IndexedFileType] AS TABLE( - [Hash] [bigint] NOT NULL, - [Sha256] [binary](32) NOT NULL, - [Sha1] [binary](20) NOT NULL, - [Md5] [binary](16) NOT NULL, - [Crc32] [int] NOT NULL, - [Size] [bigint] NOT NULL - ) - GO -/****** Object: UserDefinedFunction [dbo].[Base64ToLong] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO + [Hash] [bigint] NOT NULL, + [Sha256] [binary](32) NOT NULL, + [Sha1] [binary](20) NOT NULL, + [Md5] [binary](16) NOT NULL, + [Crc32] [int] NOT NULL, + [Size] [bigint] NOT NULL +) +GO +/****** Object: UserDefinedFunction [dbo].[Base64ToLong] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO -- ============================================= -- Author: -- Create date: @@ -119,22 +115,22 @@ CREATE FUNCTION [dbo].[Base64ToLong] RETURNS bigint AS BEGIN - -- Declare the return variable here - DECLARE @ResultVar bigint + -- Declare the return variable here + DECLARE @ResultVar bigint - -- Add the T-SQL statements to compute the return value here +-- Add the T-SQL statements to compute the return value here SELECT @ResultVar = CAST(@Input as varbinary(max)) FOR XML PATH(''), BINARY BASE64 - -- Return the result of the function - RETURN @ResultVar + -- Return the result of the function + RETURN @ResultVar END GO -/****** Object: UserDefinedFunction [dbo].[MaxMetricDate] Script Date: 12/29/2020 8:55:04 PM ******/ -SET ANSI_NULLS ON -GO -SET QUOTED_IDENTIFIER ON -GO +/****** Object: UserDefinedFunction [dbo].[MaxMetricDate] Script Date: 3/9/2021 11:12:53 PM ******/ + SET ANSI_NULLS ON + GO + SET QUOTED_IDENTIFIER ON + GO -- ============================================= -- Author: @@ -147,18 +143,18 @@ CREATE FUNCTION [dbo].[MaxMetricDate] RETURNS date AS BEGIN - -- Declare the return variable here - DECLARE @Result date + -- Declare the return variable here + DECLARE @Result date - -- Add the T-SQL statements to compute the return value here +-- Add the T-SQL statements to compute the return value here SELECT @Result = max(Timestamp) from dbo.Metrics where MetricsKey is not null -- Return the result of the function RETURN @Result END -GO -/****** Object: UserDefinedFunction [dbo].[MinMetricDate] Script Date: 12/29/2020 8:55:04 PM ******/ + GO +/****** Object: UserDefinedFunction [dbo].[MinMetricDate] Script Date: 3/9/2021 11:12:53 PM ******/ SET ANSI_NULLS ON GO SET QUOTED_IDENTIFIER ON @@ -175,114 +171,114 @@ CREATE FUNCTION [dbo].[MinMetricDate] RETURNS date AS BEGIN - -- Declare the return variable here - DECLARE @Result date + -- Declare the return variable here + DECLARE @Result date - -- Add the T-SQL statements to compute the return value here +-- Add the T-SQL statements to compute the return value here SELECT @Result = min(Timestamp) from dbo.Metrics WHERE MetricsKey is not null -- Return the result of the function RETURN @Result END -GO -/****** Object: Table [dbo].[ModLists] Script Date: 12/29/2020 8:55:04 PM ******/ + GO +/****** Object: Table [dbo].[ModLists] Script Date: 3/9/2021 11:12:53 PM ******/ SET ANSI_NULLS ON GO SET QUOTED_IDENTIFIER ON GO CREATE TABLE [dbo].[ModLists]( - [MachineURL] [nvarchar](50) NOT NULL, - [Hash] [bigint] NOT NULL, - [Metadata] [nvarchar](max) NOT NULL, - [Modlist] [nvarchar](max) NOT NULL, - [BrokenDownload] [tinyint] NOT NULL, - CONSTRAINT [PK_ModLists] PRIMARY KEY CLUSTERED + [MachineURL] [nvarchar](50) NOT NULL, + [Hash] [bigint] NOT NULL, + [Metadata] [nvarchar](max) NOT NULL, + [Modlist] [nvarchar](max) NOT NULL, + [BrokenDownload] [tinyint] NOT NULL, + CONSTRAINT [PK_ModLists] PRIMARY KEY CLUSTERED ( -[MachineURL] ASC + [MachineURL] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: View [dbo].[AllModListArchives] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: View [dbo].[AllModListArchives] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO CREATE VIEW [dbo].[AllModListArchives] AS SELECT ml.MachineURL, p.Name, p.Size, p.Hash, p.State FROM [ModLists] ml - CROSS APPLY - OPENJSON(ModList, '$.Archives') + CROSS APPLY + OPENJSON(ModList, '$.Archives') WITH ( - Name nvarchar(max) '$.Name', - Size nvarchar(max) '$.Size', - Hash nvarchar(max) '$.Hash', - State nvarchar(max) as json +Name nvarchar(max) '$.Name', +Size nvarchar(max) '$.Size', +Hash nvarchar(max) '$.Hash', +State nvarchar(max) as json - ) p - GO -/****** Object: Table [dbo].[IndexedFile] Script Date: 12/29/2020 8:55:04 PM ******/ +) p +GO +/****** Object: Table [dbo].[IndexedFile] Script Date: 3/9/2021 11:12:53 PM ******/ SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO +GO +SET QUOTED_IDENTIFIER ON +GO CREATE TABLE [dbo].[IndexedFile]( - [Hash] [bigint] NOT NULL, - [Sha256] [binary](32) NOT NULL, - [Sha1] [binary](20) NOT NULL, - [Md5] [binary](16) NOT NULL, - [Crc32] [int] NOT NULL, - [Size] [bigint] NOT NULL, - CONSTRAINT [PK_IndexedFile] PRIMARY KEY CLUSTERED -( -[Hash] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] - GO -/****** Object: Table [dbo].[ArchiveContent] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO + [Hash] [bigint] NOT NULL, + [Sha256] [binary](32) NOT NULL, + [Sha1] [binary](20) NOT NULL, + [Md5] [binary](16) NOT NULL, + [Crc32] [int] NOT NULL, + [Size] [bigint] NOT NULL, + CONSTRAINT [PK_IndexedFile] PRIMARY KEY CLUSTERED + ( + [Hash] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO +/****** Object: Table [dbo].[ArchiveContent] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO CREATE TABLE [dbo].[ArchiveContent]( - [Parent] [bigint] NOT NULL, - [Child] [bigint] NOT NULL, - [Path] [nvarchar](max) NULL, - [PathHash] AS (CONVERT([binary](32),hashbytes('SHA2_256',replace(lower([Path]),'/','\')))) PERSISTED NOT NULL - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Index [Child_Parent_IDX] Script Date: 12/29/2020 8:55:04 PM ******/ + [Parent] [bigint] NOT NULL, + [Child] [bigint] NOT NULL, + [Path] [nvarchar](max) NULL, + [PathHash] AS (CONVERT([binary](32),hashbytes('SHA2_256',replace(lower([Path]),'/','\')))) PERSISTED NOT NULL +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Index [Child_Parent_IDX] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE CLUSTERED INDEX [Child_Parent_IDX] ON [dbo].[ArchiveContent] ( [Child] ASC, [Parent] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] GO -/****** Object: Table [dbo].[AllFilesInArchive] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Table [dbo].[AllFilesInArchive] Script Date: 3/9/2021 11:12:53 PM ******/ SET ANSI_NULLS ON GO SET QUOTED_IDENTIFIER ON GO CREATE TABLE [dbo].[AllFilesInArchive]( - [TopParent] [bigint] NOT NULL, - [Child] [bigint] NOT NULL, - CONSTRAINT [PK_AllFilesInArchive] PRIMARY KEY CLUSTERED - ( - [TopParent] ASC, -[Child] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] - GO -/****** Object: View [dbo].[AllArchiveContent] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO + [TopParent] [bigint] NOT NULL, + [Child] [bigint] NOT NULL, + CONSTRAINT [PK_AllFilesInArchive] PRIMARY KEY CLUSTERED + ( + [TopParent] ASC, + [Child] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO +/****** Object: View [dbo].[AllArchiveContent] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO CREATE VIEW [dbo].[AllArchiveContent] - WITH SCHEMABINDING + WITH SCHEMABINDING AS SELECT af.TopParent, ac.Parent, af.Child, ac.Path, idx.Size FROM @@ -290,31 +286,31 @@ FROM LEFT JOIN dbo.ArchiveContent ac on af.Child = ac.Child LEFT JOIN dbo.IndexedFile idx on af.Child = idx.Hash GO -/****** Object: Table [dbo].[DownloadStates] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Table [dbo].[DownloadStates] Script Date: 3/9/2021 11:12:53 PM ******/ SET ANSI_NULLS ON GO SET QUOTED_IDENTIFIER ON GO CREATE TABLE [dbo].[DownloadStates]( - [Id] [binary](32) NOT NULL, - [Hash] [bigint] NOT NULL, - [PrimaryKey] [nvarchar](max) NOT NULL, - [IniState] [nvarchar](max) NOT NULL, - [JsonState] [nvarchar](max) NOT NULL, - CONSTRAINT [PK_DownloadStates] PRIMARY KEY CLUSTERED + [Id] [binary](32) NOT NULL, + [Hash] [bigint] NOT NULL, + [PrimaryKey] [nvarchar](max) NOT NULL, + [IniState] [nvarchar](max) NOT NULL, + [JsonState] [nvarchar](max) NOT NULL, + CONSTRAINT [PK_DownloadStates] PRIMARY KEY CLUSTERED ( -[Id] ASC + [Id] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: View [dbo].[GameFiles] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: View [dbo].[GameFiles] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO CREATE VIEW [dbo].[GameFiles] - WITH SCHEMABINDING + WITH SCHEMABINDING AS Select @@ -335,476 +331,594 @@ SET ARITHABORT ON SET ANSI_WARNINGS ON SET NUMERIC_ROUNDABORT OFF GO -/****** Object: Index [ByGameAndVersion] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Index [ByGameAndVersion] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE UNIQUE CLUSTERED INDEX [ByGameAndVersion] ON [dbo].[GameFiles] -( - [Game] ASC, - [GameVersion] ASC, - [Id] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] -GO -/****** Object: Table [dbo].[AuthoredFiles] Script Date: 12/29/2020 8:55:04 PM ******/ -SET ANSI_NULLS ON -GO -SET QUOTED_IDENTIFIER ON -GO -CREATE TABLE [dbo].[AuthoredFiles]( - [ServerAssignedUniqueId] [uniqueidentifier] NOT NULL, - [LastTouched] [datetime] NOT NULL, - [CDNFileDefinition] [nvarchar](max) NOT NULL, - [Finalized] [datetime] NULL, - CONSTRAINT [PK_AuthoredFiles] PRIMARY KEY CLUSTERED -( -[ServerAssignedUniqueId] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] + ( + [Game] ASC, + [GameVersion] ASC, + [Id] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] GO -/****** Object: View [dbo].[AuthoredFilesSummaries] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Table [dbo].[AuthoredFiles] Script Date: 3/9/2021 11:12:53 PM ******/ SET ANSI_NULLS ON GO SET QUOTED_IDENTIFIER ON GO +CREATE TABLE [dbo].[AuthoredFiles]( + [ServerAssignedUniqueId] [uniqueidentifier] NOT NULL, + [LastTouched] [datetime] NOT NULL, + [CDNFileDefinition] [nvarchar](max) NOT NULL, + [Finalized] [datetime] NULL, + CONSTRAINT [PK_AuthoredFiles] PRIMARY KEY CLUSTERED +( + [ServerAssignedUniqueId] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: View [dbo].[AuthoredFilesSummaries] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO + /****** Script for SelectTopNRows command from SSMS ******/ CREATE VIEW [dbo].[AuthoredFilesSummaries] AS SELECT [ServerAssignedUniqueId] - ,[LastTouched] - ,[Finalized] - ,JSON_VALUE(CDNFileDefinition, '$.OriginalFileName') as OriginalFileName - ,JSON_VALUE(CDNFileDefinition, '$.MungedName') as MungedName - ,JSON_VALUE(CDNFileDefinition, '$.Author') as Author - ,JSON_VALUE(CDNFILEDefinition, '$.Size') as Size + ,[LastTouched] + ,[Finalized] + ,JSON_VALUE(CDNFileDefinition, '$.OriginalFileName') as OriginalFileName + ,JSON_VALUE(CDNFileDefinition, '$.MungedName') as MungedName + ,JSON_VALUE(CDNFileDefinition, '$.Author') as Author + ,JSON_VALUE(CDNFILEDefinition, '$.Size') as Size + ,JSON_VALUE(CDNFILEDefinition, '$.Hash') as Hash FROM [wabbajack_prod].[dbo].[AuthoredFiles] - GO -/****** Object: Table [dbo].[AccessLog] Script Date: 12/29/2020 8:55:04 PM ******/ + GO +/****** Object: Table [dbo].[NexusModInfos] Script Date: 3/9/2021 11:12:53 PM ******/ + SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[NexusModInfos]( + [Game] [int] NOT NULL, + [ModId] [bigint] NOT NULL, + [LastChecked] [datetime] NOT NULL, + [Data] [nvarchar](max) NOT NULL, + CONSTRAINT [PK_NexusModInfos] PRIMARY KEY CLUSTERED +( + [Game] ASC, + [ModId] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[ModListArchives] Script Date: 3/9/2021 11:12:53 PM ******/ SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[AccessLog]( - [Id] [bigint] IDENTITY(1,1) NOT NULL, - [Timestamp] [datetime] NOT NULL, - [Action] [nvarchar](max) NOT NULL, - [Ip] [nvarchar](50) NOT NULL, - [MetricsKey] AS (json_value([Action],'$.Headers."x-metrics-key"[0]')), - CONSTRAINT [PK_AccessLog] PRIMARY KEY CLUSTERED +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[ModListArchives]( + [MachineUrl] [nvarchar](50) NOT NULL, + [Hash] [bigint] NOT NULL, + [PrimaryKeyString] [nvarchar](max) NOT NULL, + [Size] [bigint] NOT NULL, + [State] [nvarchar](max) NOT NULL, + [Name] [nvarchar](max) NULL, + CONSTRAINT [PK_ModListArchive] PRIMARY KEY CLUSTERED ( -[Id] ASC + [MachineUrl] ASC, + [Hash] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Table [dbo].[ApiKeys] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[ApiKeys]( - [APIKey] [nvarchar](260) NOT NULL, - [Owner] [nvarchar](40) NOT NULL, - CONSTRAINT [PK_ApiKeys] PRIMARY KEY CLUSTERED -( - [APIKey] ASC, -[Owner] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] - GO -/****** Object: Table [dbo].[ArchiveDownloads] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[ArchiveDownloads]( - [Id] [uniqueidentifier] NOT NULL, - [PrimaryKeyString] [nvarchar](255) NOT NULL, - [Size] [bigint] NULL, - [Hash] [bigint] NULL, - [IsFailed] [tinyint] NULL, - [DownloadFinished] [datetime] NULL, - [DownloadState] [nvarchar](max) NOT NULL, - [Downloader] [nvarchar](50) NOT NULL, - [FailMessage] [nvarchar](max) NULL, - CONSTRAINT [PK_ArchiveDownloads] PRIMARY KEY CLUSTERED -( -[Id] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Table [dbo].[ArchivePatches] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[ArchivePatches]( - [SrcPrimaryKeyStringHash] [binary](32) NOT NULL, - [SrcPrimaryKeyString] [nvarchar](max) NOT NULL, - [SrcHash] [bigint] NOT NULL, - [DestPrimaryKeyStringHash] [binary](32) NOT NULL, - [DestPrimaryKeyString] [nvarchar](max) NOT NULL, - [DestHash] [bigint] NOT NULL, - [SrcState] [nvarchar](max) NOT NULL, - [DestState] [nvarchar](max) NOT NULL, - [SrcDownload] [nvarchar](max) NULL, - [DestDownload] [nvarchar](max) NULL, - [CDNPath] [nvarchar](max) NULL, - CONSTRAINT [PK_ArchivePatches] PRIMARY KEY CLUSTERED -( - [SrcPrimaryKeyStringHash] ASC, - [SrcHash] ASC, - [DestPrimaryKeyStringHash] ASC, -[DestHash] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Table [dbo].[GameMetadata] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[NexusModPermissions] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[NexusModPermissions]( + [NexusGameID] [int] NOT NULL, + [ModID] [bigint] NOT NULL, + [Permissions] [int] NOT NULL, + CONSTRAINT [PK_NexusModPermissions] PRIMARY KEY CLUSTERED + ( + [NexusGameID] ASC, + [ModID] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO +/****** Object: Table [dbo].[GameMetadata] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO CREATE TABLE [dbo].[GameMetadata]( - [NexusGameId] [bigint] NULL, - [WabbajackName] [nvarchar](50) NOT NULL, - CONSTRAINT [PK_GameMetadata] PRIMARY KEY CLUSTERED + [NexusGameId] [bigint] NULL, + [WabbajackName] [nvarchar](50) NOT NULL, + CONSTRAINT [PK_GameMetadata] PRIMARY KEY CLUSTERED + ( + [WabbajackName] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO +/****** Object: Table [dbo].[ArchiveDownloads] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[ArchiveDownloads]( + [Id] [uniqueidentifier] NOT NULL, + [PrimaryKeyString] [nvarchar](255) NOT NULL, + [Size] [bigint] NULL, + [Hash] [bigint] NULL, + [IsFailed] [tinyint] NULL, + [DownloadFinished] [datetime] NULL, + [DownloadState] [nvarchar](max) NOT NULL, + [Downloader] [nvarchar](50) NOT NULL, + [FailMessage] [nvarchar](max) NULL, + CONSTRAINT [PK_ArchiveDownloads] PRIMARY KEY CLUSTERED ( -[WabbajackName] ASC + [Id] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[NexusModsWithOpenPerms] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[NexusModsWithOpenPerms]( + [NexusGameID] [bigint] NOT NULL, + [NexusModID] [bigint] NOT NULL, + CONSTRAINT [PK_NexusModsWithOpenPerms] PRIMARY KEY CLUSTERED + ( + [NexusGameID] ASC, + [NexusModID] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO +/****** Object: Table [dbo].[NexusAuthorsWithOpenPerms] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[NexusAuthorsWithOpenPerms]( + [NexusAuthor] [nvarchar](64) NULL +) ON [PRIMARY] +GO +/****** Object: View [dbo].[AllowedMirrors] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO + +CREATE VIEW [dbo].[AllowedMirrors](Hash, Date, Reason) +AS + +SELECT hs.Hash, GETUTCDATE(), 'File has re-upload permissions on the Nexus' FROM + (SELECT DISTINCT ad.Hash FROM dbo.NexusModPermissions p + INNER JOIN GameMetadata md on md.NexusGameId = p.NexusGameID + INNER JOIN dbo.ArchiveDownloads ad on ad.PrimaryKeyString like 'NexusDownloader+State|'+md.WabbajackName+'|'+CAST(p.ModID as nvarchar)+'|%' + WHERE p.Permissions = 1 + ) hs + +UNION + +SELECT hs.Hash, GETUTCDATE(), 'File has re-upload permissions on the Nexus' FROM + (SELECT DISTINCT ad.Hash FROM dbo.NexusModInfos info + INNER JOIN GameMetadata md on md.NexusGameId = info.Game + INNER JOIN dbo.ArchiveDownloads ad on ad.PrimaryKeyString like 'NexusDownloader+State|'+md.WabbajackName+'|'+CAST(info.ModID as nvarchar)+'|%' + WHERE JSON_VALUE(Data, '$.uploaded_by') in (SELECT NexusAuthor FROM dbo.NexusAuthorsWithOpenPerms) + ) hs + +UNION + +SELECT hs.Hash, GETUTCDATE(), 'File has re-upload permissions on the Nexus' FROM + (SELECT DISTINCT ad.Hash FROM dbo.NexusModInfos info + INNER JOIN GameMetadata md on md.NexusGameId = info.Game + INNER JOIN dbo.ArchiveDownloads ad on ad.PrimaryKeyString like 'NexusDownloader+State|'+md.WabbajackName+'|'+CAST(info.ModID as nvarchar)+'|%' + INNER JOIN dbo.NexusModsWithOpenPerms op on op.NexusGameID = info.Game AND op.NexusModID = info.ModId + + ) hs + +UNION + +SELECT hs.Hash, GETUTCDATE(), 'File has re-upload permissions on the Nexus' FROM + (SELECT DISTINCT ad.Hash FROM dbo.NexusModInfos info + INNER JOIN GameMetadata md on md.NexusGameId = info.Game + INNER JOIN dbo.ArchiveDownloads ad on ad.PrimaryKeyString like 'NexusDownloader+State|'+md.WabbajackName+'|'+CAST(info.ModID as nvarchar)+'|%' + WHERE JSON_VALUE(Data, '$.uploaded_by') = 'EnaiSiaion' + + ) hs + +UNION + +SELECT DISTINCT Hash, GETUTCDATE(), 'File is hosted on GitHub' +FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%github.com/%' + +UNION + +SELECT DISTINCT Hash, GETUTCDATE(), 'File license allows uploading to any Non-nexus site' +FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%enbdev.com/%' + + +UNION + +SELECT DISTINCT Hash, GETUTCDATE(), 'DynDOLOD file' /*, Name*/ +from dbo.ModListArchives mla WHERE Name like '%DynDoLOD%standalone%' + + +UNION + +SELECT DISTINCT Hash, GETUTCDATE(), 'xLODGen file' +from dbo.ModListArchives mla WHERE Name like 'xLODGen.%' and PrimaryKeyString like 'MegaDownloader+State|%' + + +UNION + +SELECT DISTINCT Hash, GETUTCDATE(), 'Distribution allowed by author' /*, Name*/ +from dbo.ModListArchives mla WHERE Name like '%particle%patch%' + +UNION + +SELECT DISTINCT Hash, GETUTCDATE(), 'Authored by the WJ team' /*, Name*/ +from dbo.ModListArchives mla WHERE PrimaryKeyString like 'WabbajackCDN%' GO -/****** Object: Table [dbo].[Jobs] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON +/****** Object: Table [dbo].[AllowedMirrorsCache] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[AllowedMirrorsCache]( + [Hash] [bigint] NOT NULL, + [Reason] [nvarchar](max) NOT NULL, + CONSTRAINT [PK_AllowedMirrorsCache] PRIMARY KEY CLUSTERED +( + [Hash] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: View [dbo].[ActiveMirrors] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO + + +CREATE VIEW [dbo].[ActiveMirrors] (Hash, Reason, Date) +AS +SELECT DISTINCT am.Hash, am.Reason, GETUTCDATE() from dbo.AllowedMirrorsCache am + LEFT JOIN dbo.ArchiveDownloads ad on am.Hash = ad.Hash and ad.DownloadFinished is not null and ad.IsFailed = 0 + LEFT JOIN dbo.ModListArchives mla on am.Hash = mla.Hash GO +/****** Object: Table [dbo].[AccessLog] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[AccessLog]( + [Id] [bigint] IDENTITY(1,1) NOT NULL, + [Timestamp] [datetime] NOT NULL, + [Action] [nvarchar](max) NOT NULL, + [Ip] [nvarchar](50) NOT NULL, + [MetricsKey] AS (json_value([Action],'$.Headers."x-metrics-key"[0]')), + CONSTRAINT [PK_AccessLog] PRIMARY KEY CLUSTERED +( + [Id] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[ApiKeys] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[ApiKeys]( + [APIKey] [nvarchar](260) NOT NULL, + [Owner] [nvarchar](40) NOT NULL, + CONSTRAINT [PK_ApiKeys] PRIMARY KEY CLUSTERED + ( + [APIKey] ASC, + [Owner] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO +/****** Object: Table [dbo].[ArchivePatches] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[ArchivePatches]( + [SrcPrimaryKeyStringHash] [binary](32) NOT NULL, + [SrcPrimaryKeyString] [nvarchar](max) NOT NULL, + [SrcHash] [bigint] NOT NULL, + [DestPrimaryKeyStringHash] [binary](32) NOT NULL, + [DestPrimaryKeyString] [nvarchar](max) NOT NULL, + [DestHash] [bigint] NOT NULL, + [SrcState] [nvarchar](max) NOT NULL, + [DestState] [nvarchar](max) NOT NULL, + [SrcDownload] [nvarchar](max) NULL, + [DestDownload] [nvarchar](max) NULL, + [CDNPath] [nvarchar](max) NULL, + CONSTRAINT [PK_ArchivePatches] PRIMARY KEY CLUSTERED +( + [SrcPrimaryKeyStringHash] ASC, + [SrcHash] ASC, + [DestPrimaryKeyStringHash] ASC, + [DestHash] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[IPFSPins] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[IPFSPins]( + [Hash] [bigint] NOT NULL, + [Name] [nvarchar](max) NULL, + [CID] [varchar](46) NOT NULL, + CONSTRAINT [PK_IPFSPins] PRIMARY KEY CLUSTERED +( + [Hash] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[Jobs] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO CREATE TABLE [dbo].[Jobs]( - [Id] [bigint] IDENTITY(1,1) NOT NULL, - [Priority] [int] NOT NULL, - [PrimaryKeyString] [nvarchar](max) NULL, - [Started] [datetime] NULL, - [Ended] [datetime] NULL, - [Created] [datetime] NOT NULL, - [Success] [tinyint] NULL, - [ResultContent] [nvarchar](max) NULL, - [Payload] [nvarchar](max) NULL, - [OnSuccess] [nvarchar](max) NULL, - [RunBy] [uniqueidentifier] NULL - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Index [ClusteredIndex-20200506-123511] Script Date: 12/29/2020 8:55:04 PM ******/ + [Id] [bigint] IDENTITY(1,1) NOT NULL, + [Priority] [int] NOT NULL, + [PrimaryKeyString] [nvarchar](max) NULL, + [Started] [datetime] NULL, + [Ended] [datetime] NULL, + [Created] [datetime] NOT NULL, + [Success] [tinyint] NULL, + [ResultContent] [nvarchar](max) NULL, + [Payload] [nvarchar](max) NULL, + [OnSuccess] [nvarchar](max) NULL, + [RunBy] [uniqueidentifier] NULL +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Index [ClusteredIndex-20200506-123511] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE CLUSTERED INDEX [ClusteredIndex-20200506-123511] ON [dbo].[Jobs] ( [Priority] DESC, [Started] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] GO -/****** Object: Table [dbo].[Metrics] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Table [dbo].[Metrics] Script Date: 3/9/2021 11:12:53 PM ******/ SET ANSI_NULLS ON GO SET QUOTED_IDENTIFIER ON GO CREATE TABLE [dbo].[Metrics]( - [Id] [bigint] IDENTITY(1,1) NOT NULL, - [Timestamp] [datetime] NOT NULL, - [Action] [varchar](64) NOT NULL, - [Subject] [varchar](max) NOT NULL, - [MetricsKey] [varchar](64) NULL, - [GroupingSubject] AS (case when [Action]='started_wabbajack' then [Subject] else substring([Subject],(0),case when patindex('%[0-9].%',[Subject])=(0) then len([Subject])+(1) else patindex('%[0-9].%',[Subject]) end) end), - CONSTRAINT [PK_Metrics] PRIMARY KEY CLUSTERED + [Id] [bigint] IDENTITY(1,1) NOT NULL, + [Timestamp] [datetime] NOT NULL, + [Action] [varchar](64) NOT NULL, + [Subject] [varchar](max) NOT NULL, + [MetricsKey] [varchar](64) NULL, + [GroupingSubject] AS (case when [Action]='started_wabbajack' then [Subject] else substring([Subject],(0),case when patindex('%[0-9].%',[Subject])=(0) then len([Subject])+(1) else patindex('%[0-9].%',[Subject]) end) end), + CONSTRAINT [PK_Metrics] PRIMARY KEY CLUSTERED ( -[Id] ASC + [Id] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Table [dbo].[MirroredArchives] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[MirroredArchives]( - [Hash] [bigint] NOT NULL, - [Created] [datetime] NOT NULL, - [Uploaded] [datetime] NULL, - [Rationale] [nvarchar](max) NOT NULL, - [FailMessage] [nvarchar](max) NULL, - CONSTRAINT [PK_MirroredArchives] PRIMARY KEY CLUSTERED -( -[Hash] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Table [dbo].[ModListArchives] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[ModListArchives]( - [MachineUrl] [nvarchar](50) NOT NULL, - [Hash] [bigint] NOT NULL, - [PrimaryKeyString] [nvarchar](max) NOT NULL, - [Size] [bigint] NOT NULL, - [State] [nvarchar](max) NOT NULL, - [Name] [nvarchar](max) NULL, - CONSTRAINT [PK_ModListArchive] PRIMARY KEY CLUSTERED -( - [MachineUrl] ASC, -[Hash] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO - -/****** Object: Table [dbo].[MetricsKeys] Script Date: 2/17/2021 9:20:27 PM ******/ +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[MetricsKeys] Script Date: 3/9/2021 11:12:53 PM ******/ SET ANSI_NULLS ON GO - SET QUOTED_IDENTIFIER ON GO - CREATE TABLE [dbo].[MetricsKeys]( -[MetricsKey] [varchar](64) NOT NULL, -CONSTRAINT [PK_MetricsKeys] PRIMARY KEY CLUSTERED - ( - [MetricsKey] ASC - )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] + [MetricsKey] [varchar](64) NOT NULL, + CONSTRAINT [PK_MetricsKeys] PRIMARY KEY CLUSTERED + ( + [MetricsKey] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] ) ON [PRIMARY] GO - - - -/****** Object: Table [dbo].[ModListArchiveStatus] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[ModListArchiveStatus]( - [PrimaryKeyStringHash] [binary](32) NOT NULL, - [Hash] [bigint] NOT NULL, - [PrimaryKeyString] [nvarchar](max) NOT NULL, - [IsValid] [tinyint] NOT NULL, - CONSTRAINT [PK_ModListArchiveStatus] PRIMARY KEY CLUSTERED -( - [PrimaryKeyStringHash] ASC, -[Hash] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Table [dbo].[NexusFileInfos] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[NexusFileInfos]( - [Game] [int] NOT NULL, - [ModId] [bigint] NOT NULL, - [FileId] [bigint] NOT NULL, - [LastChecked] [datetime] NOT NULL, - [Data] [nvarchar](max) NOT NULL, - CONSTRAINT [PK_NexusFileInfos] PRIMARY KEY CLUSTERED -( - [Game] ASC, - [ModId] ASC, -[FileId] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Table [dbo].[NexusKeys] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[NexusKeys]( - [ApiKey] [nvarchar](162) NOT NULL, - [DailyRemain] [int] NOT NULL, - [HourlyRemain] [int] NOT NULL, - CONSTRAINT [PK_NexusKeys] PRIMARY KEY CLUSTERED -( -[ApiKey] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] - GO -/****** Object: Table [dbo].[NexusModFiles] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[NexusModFiles]( - [Game] [int] NOT NULL, - [ModId] [bigint] NOT NULL, - [LastChecked] [datetime] NOT NULL, - [Data] [nvarchar](max) NOT NULL, - CONSTRAINT [PK_NexusModFiles] PRIMARY KEY CLUSTERED -( - [Game] ASC, -[ModId] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Table [dbo].[NexusModFilesSlow] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[NexusModFilesSlow]( - [GameId] [bigint] NOT NULL, - [FileId] [bigint] NOT NULL, - [ModId] [bigint] NOT NULL, - [LastChecked] [datetime] NOT NULL, - CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED - ( - [GameId] ASC, - [FileId] ASC, -[ModId] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] - GO -/****** Object: Table [dbo].[NexusModInfos] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[NexusModInfos]( - [Game] [int] NOT NULL, - [ModId] [bigint] NOT NULL, - [LastChecked] [datetime] NOT NULL, - [Data] [nvarchar](max) NOT NULL, - CONSTRAINT [PK_NexusModInfos] PRIMARY KEY CLUSTERED -( - [Game] ASC, -[ModId] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Table [dbo].[NexusModPermissions] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[NexusModPermissions]( - [NexusGameID] [int] NOT NULL, - [ModID] [bigint] NOT NULL, - [Permissions] [int] NOT NULL, - CONSTRAINT [PK_NexusModPermissions] PRIMARY KEY CLUSTERED - ( - [NexusGameID] ASC, -[ModID] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] - GO -/****** Object: Table [dbo].[NoPatch] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[NoPatch]( - [Hash] [bigint] NOT NULL, - [Created] [datetime] NOT NULL, - [Rationale] [nvarchar](max) NOT NULL, - CONSTRAINT [PK_NoPatch] PRIMARY KEY CLUSTERED -( -[Hash] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Table [dbo].[Patches] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[Patches]( - [SrcId] [uniqueidentifier] NOT NULL, - [DestId] [uniqueidentifier] NOT NULL, - [PatchSize] [bigint] NULL, - [Finished] [datetime] NULL, - [IsFailed] [tinyint] NULL, - [FailMessage] [varchar](max) NULL, - [LastUsed] [datetime] NULL, - [Downloads] [bigint] NOT NULL, - CONSTRAINT [PK_Patches] PRIMARY KEY CLUSTERED -( - [SrcId] ASC, -[DestId] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Table [dbo].[TarKey] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[TarKey]( - [MetricsKey] [nvarchar](64) NOT NULL, - CONSTRAINT [PK_TarKey] PRIMARY KEY CLUSTERED -( -[MetricsKey] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] - GO -/****** Object: Table [dbo].[toDeleteTemp] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[toDeleteTemp]( - [Parent] [bigint] NULL, - [Child] [bigint] NULL, - [Path] [varchar](max) NULL, - [PathHash] [binary](32) NULL - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Table [dbo].[UploadedFiles] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[UploadedFiles]( - [Id] [uniqueidentifier] NOT NULL, - [Name] [nvarchar](max) NOT NULL, - [Size] [bigint] NOT NULL, - [UploadedBy] [nvarchar](40) NOT NULL, - [Hash] [bigint] NOT NULL, - [UploadDate] [datetime] NOT NULL, - [CDNName] [nvarchar](max) NULL, - CONSTRAINT [PK_UploadedFiles] PRIMARY KEY CLUSTERED -( -[Id] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] - GO -/****** Object: Table [dbo].[VirusScanResults] Script Date: 12/29/2020 8:55:04 PM ******/ - SET ANSI_NULLS ON - GO - SET QUOTED_IDENTIFIER ON - GO -CREATE TABLE [dbo].[VirusScanResults]( - [Hash] [bigint] NOT NULL, - [IsMalware] [tinyint] NOT NULL, - CONSTRAINT [PK_VirusScanResults] PRIMARY KEY CLUSTERED - ( -[Hash] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] - ) ON [PRIMARY] - GO - SET ANSI_PADDING ON - GO -/****** Object: Index [AccessLogByIP] Script Date: 12/29/2020 8:55:04 PM ******/ -CREATE NONCLUSTERED INDEX [AccessLogByIP] ON [dbo].[AccessLog] -( - [Ip] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] -GO -SET ARITHABORT ON -SET CONCAT_NULL_YIELDS_NULL ON -SET QUOTED_IDENTIFIER ON +/****** Object: Table [dbo].[MirroredArchives] Script Date: 3/9/2021 11:12:53 PM ******/ SET ANSI_NULLS ON -SET ANSI_PADDING ON -SET ANSI_WARNINGS ON -SET NUMERIC_ROUNDABORT OFF GO -/****** Object: Index [AccessLogByMetricsKey] Script Date: 12/29/2020 8:55:04 PM ******/ -CREATE NONCLUSTERED INDEX [AccessLogByMetricsKey] ON [dbo].[AccessLog] +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[MirroredArchives]( + [Hash] [bigint] NOT NULL, + [Created] [datetime] NOT NULL, + [Uploaded] [datetime] NULL, + [Rationale] [nvarchar](max) NOT NULL, + [FailMessage] [nvarchar](max) NULL, + CONSTRAINT [PK_MirroredArchives] PRIMARY KEY CLUSTERED ( - [MetricsKey] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] + [Hash] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] GO -/****** Object: Index [IX_Child] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Table [dbo].[ModListArchiveStatus] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[ModListArchiveStatus]( + [PrimaryKeyStringHash] [binary](32) NOT NULL, + [Hash] [bigint] NOT NULL, + [PrimaryKeyString] [nvarchar](max) NOT NULL, + [IsValid] [tinyint] NOT NULL, + CONSTRAINT [PK_ModListArchiveStatus] PRIMARY KEY CLUSTERED +( + [PrimaryKeyStringHash] ASC, + [Hash] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[NexusFileInfos] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[NexusFileInfos]( + [Game] [int] NOT NULL, + [ModId] [bigint] NOT NULL, + [FileId] [bigint] NOT NULL, + [LastChecked] [datetime] NOT NULL, + [Data] [nvarchar](max) NOT NULL, + CONSTRAINT [PK_NexusFileInfos] PRIMARY KEY CLUSTERED +( + [Game] ASC, + [ModId] ASC, + [FileId] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[NexusKeys] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[NexusKeys]( + [ApiKey] [nvarchar](162) NOT NULL, + [DailyRemain] [int] NOT NULL, + [HourlyRemain] [int] NOT NULL, + CONSTRAINT [PK_NexusKeys] PRIMARY KEY CLUSTERED + ( + [ApiKey] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO +/****** Object: Table [dbo].[NexusModFiles] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[NexusModFiles]( + [Game] [int] NOT NULL, + [ModId] [bigint] NOT NULL, + [LastChecked] [datetime] NOT NULL, + [Data] [nvarchar](max) NOT NULL, + CONSTRAINT [PK_NexusModFiles] PRIMARY KEY CLUSTERED +( + [Game] ASC, + [ModId] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[NexusModFilesSlow] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[NexusModFilesSlow]( + [GameId] [bigint] NOT NULL, + [FileId] [bigint] NOT NULL, + [ModId] [bigint] NOT NULL, + [LastChecked] [datetime] NOT NULL, + CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED + ( + [GameId] ASC, + [FileId] ASC, + [ModId] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO +/****** Object: Table [dbo].[NoPatch] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[NoPatch]( + [Hash] [bigint] NOT NULL, + [Created] [datetime] NOT NULL, + [Rationale] [nvarchar](max) NOT NULL, + CONSTRAINT [PK_NoPatch] PRIMARY KEY CLUSTERED +( + [Hash] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[Patches] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[Patches]( + [SrcId] [uniqueidentifier] NOT NULL, + [DestId] [uniqueidentifier] NOT NULL, + [PatchSize] [bigint] NULL, + [Finished] [datetime] NULL, + [IsFailed] [tinyint] NULL, + [FailMessage] [varchar](max) NULL, + [LastUsed] [datetime] NULL, + [Downloads] [bigint] NOT NULL, + CONSTRAINT [PK_Patches] PRIMARY KEY CLUSTERED +( + [SrcId] ASC, + [DestId] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[TarKey] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[TarKey]( + [MetricsKey] [nvarchar](64) NOT NULL, + CONSTRAINT [PK_TarKey] PRIMARY KEY CLUSTERED + ( + [MetricsKey] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO +/****** Object: Table [dbo].[toDeleteTemp] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[toDeleteTemp]( + [Parent] [bigint] NULL, + [Child] [bigint] NULL, + [Path] [varchar](max) NULL, + [PathHash] [binary](32) NULL +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[UploadedFiles] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[UploadedFiles]( + [Id] [uniqueidentifier] NOT NULL, + [Name] [nvarchar](max) NOT NULL, + [Size] [bigint] NOT NULL, + [UploadedBy] [nvarchar](40) NOT NULL, + [Hash] [bigint] NOT NULL, + [UploadDate] [datetime] NOT NULL, + [CDNName] [nvarchar](max) NULL, + CONSTRAINT [PK_UploadedFiles] PRIMARY KEY CLUSTERED +( + [Id] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO +/****** Object: Table [dbo].[VirusScanResults] Script Date: 3/9/2021 11:12:53 PM ******/ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[VirusScanResults]( + [Hash] [bigint] NOT NULL, + [IsMalware] [tinyint] NOT NULL, + CONSTRAINT [PK_VirusScanResults] PRIMARY KEY CLUSTERED + ( + [Hash] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO +/****** Object: Index [IX_Child] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE NONCLUSTERED INDEX [IX_Child] ON [dbo].[AllFilesInArchive] ( [Child] ASC @@ -812,14 +926,14 @@ CREATE NONCLUSTERED INDEX [IX_Child] ON [dbo].[AllFilesInArchive] GO SET ANSI_PADDING ON GO -/****** Object: Index [ByAPIKey] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Index [ByAPIKey] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE UNIQUE NONCLUSTERED INDEX [ByAPIKey] ON [dbo].[ApiKeys] -( - [APIKey] ASC -) -INCLUDE([Owner]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] -GO -/****** Object: Index [_dta_index_ArchiveDownloads_5_1058102810__K6_1_3_4_7] Script Date: 12/29/2020 8:55:04 PM ******/ + ( + [APIKey] ASC + ) + INCLUDE([Owner]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] + GO +/****** Object: Index [_dta_index_ArchiveDownloads_5_1058102810__K6_1_3_4_7] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE NONCLUSTERED INDEX [_dta_index_ArchiveDownloads_5_1058102810__K6_1_3_4_7] ON [dbo].[ArchiveDownloads] ( [DownloadFinished] ASC @@ -828,7 +942,7 @@ INCLUDE([Id],[Size],[Hash],[DownloadState]) WITH (PAD_INDEX = OFF, STATISTICS_NO GO SET ANSI_PADDING ON GO -/****** Object: Index [ByDownloaderAndFinished] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Index [ByDownloaderAndFinished] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE NONCLUSTERED INDEX [ByDownloaderAndFinished] ON [dbo].[ArchiveDownloads] ( [DownloadFinished] ASC, @@ -837,21 +951,28 @@ CREATE NONCLUSTERED INDEX [ByDownloaderAndFinished] ON [dbo].[ArchiveDownloads] GO SET ANSI_PADDING ON GO -/****** Object: Index [ByPrimaryKeyAndHash] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Index [ByPrimaryKeyAndHash] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE NONCLUSTERED INDEX [ByPrimaryKeyAndHash] ON [dbo].[ArchiveDownloads] ( [PrimaryKeyString] ASC, [Hash] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] GO -/****** Object: Index [IDX_ID_HASH] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Index [IDX_ID_HASH] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE NONCLUSTERED INDEX [IDX_ID_HASH] ON [dbo].[ArchiveDownloads] ( [Id] ASC, [Hash] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] GO -/****** Object: Index [ByHash] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Index [_dta_index_AuthoredFiles_LastTouched] Script Date: 3/9/2021 11:12:53 PM ******/ +CREATE NONCLUSTERED INDEX [_dta_index_AuthoredFiles_LastTouched] ON [dbo].[AuthoredFiles] +( + [LastTouched] ASC +) +INCLUDE([ServerAssignedUniqueId],[CDNFileDefinition],[Finalized]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +GO +/****** Object: Index [ByHash] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE NONCLUSTERED INDEX [ByHash] ON [dbo].[DownloadStates] ( [Hash] ASC @@ -860,14 +981,14 @@ INCLUDE([IniState]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN GO SET ANSI_PADDING ON GO -/****** Object: Index [GameAndName-20200804-164236] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Index [GameAndName-20200804-164236] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE NONCLUSTERED INDEX [GameAndName-20200804-164236] ON [dbo].[GameMetadata] ( [NexusGameId] ASC, [WabbajackName] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] GO -/****** Object: Index [JobsByEnded] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Index [JobsByEnded] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE NONCLUSTERED INDEX [JobsByEnded] ON [dbo].[Jobs] ( [Ended] ASC @@ -876,29 +997,29 @@ INCLUDE([Id],[Priority],[Started],[Created],[Payload]) WITH (PAD_INDEX = OFF, ST GO SET ANSI_PADDING ON GO -/****** Object: Index [IDX_MetricsKey] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Index [IDX_MetricsKey] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE NONCLUSTERED INDEX [IDX_MetricsKey] ON [dbo].[Metrics] ( [MetricsKey] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] GO -/****** Object: Index [IDX_MirroredArchives_HashUploaded] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: Index [IDX_MirroredArchives_HashUploaded] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE UNIQUE NONCLUSTERED INDEX [IDX_MirroredArchives_HashUploaded] ON [dbo].[MirroredArchives] -( - [Hash] ASC, - [Uploaded] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] -GO -/****** Object: Index [IDX_HASH] Script Date: 12/29/2020 8:55:04 PM ******/ + ( + [Hash] ASC, + [Uploaded] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] + GO +/****** Object: Index [IDX_HASH] Script Date: 3/9/2021 11:12:53 PM ******/ CREATE NONCLUSTERED INDEX [IDX_HASH] ON [dbo].[ModListArchives] ( [Hash] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] GO ALTER TABLE [dbo].[ModLists] ADD DEFAULT ((0)) FOR [BrokenDownload] - GO +GO ALTER TABLE [dbo].[Patches] ADD DEFAULT ((0)) FOR [Downloads] - GO +GO ALTER TABLE [dbo].[Patches] WITH CHECK ADD CONSTRAINT [FK_DestId] FOREIGN KEY([DestId]) REFERENCES [dbo].[ArchiveDownloads] ([Id]) GO @@ -909,7 +1030,7 @@ ALTER TABLE [dbo].[Patches] WITH CHECK ADD CONSTRAINT [FK_SrcId] FOREIGN KEY([ GO ALTER TABLE [dbo].[Patches] CHECK CONSTRAINT [FK_SrcId] GO -/****** Object: StoredProcedure [dbo].[MergeAllFilesInArchive] Script Date: 12/29/2020 8:55:04 PM ******/ +/****** Object: StoredProcedure [dbo].[QueueMirroredFiles] Script Date: 3/9/2021 11:12:53 PM ******/ SET ANSI_NULLS ON GO SET QUOTED_IDENTIFIER ON @@ -919,66 +1040,18 @@ ALTER TABLE [dbo].[Patches] CHECK CONSTRAINT [FK_SrcId] -- Create date: -- Description: -- ============================================= -CREATE PROCEDURE [dbo].[MergeAllFilesInArchive] +CREATE PROCEDURE [dbo].[QueueMirroredFiles] AS BEGIN - -- SET NOCOUNT ON added to prevent extra result sets from - -- interfering with SELECT statements. - SET NOCOUNT ON; + -- SET NOCOUNT ON added to prevent extra result sets from + -- interfering with SELECT statements. + SET NOCOUNT ON; -MERGE dbo.AllFilesInArchive t USING ( - SELECT DISTINCT TopParent, unpvt.Child - FROM - (SELECT a3.Parent AS P3, a2.Parent as P2, a1.Parent P1, a0.Parent P0, a0.Parent as Parent, a0.Child FROM - dbo.ArChiveContent a0 - LEFT JOIN dbo.ArChiveContent a1 ON a0.Parent = a1.Child - LEFT JOIN dbo.ArChiveContent a2 ON a1.Parent = a2.Child - LEFT JOIN dbo.ArChiveContent a3 ON a2.Parent = a3.Child) p - UNPIVOT - (TopParent For C IN (p.P3, p.P2, p.P1, p.P0)) as unpvt - LEFT JOIN dbo.IndexedFile idf on unpvt.Child = idf.Hash - WHERE TopParent is not null) s - ON t.TopParent = s.TopParent AND t.Child = s.Child - WHEN NOT MATCHED - THEN INSERT (TopParent, Child) VALUES (s.TopParent, s.Child); +INSERT INTO dbo.AllowedMirrorsCache (Hash, Reason) SELECT Hash, Reason FROM dbo.AllowedMirrors WHERE Hash not in (SELECT Hash from dbo.AllowedMirrorsCache); + +DELETE from dbo.MirroredArchives where Hash not in (SELECT Hash from dbo.ActiveMirrors); END -GO -/****** Object: StoredProcedure [dbo].[MergeIndexedFiles] Script Date: 12/29/2020 8:55:04 PM ******/ -SET ANSI_NULLS ON -GO -SET QUOTED_IDENTIFIER ON -GO --- ============================================= -CREATE PROCEDURE [dbo].[MergeIndexedFiles] - -- Add the parameters for the stored procedure here - @Files dbo.IndexedFileType READONLY, - @Contents dbo.ArchiveContentType READONLY -AS -BEGIN - -- SET NOCOUNT ON added to prevent extra result sets from - -- interfering with SELECT statements. - SET NOCOUNT ON; -BEGIN TRANSACTION; - -MERGE dbo.IndexedFile AS TARGET - USING (SELECT DISTINCT * FROM @Files) as SOURCE - ON (TARGET.Hash = SOURCE.HASH) - WHEN NOT MATCHED BY TARGET - THEN INSERT (Hash, Sha256, Sha1, Md5, Crc32, Size) - VALUES (Source.Hash, Source.Sha256, Source.Sha1, Source.Md5, Source.Crc32, Source.Size); - -MERGE dbo.ArchiveContent AS TARGET - USING (SELECT DISTINCT * FROM @Contents) as SOURCE - ON (TARGET.Parent = SOURCE.Parent AND TARGET.PathHash = CAST(HASHBYTES('SHA2_256', REPLACE(LOWER(SOURCE.Path), '/', '\')) as binary(32))) - WHEN NOT MATCHED - THEN INSERT (Parent, Child, Path) - VALUES (Source.Parent, Source.Child, Source.Path); - - -COMMIT; - -END -GO + GO USE [master] GO ALTER DATABASE [wabbajack_prod] SET READ_WRITE diff --git a/Wabbajack.Server/Controllers/AuthoredFiles.cs b/Wabbajack.Server/Controllers/AuthoredFiles.cs index 51e9a742..96babbd9 100644 --- a/Wabbajack.Server/Controllers/AuthoredFiles.cs +++ b/Wabbajack.Server/Controllers/AuthoredFiles.cs @@ -114,8 +114,9 @@ namespace Wabbajack.BuildServer.Controllers await _discord.Send(Channel.Ham, new DiscordMessage {Content = $"{user} has finished uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})"}); - - return Ok($"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{definition.MungedName}"); + + var host = Consts.TestMode ? "test-files" : "authored-files"; + return Ok($"https://{host}.wabbajack.org/{definition.MungedName}"); } private async Task GetBunnyCdnFtpClient() @@ -172,7 +173,7 @@ namespace Wabbajack.BuildServer.Controllers {{each $.files }} - + {{/each}}
{{$.OriginalFileName}}{{$.Size}}{{$.LastTouched}}{{$.Finalized}}{{$.Author}}
{{$.OriginalFileName}}{{$.Size}}{{$.LastTouched}}{{$.Finalized}}{{$.Author}}
diff --git a/Wabbajack.Server/Controllers/ModUpgrade.cs b/Wabbajack.Server/Controllers/ModUpgrade.cs index 84bc6a13..65c53257 100644 --- a/Wabbajack.Server/Controllers/ModUpgrade.cs +++ b/Wabbajack.Server/Controllers/ModUpgrade.cs @@ -102,10 +102,11 @@ namespace Wabbajack.BuildServer.Controllers if (patch.PatchSize != 0) { //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found"); + var host = (await _creds).Username == "wabbajacktest" ? "test-files" : "patches"; await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id); return Ok( - $"https://{(await _creds).Username}.b-cdn.net/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); + $"https://{host}.wabbajack.org/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); } //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed"); diff --git a/Wabbajack.Server/DTOs/ValidationData.cs b/Wabbajack.Server/DTOs/ValidationData.cs index 3206f610..58eb7b09 100644 --- a/Wabbajack.Server/DTOs/ValidationData.cs +++ b/Wabbajack.Server/DTOs/ValidationData.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.ModListRegistry; @@ -12,6 +14,7 @@ namespace Wabbajack.Server.DTOs public List ModLists { get; set; } public ConcurrentHashSet<(Game Game, long ModId)> SlowQueriedFor { get; set; } = new ConcurrentHashSet<(Game Game, long ModId)>(); - public HashSet Mirrors { get; set; } + public Dictionary Mirrors { get; set; } + public Lazy>> AllowedMirrors { get; set; } } } diff --git a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs index ee328419..8a988330 100644 --- a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs +++ b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs @@ -248,7 +248,8 @@ namespace Wabbajack.Server.DataLayer if (await HaveMirror(hash) && files.Count > 0) { var ffile = files.First(); - var url = new Uri($"https://{(await _mirrorCreds).Username}.b-cdn.net/{hash.ToHex()}"); + var host = Consts.TestMode ? "test-files" : "mirror"; + var url = new Uri($"https://{host}.wabbajack.org/{hash.ToHex()}"); files.Add(new Archive( new WabbajackCDNDownloader.State(url)) {Hash = hash, Size = ffile.Size, Name = ffile.Name}); } diff --git a/Wabbajack.Server/DataLayer/MirroredFiles.cs b/Wabbajack.Server/DataLayer/MirroredFiles.cs index 9c3b5803..ba3a6b4f 100644 --- a/Wabbajack.Server/DataLayer/MirroredFiles.cs +++ b/Wabbajack.Server/DataLayer/MirroredFiles.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using CefSharp.DevTools.Network; using Dapper; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; @@ -24,10 +25,38 @@ namespace Wabbajack.Server.DataLayer }; } - public async Task> GetAllMirroredHashes() + public async Task> GetAllMirroredHashes() { await using var conn = await Open(); - return (await conn.QueryAsync("SELECT Hash FROM dbo.MirroredArchives")).ToHashSet(); + return (await conn.QueryAsync<(Hash, DateTime?)>("SELECT Hash, Uploaded FROM dbo.MirroredArchives")) + .GroupBy(d => d.Item1) + .ToDictionary(d => d.Key, d => d.First().Item2.HasValue); + } + + public async Task StartMirror((Hash Hash, string Reason) mirror) + { + await using var conn = await Open(); + await using var trans = await conn.BeginTransactionAsync(); + + if (await conn.QueryFirstOrDefaultAsync(@"SELECT Hash FROM dbo.MirroredArchives WHERE Hash = @Hash", + new {Hash = mirror.Hash}, trans) != default) + { + return; + } + + await conn.ExecuteAsync( + @"INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) VALUES (@Hash, GETUTCDATE(), @Reason)", + new {Hash = mirror.Hash, Reason = mirror.Reason}, trans); + await trans.CommitAsync(); + + } + + public async Task> GetAllowedMirrors() + { + await using var conn = await Open(); + return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, Reason FROM dbo.AllowedMirrorsCache")) + .GroupBy(d => d.Item1) + .ToDictionary(d => d.Key, d => d.First().Item2); } public async Task UpsertMirroredFile(MirroredFile file) @@ -70,7 +99,7 @@ namespace Wabbajack.Server.DataLayer foreach (var (key, _) in permissions) { if (!downloads.TryGetValue(key, out var hash)) continue; - if (existing.Contains(hash)) continue; + if (existing.ContainsKey(hash)) continue; await UpsertMirroredFile(new MirroredFile { @@ -130,5 +159,20 @@ namespace Wabbajack.Server.DataLayer "); } + + public async Task AddNexusModWithOpenPerms(Game gameGame, long modId) + { + await using var conn = await Open(); + + await conn.ExecuteAsync( + @"INSERT INTO dbo.NexusModsWithOpenPerms(NexusGameID, NexusModID) VALUES(@game, @mod)", + new {game = gameGame.MetaData().NexusGameId, modId}); + } + + public async Task SyncActiveMirroredFiles() + { + await using var conn = await Open(); + await conn.ExecuteAsync(@"EXEC dbo.QueueMirroredFiles"); + } } } diff --git a/Wabbajack.Server/DataLayer/ValidationData.cs b/Wabbajack.Server/DataLayer/ValidationData.cs index e2f25af3..36804276 100644 --- a/Wabbajack.Server/DataLayer/ValidationData.cs +++ b/Wabbajack.Server/DataLayer/ValidationData.cs @@ -24,6 +24,7 @@ namespace Wabbajack.Server.DataLayer ArchiveStatus = await archiveStatus, ModLists = await modLists, Mirrors = await mirrors, + AllowedMirrors = new Lazy>>(async () => await GetAllowedMirrors()), }; } diff --git a/Wabbajack.Server/Services/ArchiveDownloader.cs b/Wabbajack.Server/Services/ArchiveDownloader.cs index db2b5fe1..e9900d74 100644 --- a/Wabbajack.Server/Services/ArchiveDownloader.cs +++ b/Wabbajack.Server/Services/ArchiveDownloader.cs @@ -17,18 +17,20 @@ namespace Wabbajack.Server.Services private ArchiveMaintainer _archiveMaintainer; private NexusApiClient _nexusClient; private DiscordWebHook _discord; + private NexusKeyMaintainance _nexus; - public ArchiveDownloader(ILogger logger, AppSettings settings, SqlService sql, ArchiveMaintainer archiveMaintainer, DiscordWebHook discord, QuickSync quickSync) + public ArchiveDownloader(ILogger logger, AppSettings settings, SqlService sql, ArchiveMaintainer archiveMaintainer, DiscordWebHook discord, QuickSync quickSync, NexusKeyMaintainance nexus) : base(logger, settings, quickSync, TimeSpan.FromMinutes(10)) { _sql = sql; _archiveMaintainer = archiveMaintainer; _discord = discord; + _nexus = nexus; } public override async Task Execute() { - _nexusClient ??= await NexusApiClient.Get(); + _nexusClient ??= await _nexus.GetClient(); int count = 0; while (true) diff --git a/Wabbajack.Server/Services/DiscordFrontend.cs b/Wabbajack.Server/Services/DiscordFrontend.cs index ec25be9b..56986caf 100644 --- a/Wabbajack.Server/Services/DiscordFrontend.cs +++ b/Wabbajack.Server/Services/DiscordFrontend.cs @@ -20,8 +20,9 @@ namespace Wabbajack.Server.Services private DiscordSocketClient _client; private SqlService _sql; private MetricsKeyCache _keyCache; + private ListValidator _listValidator; - public DiscordFrontend(ILogger logger, AppSettings settings, QuickSync quickSync, SqlService sql, MetricsKeyCache keyCache) + public DiscordFrontend(ILogger logger, AppSettings settings, QuickSync quickSync, ListValidator listValidator, SqlService sql, MetricsKeyCache keyCache) { _logger = logger; _settings = settings; @@ -35,6 +36,7 @@ namespace Wabbajack.Server.Services _sql = sql; _keyCache = keyCache; + _listValidator = listValidator; } private async Task MessageReceivedAsync(SocketMessage arg) @@ -86,10 +88,15 @@ namespace Wabbajack.Server.Services else { var deleted = await _sql.PurgeList(parts[2]); + _listValidator.ValidationInfo.TryRemove(parts[2], out var _); await _quickSync.Notify(); await ReplyTo(arg, $"Purged all traces of #{parts[2]} from the server, triggered list downloading. {deleted} records removed"); } } + else if (parts[1] == "mirror-mod") + { + await MirrorModCommand(arg, parts); + } else if (parts[1] == "users") { await ReplyTo(arg, $"Wabbajack has {await _keyCache.KeyCount()} known unique users"); @@ -97,6 +104,40 @@ namespace Wabbajack.Server.Services } } + private async Task MirrorModCommand(SocketMessage msg, string[] parts) + { + if (parts.Length != 2) + { + await ReplyTo(msg, "Command is: mirror-mod "); + return; + } + + if (long.TryParse(parts[2], out var modId)) + { + await ReplyTo(msg, $"Got {modId} for a mod-id, expected a integer"); + return; + } + + if (GameRegistry.TryGetByFuzzyName(parts[1], out var game)) + { + var gameNames = GameRegistry.Games.Select(g => g.Value.NexusName) + .Where(g => !string.IsNullOrWhiteSpace(g)) + .Select(g => (string)g) + .ToHashSet(); + var joined = string.Join(", ", gameNames.OrderBy(g => g)); + await ReplyTo(msg, $"Got {parts[1]} for a game name, expected something like: {joined}"); + } + + if (game!.NexusGameId == default) + { + await ReplyTo(msg, $"No NexusGameID found for {game}"); + } + + await _sql.AddNexusModWithOpenPerms(game.Game, modId); + await _quickSync.Notify(); + await ReplyTo(msg, "Done, and I notified the uploader"); + } + private async Task PurgeNexusCache(SocketMessage arg, string mod) { if (Uri.TryCreate(mod, UriKind.Absolute, out var url)) diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 39073fde..6f0dab85 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -68,8 +68,13 @@ namespace Wabbajack.Server.Services var (_, result) = await ValidateArchive(data, archive); if (result == ArchiveStatus.InValid) { - if (data.Mirrors.Contains(archive.Hash)) - return (archive, ArchiveStatus.Mirrored); + if (data.Mirrors.TryGetValue(archive.Hash, out var done)) + return (archive, done ? ArchiveStatus.Mirrored : ArchiveStatus.Updating); + if ((await data.AllowedMirrors.Value).TryGetValue(archive.Hash, out var reason)) + { + await _sql.StartMirror((archive.Hash, reason)); + return (archive, ArchiveStatus.Updating); + } return await TryToHeal(data, archive, metadata); } @@ -88,7 +93,7 @@ namespace Wabbajack.Server.Services } }); - var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid); + var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid || f.Item2 == ArchiveStatus.Updating); var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated); var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating); var mirroredCount = archives.Count(f => f.Item2 == ArchiveStatus.Mirrored); @@ -236,7 +241,12 @@ namespace Wabbajack.Server.Services return _archives.TryGetPath(foundArchive.Archive.Hash, out var path) ? path : default; }; - + + if (archive.State is NexusDownloader.State) + { + DownloadDispatcher.GetInstance().Client = await _nexus.GetClient(); + } + var upgrade = await DownloadDispatcher.FindUpgrade(archive, resolver); diff --git a/Wabbajack.Server/Services/MirrorUploader.cs b/Wabbajack.Server/Services/MirrorUploader.cs index 0db04f40..0d0c73ba 100644 --- a/Wabbajack.Server/Services/MirrorUploader.cs +++ b/Wabbajack.Server/Services/MirrorUploader.cs @@ -25,6 +25,8 @@ namespace Wabbajack.Server.Services private ArchiveMaintainer _archives; private DiscordWebHook _discord; + public bool ActiveFileSyncEnabled { get; set; } = true; + public MirrorUploader(ILogger logger, AppSettings settings, SqlService sql, QuickSync quickSync, ArchiveMaintainer archives, DiscordWebHook discord) : base(logger, settings, quickSync, TimeSpan.FromHours(1)) { @@ -37,9 +39,16 @@ namespace Wabbajack.Server.Services { int uploaded = 0; + + if (ActiveFileSyncEnabled) + await _sql.SyncActiveMirroredFiles(); TOP: var toUpload = await _sql.GetNextMirroredFile(); - if (toUpload == default) return uploaded; + if (toUpload == default) + { + await DeleteOldMirrorFiles(); + return uploaded; + } uploaded += 1; try @@ -93,16 +102,21 @@ namespace Wabbajack.Server.Services fs.Position = part.Offset; await fs.ReadAsync(buffer); } - - using var client = await GetClient(creds); - var name = MakePath(part.Index); - await client.UploadAsync(new MemoryStream(buffer), name); + await CircuitBreaker.WithAutoRetryAllAsync(async () =>{ + using var client = await GetClient(creds); + var name = MakePath(part.Index); + await client.UploadAsync(new MemoryStream(buffer), name); + }); + }); - using (var client = await GetClient(creds)) + await CircuitBreaker.WithAutoRetryAllAsync(async () => { + using var client = await GetClient(creds); _logger.LogInformation($"Finishing mirror upload"); + + await using var ms = new MemoryStream(); await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true)) { @@ -112,7 +126,7 @@ namespace Wabbajack.Server.Services ms.Position = 0; var remoteName = $"{definition.Hash.ToHex()}/definition.json.gz"; await client.UploadAsync(ms, remoteName); - } + }); await toUpload.Finish(_sql); } @@ -130,11 +144,64 @@ namespace Wabbajack.Server.Services goto TOP; } - private static async Task GetClient(BunnyCdnFtpInfo creds) + private static async Task GetClient(BunnyCdnFtpInfo creds = null) { - var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password)); - await ftpClient.ConnectAsync(); - return ftpClient; + return await CircuitBreaker.WithAutoRetryAllAsync(async () => + { + creds ??= await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors); + + var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password)); + ftpClient.DataConnectionType = FtpDataConnectionType.EPSV; + await ftpClient.ConnectAsync(); + return ftpClient; + }); + } + + /// + /// Gets a list of all the Mirrored file hashes that physically exist on the CDN (via FTP lookup) + /// + /// + public async Task> GetHashesOnCDN() + { + using var ftpClient = await GetClient(); + var serverFiles = (await ftpClient.GetNameListingAsync("\\")); + + return serverFiles + .Select(f => ((RelativePath)f).FileName) + .Select(l => + { + try + { + return Hash.FromHex((string)l); + } + catch (Exception) { return default; } + }) + .Where(h => h != default) + .ToHashSet(); + } + + public async Task DeleteOldMirrorFiles() + { + var existingHashes = await GetHashesOnCDN(); + var fromSql = await _sql.GetAllMirroredHashes(); + + foreach (var (hash, _) in fromSql.Where(s => s.Value)) + { + Utils.Log($"Removing {hash} from SQL it's no longer in the CDN"); + if (!existingHashes.Contains(hash)) + await _sql.DeleteMirroredFile(hash); + } + + var toDelete = existingHashes.Where(h => !fromSql.ContainsKey(h)).ToArray(); + + using var client = await GetClient(); + foreach (var hash in toDelete) + { + await _discord.Send(Channel.Spam, + new DiscordMessage {Content = $"Removing mirrored file {hash}, as it's no longer in sql"}); + Utils.Log($"Removing {hash} from the CDN it's no longer in SQL"); + await client.DeleteDirectoryAsync(hash.ToHex()); + } } } } diff --git a/Wabbajack.Server/Services/ModListDownloader.cs b/Wabbajack.Server/Services/ModListDownloader.cs index eb253d32..3c7a82ea 100644 --- a/Wabbajack.Server/Services/ModListDownloader.cs +++ b/Wabbajack.Server/Services/ModListDownloader.cs @@ -112,6 +112,11 @@ namespace Wabbajack.Server.Services } } + await _discord.Send(Channel.Ham, + new DiscordMessage + { + Content = $"Ingesting {list.Links.MachineURL} version {modlist.Version}" + }); await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist, false); } catch (Exception ex) diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index f92a898e..42b67bd3 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -141,7 +141,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); - app.UseService(); + //app.UseService(); app.UseService(); app.UseService(); //app.UseService();