mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
Port over Metrics and Nexus tests delete old server projects
This commit is contained in:
Timothy Baldridge
parent
6abddc68bf
commit
3bbc130cc0
@ -1,14 +0,0 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Internal;
|
||||
|
||||
namespace Wabbajack.BuildServer.Model
|
||||
{
|
||||
public static class Extensions
|
||||
{
|
||||
public static void AddWabbajackDB<T>(this )
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
using System.Data;
|
||||
using System.Data.Common;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
|
||||
namespace Wabbajack.BuildServer.Model.Models
|
||||
{
|
||||
public class DbFactory
|
||||
{
|
||||
|
||||
|
||||
public static IDbConnection Connect()
|
||||
{
|
||||
return new SqlConnection(Configuration);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,34 +0,0 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.BuildServer.Controllers;
|
||||
using Wabbajack.Common;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace Wabbajack.BuildServer.Test
|
||||
{
|
||||
[Collection("ServerTests")]
|
||||
public class BasicServerTests : ABuildServerSystemTest
|
||||
{
|
||||
|
||||
|
||||
|
||||
[Fact]
|
||||
public async Task CanGetHeartbeat()
|
||||
{
|
||||
var heartbeat = (await _client.GetStringAsync(MakeURL("heartbeat"))).FromJsonString<Heartbeat.HeartbeatResult>();
|
||||
Assert.True(heartbeat.Uptime > TimeSpan.Zero);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanContactAuthedEndpoint()
|
||||
{
|
||||
var logs = await _authedClient.GetStringAsync(MakeURL("heartbeat/logs"));
|
||||
Assert.NotEmpty(logs);
|
||||
}
|
||||
|
||||
public BasicServerTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,140 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using Wabbajack.Lib.FileUploader;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
using Xunit.Priority;
|
||||
|
||||
namespace Wabbajack.BuildServer.Test
|
||||
{
|
||||
[Collection("ServerTests")]
|
||||
public class IndexedFilesTests : ABuildServerSystemTest
|
||||
{
|
||||
|
||||
[Fact, Priority(1)]
|
||||
public async Task CanIngestExportedInis()
|
||||
{
|
||||
var sql = Fixture.GetService<SqlService>();
|
||||
var to = Fixture.ServerTempFolder.Combine("IniIngest");
|
||||
await @"sql\DownloadStates".RelativeTo(AbsolutePath.EntryPoint).CopyDirectoryToAsync(to);
|
||||
var result = await _authedClient.GetStringAsync(MakeURL("indexed_files/ingest/IniIngest"));
|
||||
Assert.Equal("5", result);
|
||||
|
||||
await ClearJobQueue();
|
||||
}
|
||||
|
||||
[Fact, Priority(2)]
|
||||
public async Task CanQueryViaHash()
|
||||
{
|
||||
var hashes = new HashSet<Hash>
|
||||
{
|
||||
Hash.FromHex("097ad17ef4b9f5b7"),
|
||||
Hash.FromHex("96fb53c3dc6397d2"),
|
||||
Hash.FromHex("97a6d27b7becba19")
|
||||
};
|
||||
|
||||
foreach (var hash in hashes)
|
||||
{
|
||||
Utils.Log($"Testing Archive {hash}");
|
||||
var ini = await ClientAPI.GetModIni(hash);
|
||||
Assert.NotNull(ini);
|
||||
Assert.NotNull(DownloadDispatcher.ResolveArchive(ini.LoadIniString()));
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanNotifyOfInis()
|
||||
{
|
||||
var archive =
|
||||
new Archive(
|
||||
new NexusDownloader.State
|
||||
{
|
||||
Game = Game.SkyrimSpecialEdition,
|
||||
ModID = long.MaxValue >> 3,
|
||||
FileID = long.MaxValue >> 3,
|
||||
})
|
||||
{
|
||||
Name = Guid.NewGuid().ToString()
|
||||
};
|
||||
Assert.True(await AuthorAPI.UploadPackagedInis(new[] {archive}));
|
||||
|
||||
var SQL = Fixture.GetService<SqlService>();
|
||||
var job = await SQL.GetJob();
|
||||
Assert.NotNull(job);
|
||||
Assert.IsType<IndexJob>(job.Payload);
|
||||
var payload = (IndexJob)job.Payload;
|
||||
|
||||
Assert.IsType<NexusDownloader.State>(payload.Archive.State);
|
||||
|
||||
var casted = (NexusDownloader.State)payload.Archive.State;
|
||||
Assert.Equal(Game.SkyrimSpecialEdition, casted.Game);
|
||||
|
||||
// Insert the record into SQL
|
||||
await SQL.AddDownloadState(Hash.FromHex("00e8bbbf591f61a3"), casted);
|
||||
|
||||
// Enqueue the same file again
|
||||
Assert.True(await AuthorAPI.UploadPackagedInis(new[] {archive}));
|
||||
|
||||
// File is aleady indexed so nothing gets enqueued
|
||||
Assert.Null(await SQL.GetJob());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanGetGameFiles()
|
||||
{
|
||||
var sql = Fixture.GetService<SqlService>();
|
||||
await sql.AddDownloadState(Hash.FromLong(1),
|
||||
new GameFileSourceDownloader.State("1.2.3.4")
|
||||
{
|
||||
Game = Game.SkyrimSpecialEdition,
|
||||
Hash = Hash.FromLong(1),
|
||||
GameFile = (RelativePath)@"Data\foo.bsa",
|
||||
});
|
||||
await sql.AddDownloadState(Hash.FromLong(2),
|
||||
new GameFileSourceDownloader.State("1.2.3.4")
|
||||
{
|
||||
Game = Game.SkyrimSpecialEdition,
|
||||
Hash = Hash.FromLong(2),
|
||||
GameFile = (RelativePath)@"Data\foo - Textures.bsa",
|
||||
});
|
||||
|
||||
|
||||
await sql.AddDownloadState(Hash.FromLong(3),
|
||||
new GameFileSourceDownloader.State("1.2.3.4")
|
||||
{
|
||||
Game = Game.Skyrim,
|
||||
Hash = Hash.FromLong(3),
|
||||
GameFile = (RelativePath)@"Data\foo - Textures.bsa",
|
||||
});
|
||||
|
||||
await sql.AddDownloadState(Hash.FromLong(4),
|
||||
new GameFileSourceDownloader.State("1.9.3.4")
|
||||
{
|
||||
Game = Game.SkyrimSpecialEdition,
|
||||
Hash = Hash.FromLong(4),
|
||||
GameFile = (RelativePath)@"Data\foo - Textures.bsa",
|
||||
});
|
||||
|
||||
var results = await ClientAPI.GetGameFiles(Game.SkyrimSpecialEdition, Version.Parse("1.2.3.4"));
|
||||
|
||||
Assert.Equal(new Dictionary<RelativePath, Hash>
|
||||
{
|
||||
{(RelativePath)@"Data\foo.bsa", Hash.FromLong(1)},
|
||||
{(RelativePath)@"Data\foo - Textures.bsa", Hash.FromLong(2)},
|
||||
}, results);
|
||||
|
||||
}
|
||||
|
||||
public IndexedFilesTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,59 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace Wabbajack.BuildServer.Test
|
||||
{
|
||||
[Collection("ServerTests")]
|
||||
|
||||
public class BasicTest : ABuildServerSystemTest
|
||||
{
|
||||
[Fact]
|
||||
public async Task CanEneuqueAndGetJobs()
|
||||
{
|
||||
var job = new Job {Payload = new GetNexusUpdatesJob()};
|
||||
var sqlService = Fixture.GetService<SqlService>();
|
||||
await sqlService.EnqueueJob(job);
|
||||
var found = await sqlService.GetJob();
|
||||
Assert.NotNull(found);
|
||||
Assert.IsAssignableFrom<GetNexusUpdatesJob>(found.Payload);
|
||||
found.Result = JobResult.Success();
|
||||
await sqlService.FinishJob(found);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PriorityMatters()
|
||||
{
|
||||
await ClearJobQueue();
|
||||
var sqlService = Fixture.GetService<SqlService>();
|
||||
var priority = new List<Job.JobPriority>
|
||||
{
|
||||
Job.JobPriority.Normal, Job.JobPriority.High, Job.JobPriority.Low
|
||||
};
|
||||
foreach (var pri in priority)
|
||||
await sqlService.EnqueueJob(new Job {Payload = new GetNexusUpdatesJob(), Priority = pri});
|
||||
|
||||
foreach (var pri in priority.OrderByDescending(p => (int)p))
|
||||
{
|
||||
var found = await sqlService.GetJob();
|
||||
Assert.NotNull(found);
|
||||
Assert.Equal(pri, found.Priority);
|
||||
found.Result = JobResult.Success();
|
||||
|
||||
// Finish the job so the next can run
|
||||
await sqlService.FinishJob(found);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public BasicTest(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,61 +0,0 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace Wabbajack.BuildServer.Test
|
||||
{
|
||||
|
||||
public class JobTests : ABuildServerSystemTest
|
||||
{
|
||||
public JobTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
|
||||
{
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanRunNexusUpdateJob()
|
||||
{
|
||||
var sql = Fixture.GetService<SqlService>();
|
||||
|
||||
var oldRecords = await NexusUpdatesFeeds.GetUpdates();
|
||||
foreach (var record in oldRecords)
|
||||
{
|
||||
await sql.AddNexusModInfo(record.Game, record.ModId, DateTime.UtcNow - TimeSpan.FromDays(1),
|
||||
new ModInfo());
|
||||
await sql.AddNexusModFiles(record.Game, record.ModId, DateTime.UtcNow - TimeSpan.FromDays(1),
|
||||
new NexusApiClient.GetModFilesResponse());
|
||||
|
||||
Assert.NotNull(await sql.GetModFiles(record.Game, record.ModId));
|
||||
Assert.NotNull(await sql.GetNexusModInfoString(record.Game, record.ModId));
|
||||
}
|
||||
|
||||
Utils.Log($"Ingested {oldRecords.Count()} nexus records");
|
||||
|
||||
// We know this will load the same records as above, but the date will be more recent, so the above records
|
||||
// should no longer exist in SQL after this job is run
|
||||
await sql.EnqueueJob(new Job {Payload = new GetNexusUpdatesJob()});
|
||||
await RunAllJobs();
|
||||
|
||||
foreach (var record in oldRecords)
|
||||
{
|
||||
Assert.Null(await sql.GetModFiles(record.Game, record.ModId));
|
||||
Assert.Null(await sql.GetNexusModInfoString(record.Game, record.ModId));
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanPrimeTheNexusCache()
|
||||
{
|
||||
var sql = Fixture.GetService<SqlService>();
|
||||
|
||||
Assert.True(await GetNexusUpdatesJob.UpdateNexusCacheFast(sql) > 0);
|
||||
Assert.True(await GetNexusUpdatesJob.UpdateNexusCacheFast(sql) == 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,39 +0,0 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.Common;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace Wabbajack.BuildServer.Test
|
||||
{
|
||||
public class LoginTests : ABuildServerSystemTest
|
||||
{
|
||||
public LoginTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
|
||||
{
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanCreateLogins()
|
||||
{
|
||||
var newUserName = Guid.NewGuid().ToString();
|
||||
|
||||
var newKey = await _authedClient.GetStringAsync(MakeURL($"users/add/{newUserName}"));
|
||||
|
||||
Assert.NotEmpty(newKey);
|
||||
Assert.NotNull(newKey);
|
||||
Assert.NotEqual(newKey, Fixture.APIKey);
|
||||
|
||||
|
||||
var done = await _authedClient.GetStringAsync(MakeURL("users/export"));
|
||||
Assert.Equal("done", done);
|
||||
|
||||
foreach (var (userName, apiKey) in new[] {(newUserName, newKey), (Fixture.User, Fixture.APIKey)})
|
||||
{
|
||||
var exported = await Fixture.ServerTempFolder.Combine("exported_users", userName, Consts.AuthorAPIKeyFile)
|
||||
.ReadAllTextAsync();
|
||||
Assert.Equal(exported, apiKey);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,250 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Security.Policy;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using HtmlAgilityPack;
|
||||
using Wabbajack.BuildServer.BackendServices;
|
||||
using Wabbajack.BuildServer.Controllers;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using Wabbajack.Lib.FileUploader;
|
||||
using Wabbajack.Lib.ModListRegistry;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
using IndexedFile = Wabbajack.BuildServer.Models.IndexedFile;
|
||||
|
||||
namespace Wabbajack.BuildServer.Test
|
||||
{
|
||||
public class ModListValidationTests : ABuildServerSystemTest
|
||||
{
|
||||
public ModListValidationTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanLoadMetadataFromTestServer()
|
||||
{
|
||||
var modlist = await MakeModList();
|
||||
Consts.ModlistMetadataURL = modlist.ToString();
|
||||
var data = await ModlistMetadata.LoadFromGithub();
|
||||
Assert.Single(data);
|
||||
Assert.Equal("test_list", data.First().Links.MachineURL);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanIngestModLists()
|
||||
{
|
||||
var modlist = await MakeModList();
|
||||
Consts.ModlistMetadataURL = modlist.ToString();
|
||||
var sql = Fixture.GetService<SqlService>();
|
||||
var service = new ListIngest(sql, Fixture.GetService<AppSettings>());
|
||||
await service.Execute();
|
||||
|
||||
foreach (var list in ModListMetaData)
|
||||
{
|
||||
Assert.True(await sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash));
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanValidateModLists()
|
||||
{
|
||||
await ClearJobQueue();
|
||||
var modlists = await MakeModList();
|
||||
Consts.ModlistMetadataURL = modlists.ToString();
|
||||
Utils.Log("Updating modlists");
|
||||
await RevalidateLists();
|
||||
|
||||
ListValidation.ResetCache();
|
||||
|
||||
Utils.Log("Checking validated results");
|
||||
var data = await ModlistMetadata.LoadFromGithub();
|
||||
Assert.Single(data);
|
||||
Assert.Equal(0, data.First().ValidationSummary.Failed);
|
||||
Assert.Equal(1, data.First().ValidationSummary.Passed);
|
||||
|
||||
await CheckListFeeds(0, 1);
|
||||
|
||||
Utils.Log("Break List");
|
||||
var archive = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
|
||||
await archive.MoveToAsync(archive.WithExtension(new Extension(".moved")), true);
|
||||
|
||||
// We can revalidate but the non-nexus archives won't be checked yet since the list didn't change
|
||||
await RevalidateLists();
|
||||
|
||||
data = await ModlistMetadata.LoadFromGithub();
|
||||
Assert.Single(data);
|
||||
Assert.Equal(0, data.First().ValidationSummary.Failed);
|
||||
Assert.Equal(1, data.First().ValidationSummary.Passed);
|
||||
|
||||
// Run the non-nexus validator
|
||||
var evalService = new ValidateNonNexusArchives(Fixture.GetService<SqlService>(), Fixture.GetService<AppSettings>());
|
||||
await evalService.Execute();
|
||||
|
||||
ListValidation.ResetCache();
|
||||
|
||||
data = await ModlistMetadata.LoadFromGithub();
|
||||
Assert.Single(data);
|
||||
Assert.Equal(1, data.First().ValidationSummary.Failed);
|
||||
Assert.Equal(0, data.First().ValidationSummary.Passed);
|
||||
|
||||
await CheckListFeeds(1, 0);
|
||||
|
||||
Utils.Log("Fix List");
|
||||
await archive.WithExtension(new Extension(".moved")).MoveToAsync(archive, false);
|
||||
|
||||
await RevalidateLists();
|
||||
// Rerun the validation service to fix the list
|
||||
await evalService.Execute();
|
||||
|
||||
ListValidation.ResetCache();
|
||||
|
||||
data = await ModlistMetadata.LoadFromGithub();
|
||||
Assert.Single(data);
|
||||
Assert.Equal(0, data.First().ValidationSummary.Failed);
|
||||
Assert.Equal(1, data.First().ValidationSummary.Passed);
|
||||
|
||||
await CheckListFeeds(0, 1);
|
||||
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanUpgradeHttpDownloads()
|
||||
{
|
||||
await ClearJobQueue();
|
||||
var modlists = await MakeModList();
|
||||
|
||||
await IndexFile(ModListData.Archives.First());
|
||||
|
||||
Consts.ModlistMetadataURL = modlists.ToString();
|
||||
Utils.Log("Updating modlists");
|
||||
await RevalidateLists();
|
||||
|
||||
Utils.Log("Checking validated results");
|
||||
var data = await ModlistMetadata.LoadFromGithub();
|
||||
Assert.Single(data);
|
||||
Assert.Equal(0, data.First().ValidationSummary.Failed);
|
||||
Assert.Equal(1, data.First().ValidationSummary.Passed);
|
||||
|
||||
await CheckListFeeds(0, 1);
|
||||
|
||||
var archive = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
|
||||
archive.Delete();
|
||||
await archive.WriteAllBytesAsync(Encoding.UTF8.GetBytes("More Cheese for Everyone!"));
|
||||
|
||||
var evalService = new ValidateNonNexusArchives(Fixture.GetService<SqlService>(), Fixture.GetService<AppSettings>());
|
||||
await evalService.Execute();
|
||||
await RevalidateLists();
|
||||
|
||||
ListValidation.ResetCache();
|
||||
|
||||
Utils.Log("Checking updated results");
|
||||
data = await ModlistMetadata.LoadFromGithub();
|
||||
Assert.Single(data);
|
||||
Assert.Equal(0, data.First().ValidationSummary.Failed);
|
||||
Assert.Equal(0, data.First().ValidationSummary.Passed);
|
||||
Assert.Equal(1, data.First().ValidationSummary.Updating);
|
||||
|
||||
await CheckListFeeds(1, 0);
|
||||
|
||||
}
|
||||
|
||||
private async Task IndexFile(Archive archive)
|
||||
{
|
||||
var job = new IndexJob {Archive = archive};
|
||||
await job.Execute(Fixture.GetService<SqlService>(), Fixture.GetService<AppSettings>());
|
||||
}
|
||||
|
||||
private async Task RevalidateLists()
|
||||
{
|
||||
var sql = Fixture.GetService<SqlService>();
|
||||
var settings = Fixture.GetService<AppSettings>();
|
||||
|
||||
var jobService = new ListIngest(sql, settings);
|
||||
await jobService.Execute();
|
||||
}
|
||||
|
||||
private async Task CheckListFeeds(int failed, int passed)
|
||||
{
|
||||
var statusJson = await _client.GetJsonAsync<DetailedStatus>(MakeURL("lists/status/test_list.json"));
|
||||
Assert.Equal(failed, statusJson.Archives.Count(a => a.IsFailing));
|
||||
Assert.Equal(passed, statusJson.Archives.Count(a => !a.IsFailing));
|
||||
|
||||
|
||||
var statusHtml = await _client.GetHtmlAsync(MakeURL("lists/status/test_list.html"));
|
||||
Assert.NotEmpty(statusHtml.DocumentNode.Descendants().Where(n => n.InnerHtml == $"Failed ({failed}):"));
|
||||
Assert.NotEmpty(statusHtml.DocumentNode.Descendants().Where(n => n.InnerHtml == $"Passed ({passed}):"));
|
||||
|
||||
var statusRss = await _client.GetHtmlAsync(MakeURL("lists/status/test_list/broken.rss"));
|
||||
Assert.Equal(failed, statusRss.DocumentNode.SelectNodes("//item")?.Count ?? 0);
|
||||
}
|
||||
|
||||
|
||||
private async Task<Uri> MakeModList()
|
||||
{
|
||||
var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!");
|
||||
var test_archive_path = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
|
||||
await test_archive_path.WriteAllBytesAsync(archive_data);
|
||||
|
||||
|
||||
|
||||
ModListData = new ModList();
|
||||
ModListData.Archives.Add(
|
||||
new Archive(new HTTPDownloader.State(MakeURL("test_archive.txt")))
|
||||
{
|
||||
Hash = await test_archive_path.FileHashAsync(),
|
||||
Name = "test_archive",
|
||||
Size = test_archive_path.Size,
|
||||
});
|
||||
|
||||
var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder);
|
||||
|
||||
await using (var fs = modListPath.Create())
|
||||
{
|
||||
using var za = new ZipArchive(fs, ZipArchiveMode.Create);
|
||||
var entry = za.CreateEntry("modlist");
|
||||
await using var es = entry.Open();
|
||||
ModListData.ToJson(es);
|
||||
}
|
||||
|
||||
ModListMetaData = new List<ModlistMetadata>
|
||||
{
|
||||
new ModlistMetadata
|
||||
{
|
||||
Official = false,
|
||||
Author = "Test Suite",
|
||||
Description = "A test",
|
||||
DownloadMetadata = new DownloadMetadata
|
||||
{
|
||||
Hash = await modListPath.FileHashAsync(),
|
||||
Size = modListPath.Size
|
||||
},
|
||||
Links = new ModlistMetadata.LinksObject
|
||||
{
|
||||
MachineURL = "test_list",
|
||||
Download = MakeURL("test_modlist.wabbajack")
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder);
|
||||
|
||||
ModListMetaData.ToJson(metadataPath);
|
||||
|
||||
return new Uri(MakeURL("test_mod_list_metadata.json"));
|
||||
}
|
||||
|
||||
public ModList ModListData { get; set; }
|
||||
|
||||
public List<ModlistMetadata> ModListMetaData { get; set; }
|
||||
}
|
||||
}
|
||||
@ -1,122 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using Dapper;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
using Wabbajack.VirtualFileSystem;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
using Xunit.Priority;
|
||||
|
||||
namespace Wabbajack.BuildServer.Test
|
||||
{
|
||||
public class ModlistUpdater : ABuildServerSystemTest
|
||||
{
|
||||
public ModlistUpdater(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
|
||||
{
|
||||
}
|
||||
|
||||
[Fact, Priority(0)]
|
||||
public async Task CanIndexAndUpdateFiles()
|
||||
{
|
||||
var sql = Fixture.GetService<SqlService>();
|
||||
var modId = long.MaxValue >> 1;
|
||||
var oldFileId = long.MaxValue >> 2;
|
||||
var newFileId = (long.MaxValue >> 2) + 1;
|
||||
|
||||
var oldFileData = RandomData();
|
||||
var newFileData = RandomData();
|
||||
var oldDataHash = oldFileData.xxHash();
|
||||
var newDataHash = newFileData.xxHash();
|
||||
|
||||
await "old_file_data.random".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(oldFileData);
|
||||
await "new_file_data.random".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(newFileData);
|
||||
|
||||
await sql.EnqueueJob(new Job
|
||||
{
|
||||
Payload = new IndexJob
|
||||
{
|
||||
Archive = new Archive(new HTTPDownloader.State(MakeURL("old_file_data.random")))
|
||||
{
|
||||
Name = "Oldfile",
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await sql.EnqueueJob(new Job
|
||||
{
|
||||
Payload = new IndexJob
|
||||
{
|
||||
Archive = new Archive(new HTTPDownloader.State(MakeURL("new_file_data.random")))
|
||||
{
|
||||
Name = "Newfile",
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await RunAllJobs();
|
||||
|
||||
Assert.True(await sql.HaveIndexdFile(oldDataHash));
|
||||
Assert.True(await sql.HaveIndexdFile(newDataHash));
|
||||
|
||||
var settings = Fixture.GetService<AppSettings>();
|
||||
Assert.Equal($"Oldfile_{oldDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(oldDataHash));
|
||||
Assert.Equal($"Newfile_{newDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(newDataHash));
|
||||
|
||||
Utils.Log($"Download Updating {oldDataHash} -> {newDataHash}");
|
||||
await using var conn = await sql.Open();
|
||||
|
||||
await conn.ExecuteAsync("DELETE FROM dbo.DownloadStates WHERE Hash in (@OldHash, @NewHash);",
|
||||
new {OldHash = (long)oldDataHash, NewHash = (long)newDataHash});
|
||||
|
||||
await sql.AddDownloadState(oldDataHash, new NexusDownloader.State
|
||||
{
|
||||
Game = Game.Oblivion,
|
||||
ModID = modId,
|
||||
FileID = oldFileId
|
||||
});
|
||||
|
||||
await sql.AddDownloadState(newDataHash, new NexusDownloader.State
|
||||
{
|
||||
Game = Game.Oblivion,
|
||||
ModID = modId,
|
||||
FileID = newFileId
|
||||
});
|
||||
|
||||
Assert.NotNull(await sql.GetNexusStateByHash(oldDataHash));
|
||||
Assert.NotNull(await sql.GetNexusStateByHash(newDataHash));
|
||||
|
||||
// No nexus info, so no upgrade
|
||||
var noUpgrade = await ClientAPI.GetModUpgrade(oldDataHash);
|
||||
Assert.Null(noUpgrade);
|
||||
|
||||
// Add Nexus info
|
||||
await sql.AddNexusModFiles(Game.Oblivion, modId, DateTime.Now,
|
||||
new NexusApiClient.GetModFilesResponse
|
||||
{
|
||||
files = new List<NexusFileInfo>
|
||||
{
|
||||
new NexusFileInfo {category_name = "MAIN", file_id = newFileId, file_name = "New File"},
|
||||
new NexusFileInfo {category_name = null, file_id = oldFileId, file_name = "Old File"}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
var enqueuedUpgrade = await ClientAPI.GetModUpgrade(oldDataHash);
|
||||
|
||||
// Not Null because upgrade was enqueued
|
||||
Assert.NotNull(enqueuedUpgrade);
|
||||
|
||||
await RunAllJobs();
|
||||
|
||||
Assert.True($"{oldDataHash.ToHex()}_{newDataHash.ToHex()}".RelativeTo(Fixture.ServerUpdatesFolder).IsFile);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,94 +0,0 @@
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.FileUploader;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
using Xunit.Priority;
|
||||
|
||||
namespace Wabbajack.BuildServer.Test
|
||||
{
|
||||
[Collection("ServerTests")]
|
||||
public class UploadedFilesTest : ABuildServerSystemTest
|
||||
{
|
||||
[Fact, Priority(1)]
|
||||
public async Task CanIngestMongoDBExports()
|
||||
{
|
||||
var data = await @"sql\uploaded_files_ingest.json".RelativeTo(AbsolutePath.EntryPoint).ReadAllTextAsync();
|
||||
data = data.Replace("<testuser>", Fixture.User);
|
||||
await Fixture.ServerTempFolder.Combine("uploaded_files_ingest.json").WriteAllTextAsync(data);
|
||||
using var response = await _authedClient.GetAsync(MakeURL("ingest/uploaded_files/uploaded_files_ingest.json"));
|
||||
var result = await response.Content.ReadAsStringAsync();
|
||||
Utils.Log("Loaded: " + result);
|
||||
|
||||
|
||||
Assert.Equal("4", result);
|
||||
}
|
||||
|
||||
[Fact, Priority(1)]
|
||||
public async Task CanListMyUploadedFiles()
|
||||
{
|
||||
var result = (await _authedClient.GetStringAsync(MakeURL("uploaded_files/list"))).FromJsonString<string[]>();
|
||||
Utils.Log("Loaded: " + result);
|
||||
|
||||
|
||||
Assert.True(result.Length >= 2, result.Length.ToString());
|
||||
Assert.Contains("file1-90db7c47-a8ae-4a62-9c2e-b7d357a16665.zip", result);
|
||||
Assert.Contains("file2-63f8f868-0f4d-4997-922b-ee952984973a.zip", result);
|
||||
// These are from other users
|
||||
Assert.DoesNotContain("file2-1f18f301-67eb-46c9-928a-088f6666bf61.zip", result);
|
||||
Assert.DoesNotContain("file3-17b3e918-8409-48e6-b7ff-6af858bfd1ba.zip", result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanUploadFilesUsingClientApi()
|
||||
{
|
||||
using (var file = new TempFile())
|
||||
{
|
||||
var data = new byte[1024 * 1024 * 8 * 4];
|
||||
await using (var fs = file.Path.Create())
|
||||
{
|
||||
await fs.WriteAsync(data);
|
||||
}
|
||||
|
||||
Utils.Log($"Uploading {file.Path.Size.ToFileSizeString()} file");
|
||||
var result = await AuthorAPI.UploadFile(file.Path,
|
||||
progress => Utils.Log($"Uploading : {progress * 100}%"), Fixture.APIKey);
|
||||
|
||||
Utils.Log($"Result {result}");
|
||||
Assert.StartsWith("https://wabbajackpush.b-cdn.net/" +(string)file.Path.FileNameWithoutExtension, result);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanDeleteFilesUsingClientApi()
|
||||
{
|
||||
using (var file = new TempFile())
|
||||
{
|
||||
var data = new byte[1024];
|
||||
await using (var fs = file.Path.Create())
|
||||
{
|
||||
await fs.WriteAsync(data);
|
||||
}
|
||||
|
||||
Utils.Log($"Uploading {file.Path.Size.ToFileSizeString()} file");
|
||||
var result = await AuthorAPI.UploadFile(file.Path,
|
||||
progress => Utils.Log($"Uploading : {progress * 100}%"), Fixture.APIKey);
|
||||
|
||||
Utils.Log($"Delete {result}");
|
||||
await AuthorAPI.DeleteFile((string)((RelativePath)new Uri(result).AbsolutePath).FileName);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public UploadedFilesTest(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,63 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
|
||||
<PackageReference Include="xunit" Version="2.4.1" />
|
||||
<PackageReference Include="Xunit.Priority" Version="1.1.6" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
|
||||
<PackageReference Include="coverlet.collector" Version="1.2.1" />
|
||||
<PackageReference Include="System.Data.SqlClient" Version="4.8.1" />
|
||||
<PackageReference Include="XunitContext" Version="1.9.3" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<EmbeddedResource Include="sql\wabbajack_db.sql" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Wabbajack.BuildServer\Wabbajack.BuildServer.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Update="sql\nexus_export.json">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="sql\DownloadStates\097ad17ef4b9f5b7_68d29ad947f2bf80d887407b6e8794c37ac08f3728eca95c8774184c56df3800.ini">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="sql\DownloadStates\96fb53c3dc6397d2_9ff1b17c4fafdb70ef51390a1706d8aec66cdc09ca950f8a9daa1570db9b1c94.ini">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="sql\DownloadStates\97a6d27b7becba19_6ba040ef3bc1775bb41f97427fb830a907b9b74ccbe056624c537c8e5f214529.ini">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="sql\DownloadStates\e5223a83ab49e25c_1be0991cec07ee378b0891ce576cb75b3a7adc56232945772961e3a9428f17e5.ini">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="sql\DownloadStates\e5409bdeb0e77bd3_985c554f1bf98c1569fcbb2926f38e61c86e4ce6a416e6cb6cf020913f24d802.ini">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="sql\NotifyStates\00e8bbbf591f61a3_6a5eb07c4b3c03fde38c9223a94a38c9076ef8fc8167f77c875c58db8f2aefd2.ini">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="xunit.runner.json">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="sql\uploaded_files_ingest.json">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Folder Include="sql\DownloadStates" />
|
||||
<Folder Include="sql\NexusCache" />
|
||||
<Folder Include="sql\NotifyStates" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@ -1,4 +0,0 @@
|
||||
[General]
|
||||
gameName=Skyrim
|
||||
modID=58118
|
||||
fileID=1000126774
|
||||
@ -1,4 +0,0 @@
|
||||
[General]
|
||||
gameName=fallout4
|
||||
modID=34297
|
||||
fileID=141870
|
||||
@ -1,4 +0,0 @@
|
||||
[General]
|
||||
gameName=SkyrimSE
|
||||
modID=23774
|
||||
fileID=98580
|
||||
@ -1,4 +0,0 @@
|
||||
[General]
|
||||
gameName=skyrimspecialedition
|
||||
modID=13675
|
||||
fileID=121575
|
||||
@ -1,4 +0,0 @@
|
||||
[General]
|
||||
gameName=fallout4
|
||||
modID=33578
|
||||
fileID=137486
|
||||
@ -1,4 +0,0 @@
|
||||
[General]
|
||||
gameName=skyrim
|
||||
modID=81066
|
||||
fileID=1000284635
|
||||
File diff suppressed because one or more lines are too long
@ -1,4 +0,0 @@
|
||||
{"_id":"90db7c47-a8ae-4a62-9c2e-b7d357a16665","Name":"file1.zip","Size":{"$numberInt":"56946233"},"Hash":"8t3PgB/J/Ps=","Uploader":"<testuser>","UploadDate":{"$date":{"$numberLong":"1579472623183"}}}
|
||||
{"_id":"63f8f868-0f4d-4997-922b-ee952984973a","Name":"file2.zip","Size":{"$numberInt":"558782831"},"Hash":"XYl4fIcAUWo=","Uploader":"<testuser>","UploadDate":{"$date":{"$numberLong":"1579650208226"}},"CDNName":"wabbajackpush"}
|
||||
{"_id":"1f18f301-67eb-46c9-928a-088f6666bf61","Name":"file2.zip","Size":{"$numberInt":"397558132"},"Hash":"uhXJw1MTdJI=","Uploader":"other","UploadDate":{"$date":{"$numberLong":"1579661152181"}},"CDNName":"wabbajackpush"}
|
||||
{"_id":"17b3e918-8409-48e6-b7ff-6af858bfd1ba","Name":"file3.zip","Size":{"$numberInt":"633419999"},"Hash":"yyDD50X2kNM=","Uploader":"other","UploadDate":{"$date":{"$numberLong":"1579662459461"}},"CDNName":"wabbajackpush"}
|
||||
@ -1,615 +0,0 @@
|
||||
USE [master]
|
||||
GO
|
||||
/****** Object: Database [wabbajack_prod] Script Date: 3/28/2020 4:58:58 PM ******/
|
||||
CREATE DATABASE [wabbajack_prod]
|
||||
CONTAINMENT = NONE
|
||||
WITH CATALOG_COLLATION = DATABASE_DEFAULT
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET COMPATIBILITY_LEVEL = 150
|
||||
GO
|
||||
IF (1 = FULLTEXTSERVICEPROPERTY('IsFullTextInstalled'))
|
||||
begin
|
||||
EXEC [wabbajack_prod].[dbo].[sp_fulltext_database] @action = 'enable'
|
||||
end
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET ANSI_NULL_DEFAULT OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET ANSI_NULLS OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET ANSI_PADDING OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET ANSI_WARNINGS OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET ARITHABORT OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET AUTO_CLOSE OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET AUTO_SHRINK OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET AUTO_UPDATE_STATISTICS ON
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET CURSOR_CLOSE_ON_COMMIT OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET CURSOR_DEFAULT GLOBAL
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET CONCAT_NULL_YIELDS_NULL OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET NUMERIC_ROUNDABORT OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET QUOTED_IDENTIFIER OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET RECURSIVE_TRIGGERS OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET DISABLE_BROKER
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET AUTO_UPDATE_STATISTICS_ASYNC OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET DATE_CORRELATION_OPTIMIZATION OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET TRUSTWORTHY OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET ALLOW_SNAPSHOT_ISOLATION OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET PARAMETERIZATION SIMPLE
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET READ_COMMITTED_SNAPSHOT OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET HONOR_BROKER_PRIORITY OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET RECOVERY FULL
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET MULTI_USER
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET PAGE_VERIFY CHECKSUM
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET DB_CHAINING OFF
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET FILESTREAM( NON_TRANSACTED_ACCESS = OFF )
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET TARGET_RECOVERY_TIME = 60 SECONDS
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET DELAYED_DURABILITY = DISABLED
|
||||
GO
|
||||
EXEC sys.sp_db_vardecimal_storage_format N'wabbajack_prod', N'ON'
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET QUERY_STORE = OFF
|
||||
GO
|
||||
USE [wabbajack_prod]
|
||||
GO
|
||||
/****** Object: Schema [test] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
CREATE SCHEMA [test]
|
||||
GO
|
||||
/****** Object: UserDefinedTableType [dbo].[ArchiveContentType] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
CREATE TYPE [dbo].[ArchiveContentType] AS TABLE(
|
||||
[Parent] [bigint] NOT NULL,
|
||||
[Child] [bigint] NOT NULL,
|
||||
[Path] [nvarchar](max) NOT NULL
|
||||
)
|
||||
GO
|
||||
/****** Object: UserDefinedTableType [dbo].[IndexedFileType] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
CREATE TYPE [dbo].[IndexedFileType] AS TABLE(
|
||||
[Hash] [bigint] NOT NULL,
|
||||
[Sha256] [binary](32) NOT NULL,
|
||||
[Sha1] [binary](20) NOT NULL,
|
||||
[Md5] [binary](16) NOT NULL,
|
||||
[Crc32] [int] NOT NULL,
|
||||
[Size] [bigint] NOT NULL
|
||||
)
|
||||
GO
|
||||
/****** Object: UserDefinedFunction [dbo].[Base64ToLong] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
SET ANSI_NULLS ON
|
||||
GO
|
||||
SET QUOTED_IDENTIFIER ON
|
||||
GO
|
||||
-- =============================================
|
||||
-- Author: <Author,,Name>
|
||||
-- Create date: <Create Date, ,>
|
||||
-- Description: <Description, ,>
|
||||
-- =============================================
|
||||
CREATE FUNCTION [dbo].[Base64ToLong]
|
||||
(
|
||||
-- Add the parameters for the function here
|
||||
@Input varchar
|
||||
)
|
||||
RETURNS bigint
|
||||
AS
|
||||
BEGIN
|
||||
-- Declare the return variable here
|
||||
DECLARE @ResultVar bigint
|
||||
|
||||
-- Add the T-SQL statements to compute the return value here
|
||||
SELECT @ResultVar = CAST('string' as varbinary(max)) FOR XML PATH(''), BINARY BASE64
|
||||
|
||||
-- Return the result of the function
|
||||
RETURN @ResultVar
|
||||
|
||||
END
|
||||
GO
|
||||
/****** Object: UserDefinedFunction [dbo].[MaxMetricDate] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
SET ANSI_NULLS ON
|
||||
GO
|
||||
SET QUOTED_IDENTIFIER ON
|
||||
GO
|
||||
|
||||
-- =============================================
|
||||
-- Author: <Author,,Name>
|
||||
-- Create date: <Create Date, ,>
|
||||
-- Description: <Description, ,>
|
||||
-- =============================================
|
||||
CREATE FUNCTION [dbo].[MaxMetricDate]
|
||||
(
|
||||
)
|
||||
RETURNS date
|
||||
AS
|
||||
BEGIN
|
||||
-- Declare the return variable here
|
||||
DECLARE @Result date
|
||||
|
||||
-- Add the T-SQL statements to compute the return value here
|
||||
SELECT @Result = max(Timestamp) from dbo.Metrics where MetricsKey is not null
|
||||
|
||||
-- Return the result of the function
|
||||
RETURN @Result
|
||||
|
||||
END
|
||||
GO
|
||||
/****** Object: UserDefinedFunction [dbo].[MinMetricDate] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
SET ANSI_NULLS ON
|
||||
GO
|
||||
SET QUOTED_IDENTIFIER ON
|
||||
GO
|
||||
|
||||
-- =============================================
|
||||
-- Author: <Author,,Name>
|
||||
-- Create date: <Create Date, ,>
|
||||
-- Description: <Description, ,>
|
||||
-- =============================================
|
||||
CREATE FUNCTION [dbo].[MinMetricDate]
|
||||
(
|
||||
)
|
||||
RETURNS date
|
||||
AS
|
||||
BEGIN
|
||||
-- Declare the return variable here
|
||||
DECLARE @Result date
|
||||
|
||||
-- Add the T-SQL statements to compute the return value here
|
||||
SELECT @Result = min(Timestamp) from dbo.Metrics WHERE MetricsKey is not null
|
||||
|
||||
-- Return the result of the function
|
||||
RETURN @Result
|
||||
|
||||
END
|
||||
GO
|
||||
/****** Object: Table [dbo].[IndexedFile] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
SET ANSI_NULLS ON
|
||||
GO
|
||||
SET QUOTED_IDENTIFIER ON
|
||||
GO
|
||||
CREATE TABLE [dbo].[IndexedFile](
|
||||
[Hash] [bigint] NOT NULL,
|
||||
[Sha256] [binary](32) NOT NULL,
|
||||
[Sha1] [binary](20) NOT NULL,
|
||||
[Md5] [binary](16) NOT NULL,
|
||||
[Crc32] [int] NOT NULL,
|
||||
[Size] [bigint] NOT NULL,
|
||||
CONSTRAINT [PK_IndexedFile] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[Hash] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
) ON [PRIMARY]
|
||||
GO
|
||||
|
||||
/****** Object: Table [dbo].[Jobs] ******/
|
||||
CREATE TABLE [dbo].[Jobs](
|
||||
[Id] [bigint] IDENTITY(1,1) NOT NULL,
|
||||
[Priority] [int] NOT NULL,
|
||||
[PrimaryKeyString] [nvarchar](max) NULL,
|
||||
[Started] [datetime] NULL,
|
||||
[Ended] [datetime] NULL,
|
||||
[Created] [datetime] NOT NULL,
|
||||
[Success] [tinyint] NULL,
|
||||
[ResultContent] [nvarchar](max) NULL,
|
||||
[Payload] [nvarchar](max) NULL,
|
||||
[OnSuccess] [nvarchar](max) NULL,
|
||||
[RunBy] [uniqueidentifier] NULL
|
||||
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
|
||||
GO
|
||||
|
||||
|
||||
|
||||
/****** Object: Table [dbo].[ArchiveContent] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
SET ANSI_NULLS ON
|
||||
GO
|
||||
SET QUOTED_IDENTIFIER ON
|
||||
GO
|
||||
CREATE TABLE [dbo].[ArchiveContent](
|
||||
[Parent] [bigint] NOT NULL,
|
||||
[Child] [bigint] NOT NULL,
|
||||
[Path] [nvarchar](max) NULL,
|
||||
[PathHash] AS (CONVERT([binary](32),hashbytes('SHA2_256',[Path]))) PERSISTED NOT NULL
|
||||
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
|
||||
GO
|
||||
|
||||
CREATE STATISTICS [Child_Parent_Stat] ON [dbo].[ArchiveContent]([Child], [Parent])
|
||||
GO
|
||||
CREATE CLUSTERED INDEX [Child_Parent_IDX] ON [dbo].[ArchiveContent]
|
||||
(
|
||||
[Child] ASC,
|
||||
[Parent] ASC
|
||||
)WITH (SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF) ON [PRIMARY]
|
||||
GO
|
||||
/****** Object: Table [dbo].[AllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
SET ANSI_NULLS ON
|
||||
GO
|
||||
SET QUOTED_IDENTIFIER ON
|
||||
GO
|
||||
CREATE TABLE [dbo].[AllFilesInArchive](
|
||||
[TopParent] [bigint] NOT NULL,
|
||||
[Child] [bigint] NOT NULL,
|
||||
CONSTRAINT [PK_AllFilesInArchive] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[TopParent] ASC,
|
||||
[Child] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
) ON [PRIMARY]
|
||||
GO
|
||||
/****** Object: View [dbo].[AllArchiveContent] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
SET ANSI_NULLS ON
|
||||
GO
|
||||
SET QUOTED_IDENTIFIER ON
|
||||
GO
|
||||
|
||||
|
||||
CREATE VIEW [dbo].[AllArchiveContent]
|
||||
WITH SCHEMABINDING
|
||||
AS
|
||||
SELECT af.TopParent, ac.Parent, af.Child, ac.Path, idx.Size
|
||||
FROM
|
||||
dbo.AllFilesInArchive af
|
||||
LEFT JOIN dbo.ArchiveContent ac on af.Child = ac.Child
|
||||
LEFT JOIN dbo.IndexedFile idx on af.Child = idx.Hash
|
||||
GO
|
||||
|
||||
/****** Object: Table [dbo].[NexusFileInfos] Script Date: 4/1/2020 2:41:00 PM ******/
|
||||
CREATE TABLE [dbo].[NexusFileInfos](
|
||||
[Game] [int] NOT NULL,
|
||||
[ModId] [bigint] NOT NULL,
|
||||
[FileId] [bigint] NOT NULL,
|
||||
[LastChecked] [datetime] NOT NULL,
|
||||
[Data] [nvarchar](max) NOT NULL,
|
||||
CONSTRAINT [PK_NexusFileInfos] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[Game] ASC,
|
||||
[ModId] ASC,
|
||||
[FileId] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
|
||||
GO
|
||||
|
||||
/****** Object: Table [dbo].[NexusModFiles] Script Date: 4/1/2020 2:41:04 PM ******/
|
||||
|
||||
CREATE TABLE [dbo].[NexusModFiles](
|
||||
[Game] [int] NOT NULL,
|
||||
[ModId] [bigint] NOT NULL,
|
||||
[LastChecked] [datetime] NOT NULL,
|
||||
[Data] [nvarchar](max) NOT NULL,
|
||||
CONSTRAINT [PK_NexusModFiles] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[Game] ASC,
|
||||
[ModId] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
|
||||
GO
|
||||
|
||||
/****** Object: Table [dbo].[NexusModInfos] Script Date: 4/1/2020 2:41:07 PM ******/
|
||||
|
||||
CREATE TABLE [dbo].[NexusModInfos](
|
||||
[Game] [int] NOT NULL,
|
||||
[ModId] [bigint] NOT NULL,
|
||||
[LastChecked] [datetime] NOT NULL,
|
||||
[Data] [nvarchar](max) NOT NULL,
|
||||
CONSTRAINT [PK_NexusModInfos] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[Game] ASC,
|
||||
[ModId] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
|
||||
GO
|
||||
|
||||
/****** Object: Table [dbo].[ModLists] Script Date: 4/2/2020 3:59:19 PM ******/
|
||||
CREATE TABLE [dbo].[ModLists](
|
||||
[MachineURL] [nvarchar](50) NOT NULL,
|
||||
[Hash] [bigint] NOT NULL,
|
||||
[Metadata] [nvarchar](max) NOT NULL,
|
||||
[Modlist] [nvarchar](max) NOT NULL,
|
||||
CONSTRAINT [PK_ModLists] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[MachineURL] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
|
||||
GO
|
||||
|
||||
/****** Object: Table [dbo].[ModListArchive] Script Date: 4/11/2020 10:33:20 AM ******/
|
||||
|
||||
CREATE TABLE [dbo].[ModListArchives](
|
||||
[MachineUrl] [nvarchar](50) NOT NULL,
|
||||
[Hash] [bigint] NOT NULL,
|
||||
[PrimaryKeyString] [nvarchar](max) NOT NULL,
|
||||
[Size] [bigint] NOT NULL,
|
||||
[State] [nvarchar](max) NOT NULL,
|
||||
CONSTRAINT [PK_ModListArchive] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[MachineUrl] ASC,
|
||||
[Hash] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
) ON [PRIMARY]
|
||||
GO
|
||||
|
||||
/****** Object: Table [dbo].[ModListArchiveStatus] Script Date: 4/11/2020 9:44:25 PM ******/
|
||||
|
||||
CREATE TABLE [dbo].[ModListArchiveStatus](
|
||||
[PrimaryKeyStringHash] [binary](32) NOT NULL,
|
||||
[Hash] [bigint] NOT NULL,
|
||||
[PrimaryKeyString] [nvarchar](max) NOT NULL,
|
||||
[IsValid] [tinyint] NOT NULL,
|
||||
CONSTRAINT [PK_ModListArchiveStatus] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[PrimaryKeyStringHash] ASC,
|
||||
[Hash] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
|
||||
GO
|
||||
|
||||
/****** Object: Table [dbo].[ArchivePatches] Script Date: 4/13/2020 9:39:25 PM ******/
|
||||
CREATE TABLE [dbo].[ArchivePatches](
|
||||
[SrcPrimaryKeyStringHash] [binary](32) NOT NULL,
|
||||
[SrcPrimaryKeyString] [nvarchar](max) NOT NULL,
|
||||
[SrcHash] [bigint] NOT NULL,
|
||||
[DestPrimaryKeyStringHash] [binary](32) NOT NULL,
|
||||
[DestPrimaryKeyString] [nvarchar](max) NOT NULL,
|
||||
[DestHash] [bigint] NOT NULL,
|
||||
[SrcState] [nvarchar](max) NOT NULL,
|
||||
[DestState] [nvarchar](max) NOT NULL,
|
||||
[SrcDownload] [nvarchar](max) NULL,
|
||||
[DestDownload] [nvarchar](max) NULL,
|
||||
[CDNPath] [nvarchar](max) NULL,
|
||||
CONSTRAINT [PK_ArchivePatches] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[SrcPrimaryKeyStringHash] ASC,
|
||||
[SrcHash] ASC,
|
||||
[DestPrimaryKeyStringHash] ASC,
|
||||
[DestHash] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
|
||||
GO
|
||||
|
||||
|
||||
/****** Object: Table [dbo].[Metrics] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
SET ANSI_NULLS ON
|
||||
GO
|
||||
SET QUOTED_IDENTIFIER ON
|
||||
GO
|
||||
CREATE TABLE [dbo].[Metrics](
|
||||
[Id] [bigint] IDENTITY(1,1) NOT NULL,
|
||||
[Timestamp] [datetime] NOT NULL,
|
||||
[Action] [nvarchar](64) NOT NULL,
|
||||
[Subject] [nvarchar](max) NOT NULL,
|
||||
[MetricsKey] [nvarchar](64) NULL,
|
||||
[GroupingSubject] AS (substring([Subject],(0),case when patindex('%[0-9].%',[Subject])=(0) then len([Subject])+(1) else patindex('%[0-9].%',[Subject]) end)),
|
||||
CONSTRAINT [PK_Metrics] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[Id] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
|
||||
GO
|
||||
/****** Uploaded Files [UploadedFiles] *************/
|
||||
|
||||
CREATE TABLE [dbo].[UploadedFiles](
|
||||
[Id] [uniqueidentifier] NOT NULL,
|
||||
[Name] [nvarchar](max) NOT NULL,
|
||||
[Size] [bigint] NOT NULL,
|
||||
[UploadedBy] [nvarchar](40) NOT NULL,
|
||||
[Hash] [bigint] NOT NULL,
|
||||
[UploadDate] [datetime] NOT NULL,
|
||||
[CDNName] [nvarchar](max) NULL,
|
||||
CONSTRAINT [PK_UploadedFiles] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[Id] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
|
||||
GO
|
||||
/****** API Keys [ApiKeys] ********/
|
||||
CREATE TABLE [dbo].[ApiKeys](
|
||||
[APIKey] [nvarchar](260) NOT NULL,
|
||||
[Owner] [nvarchar](40) NOT NULL,
|
||||
CONSTRAINT [PK_ApiKeys] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[APIKey] ASC,
|
||||
[Owner] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
) ON [PRIMARY]
|
||||
GO
|
||||
CREATE UNIQUE NONCLUSTERED INDEX [ByAPIKey] ON [dbo].[ApiKeys]
|
||||
(
|
||||
[APIKey] ASC
|
||||
)
|
||||
INCLUDE([Owner]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
GO
|
||||
|
||||
/****** Object: Table [dbo].[DownloadStates] Script Date: 3/31/2020 6:22:47 AM ******/
|
||||
|
||||
CREATE TABLE [dbo].[DownloadStates](
|
||||
[Id] [binary](32) NOT NULL,
|
||||
[Hash] [bigint] NOT NULL,
|
||||
[PrimaryKey] [nvarchar](max) NOT NULL,
|
||||
[IniState] [nvarchar](max) NOT NULL,
|
||||
[JsonState] [nvarchar](max) NOT NULL,
|
||||
CONSTRAINT [PK_DownloadStates] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[Id] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
|
||||
GO
|
||||
|
||||
|
||||
CREATE NONCLUSTERED INDEX [ByHash] ON [dbo].[DownloadStates]
|
||||
(
|
||||
[Hash] ASC
|
||||
)
|
||||
INCLUDE([IniState]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
GO
|
||||
|
||||
/****** Object: View [dbo].[GameFiles] Script Date: 4/30/2020 4:23:25 PM ******/
|
||||
|
||||
CREATE VIEW [dbo].[GameFiles]
|
||||
WITH SCHEMABINDING
|
||||
AS
|
||||
|
||||
Select
|
||||
Id,
|
||||
CONVERT(NVARCHAR(20), JSON_VALUE(JsonState,'$.GameVersion')) as GameVersion,
|
||||
CONVERT(NVARCHAR(32),JSON_VALUE(JsonState,'$.Game')) as Game,
|
||||
JSON_VALUE(JsonState,'$.GameFile') as Path,
|
||||
Hash as Hash
|
||||
FROM dbo.DownloadStates
|
||||
WHERE PrimaryKey like 'GameFileSourceDownloader+State|%'
|
||||
AND JSON_VALUE(JsonState,'$.GameFile') NOT LIKE '%.xxhash'
|
||||
GO
|
||||
|
||||
CREATE UNIQUE CLUSTERED INDEX [ByGameAndVersion] ON [dbo].[GameFiles]
|
||||
(
|
||||
[Game] ASC,
|
||||
[GameVersion] ASC,
|
||||
[Id] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
GO
|
||||
|
||||
|
||||
|
||||
/****** Object: Index [IX_Child] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
CREATE NONCLUSTERED INDEX [IX_Child] ON [dbo].[AllFilesInArchive]
|
||||
(
|
||||
[Child] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
GO
|
||||
/****** Object: Index [IX_ArchiveContent_Child] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
CREATE NONCLUSTERED INDEX [IX_ArchiveContent_Child] ON [dbo].[ArchiveContent]
|
||||
(
|
||||
[Child] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
GO
|
||||
SET ARITHABORT ON
|
||||
SET CONCAT_NULL_YIELDS_NULL ON
|
||||
SET QUOTED_IDENTIFIER ON
|
||||
SET ANSI_NULLS ON
|
||||
SET ANSI_PADDING ON
|
||||
SET ANSI_WARNINGS ON
|
||||
SET NUMERIC_ROUNDABORT OFF
|
||||
GO
|
||||
/****** Object: Index [PK_ArchiveContent] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
CREATE UNIQUE NONCLUSTERED INDEX [PK_ArchiveContent] ON [dbo].[ArchiveContent]
|
||||
(
|
||||
[Parent] ASC,
|
||||
[PathHash] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
GO
|
||||
SET ANSI_PADDING ON
|
||||
GO
|
||||
/****** Object: Index [IX_IndexedFile_By_SHA256] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
CREATE UNIQUE NONCLUSTERED INDEX [IX_IndexedFile_By_SHA256] ON [dbo].[IndexedFile]
|
||||
(
|
||||
[Sha256] ASC
|
||||
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
|
||||
GO
|
||||
/****** Object: StoredProcedure [dbo].[MergeAllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
SET ANSI_NULLS ON
|
||||
GO
|
||||
SET QUOTED_IDENTIFIER ON
|
||||
GO
|
||||
-- =============================================
|
||||
-- Author: <Author,,Name>
|
||||
-- Create date: <Create Date,,>
|
||||
-- Description: <Description,,>
|
||||
-- =============================================
|
||||
CREATE PROCEDURE [dbo].[MergeAllFilesInArchive]
|
||||
AS
|
||||
BEGIN
|
||||
-- SET NOCOUNT ON added to prevent extra result sets from
|
||||
-- interfering with SELECT statements.
|
||||
SET NOCOUNT ON;
|
||||
|
||||
MERGE dbo.AllFilesInArchive t USING (
|
||||
SELECT DISTINCT TopParent, unpvt.Child
|
||||
FROM
|
||||
(SELECT a3.Parent AS P3, a2.Parent as P2, a1.Parent P1, a0.Parent P0, a0.Parent as Parent, a0.Child FROM
|
||||
dbo.ArChiveContent a0
|
||||
LEFT JOIN dbo.ArChiveContent a1 ON a0.Parent = a1.Child
|
||||
LEFT JOIN dbo.ArChiveContent a2 ON a1.Parent = a2.Child
|
||||
LEFT JOIN dbo.ArChiveContent a3 ON a2.Parent = a3.Child) p
|
||||
UNPIVOT
|
||||
(TopParent For C IN (p.P3, p.P2, p.P1, p.P0)) as unpvt
|
||||
LEFT JOIN dbo.IndexedFile idf on unpvt.Child = idf.Hash
|
||||
WHERE TopParent is not null) s
|
||||
ON t.TopParent = s.TopParent AND t.Child = s.Child
|
||||
WHEN NOT MATCHED
|
||||
THEN INSERT (TopParent, Child) VALUES (s.TopParent, s.Child);
|
||||
END
|
||||
GO
|
||||
/****** Object: StoredProcedure [dbo].[MergeIndexedFiles] Script Date: 3/28/2020 4:58:59 PM ******/
|
||||
SET ANSI_NULLS ON
|
||||
GO
|
||||
SET QUOTED_IDENTIFIER ON
|
||||
GO
|
||||
-- Description: <Description,,>
|
||||
-- =============================================
|
||||
CREATE PROCEDURE [dbo].[MergeIndexedFiles]
|
||||
-- Add the parameters for the stored procedure here
|
||||
@Files dbo.IndexedFileType READONLY,
|
||||
@Contents dbo.ArchiveContentType READONLY
|
||||
AS
|
||||
BEGIN
|
||||
-- SET NOCOUNT ON added to prevent extra result sets from
|
||||
-- interfering with SELECT statements.
|
||||
SET NOCOUNT ON;
|
||||
BEGIN TRANSACTION;
|
||||
|
||||
MERGE dbo.IndexedFile AS TARGET
|
||||
USING (SELECT DISTINCT * FROM @Files) as SOURCE
|
||||
ON (TARGET.Hash = SOURCE.HASH)
|
||||
WHEN NOT MATCHED BY TARGET
|
||||
THEN INSERT (Hash, Sha256, Sha1, Md5, Crc32, Size)
|
||||
VALUES (Source.Hash, Source.Sha256, Source.Sha1, Source.Md5, Source.Crc32, Source.Size);
|
||||
|
||||
MERGE dbo.ArchiveContent AS TARGET
|
||||
USING (SELECT DISTINCT * FROM @Contents) as SOURCE
|
||||
ON (TARGET.Parent = SOURCE.Parent AND TARGET.PathHash = CAST(HASHBYTES('SHA2_256', SOURCE.Path) as binary(32)))
|
||||
WHEN NOT MATCHED
|
||||
THEN INSERT (Parent, Child, Path)
|
||||
VALUES (Source.Parent, Source.Child, Source.Path);
|
||||
|
||||
MERGE dbo.AllFilesInArchive t USING (
|
||||
SELECT DISTINCT TopParent, unpvt.Child
|
||||
FROM
|
||||
(SELECT a3.Parent AS P3, a2.Parent as P2, a1.Parent P1, a0.Parent P0, a0.Parent as Parent, a0.Child FROM
|
||||
dbo.ArChiveContent a0
|
||||
LEFT JOIN dbo.ArChiveContent a1 ON a0.Parent = a1.Child
|
||||
LEFT JOIN dbo.ArChiveContent a2 ON a1.Parent = a2.Child
|
||||
LEFT JOIN dbo.ArChiveContent a3 ON a2.Parent = a3.Child) p
|
||||
UNPIVOT
|
||||
(TopParent For C IN (p.P3, p.P2, p.P1, p.P0)) as unpvt
|
||||
LEFT JOIN dbo.IndexedFile idf on unpvt.Child = idf.Hash
|
||||
WHERE TopParent is not null
|
||||
AND Child in (SELECT DISTINCT Hash FROM @Files)) s
|
||||
ON t.TopParent = s.TopParent AND t.Child = s.Child
|
||||
WHEN NOT MATCHED
|
||||
THEN INSERT (TopParent, Child) VALUES (s.TopParent, s.Child);
|
||||
|
||||
COMMIT;
|
||||
|
||||
END
|
||||
GO
|
||||
USE [master]
|
||||
GO
|
||||
ALTER DATABASE [wabbajack_prod] SET READ_WRITE
|
||||
GO
|
||||
@ -1,4 +0,0 @@
|
||||
{
|
||||
"parallelizeTestCollections": false,
|
||||
"maxParallelThreads": 1
|
||||
}
|
||||
@ -1,90 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Security.Claims;
|
||||
using System.Text.Encodings.Web;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Authentication;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Newtonsoft.Json;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
|
||||
namespace Wabbajack.BuildServer
|
||||
{
|
||||
|
||||
public class ApiKeyAuthenticationOptions : AuthenticationSchemeOptions
|
||||
{
|
||||
public const string DefaultScheme = "API Key";
|
||||
public string Scheme => DefaultScheme;
|
||||
public string AuthenticationType = DefaultScheme;
|
||||
}
|
||||
|
||||
public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthenticationOptions>
|
||||
{
|
||||
private const string ProblemDetailsContentType = "application/problem+json";
|
||||
private readonly SqlService _db;
|
||||
private const string ApiKeyHeaderName = "X-Api-Key";
|
||||
|
||||
public ApiKeyAuthenticationHandler(
|
||||
IOptionsMonitor<ApiKeyAuthenticationOptions> options,
|
||||
ILoggerFactory logger,
|
||||
UrlEncoder encoder,
|
||||
ISystemClock clock,
|
||||
SqlService db) : base(options, logger, encoder, clock)
|
||||
{
|
||||
_db = db;
|
||||
}
|
||||
|
||||
protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
|
||||
{
|
||||
if (!Request.Headers.TryGetValue(ApiKeyHeaderName, out var apiKeyHeaderValues))
|
||||
{
|
||||
return AuthenticateResult.NoResult();
|
||||
}
|
||||
|
||||
var providedApiKey = apiKeyHeaderValues.FirstOrDefault();
|
||||
|
||||
if (apiKeyHeaderValues.Count == 0 || string.IsNullOrWhiteSpace(providedApiKey))
|
||||
{
|
||||
return AuthenticateResult.NoResult();
|
||||
}
|
||||
|
||||
var owner = await _db.LoginByAPIKey(providedApiKey);
|
||||
|
||||
if (owner != null)
|
||||
{
|
||||
var claims = new List<Claim> {new Claim(ClaimTypes.Name, owner)};
|
||||
|
||||
/*
|
||||
claims.AddRange(existingApiKey.Roles.Select(role => new Claim(ClaimTypes.Role, role)));
|
||||
*/
|
||||
|
||||
var identity = new ClaimsIdentity(claims, Options.AuthenticationType);
|
||||
var identities = new List<ClaimsIdentity> {identity};
|
||||
var principal = new ClaimsPrincipal(identities);
|
||||
var ticket = new AuthenticationTicket(principal, Options.Scheme);
|
||||
|
||||
return AuthenticateResult.Success(ticket);
|
||||
}
|
||||
|
||||
return AuthenticateResult.Fail("Invalid API Key provided.");
|
||||
}
|
||||
|
||||
protected override async Task HandleChallengeAsync(AuthenticationProperties properties)
|
||||
{
|
||||
Response.StatusCode = 401;
|
||||
Response.ContentType = ProblemDetailsContentType;
|
||||
await Response.WriteAsync("Unauthorized");
|
||||
}
|
||||
|
||||
protected override async Task HandleForbiddenAsync(AuthenticationProperties properties)
|
||||
{
|
||||
Response.StatusCode = 403;
|
||||
Response.ContentType = ProblemDetailsContentType;
|
||||
await Response.WriteAsync("forbidden");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,34 +0,0 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Wabbajack.Common;
|
||||
|
||||
namespace Wabbajack.BuildServer
|
||||
{
|
||||
public class AppSettings
|
||||
{
|
||||
public AppSettings(IConfiguration config)
|
||||
{
|
||||
config.Bind("WabbajackSettings", this);
|
||||
}
|
||||
|
||||
public string DownloadDir { get; set; }
|
||||
public AbsolutePath DownloadPath => (AbsolutePath)DownloadDir;
|
||||
public string ArchiveDir { get; set; }
|
||||
public AbsolutePath ArchivePath => (AbsolutePath)ArchiveDir;
|
||||
|
||||
public string TempFolder { get; set; }
|
||||
|
||||
public AbsolutePath TempPath => (AbsolutePath)TempFolder;
|
||||
|
||||
public bool JobScheduler { get; set; }
|
||||
public bool JobRunner { get; set; }
|
||||
|
||||
public bool RunFrontEndJobs { get; set; }
|
||||
public bool RunBackEndJobs { get; set; }
|
||||
|
||||
public string BunnyCDN_User { get; set; }
|
||||
public string BunnyCDN_Password { get; set; }
|
||||
public string SqlConnection { get; set; }
|
||||
|
||||
public int MaxJobs { get; set; } = 2;
|
||||
}
|
||||
}
|
||||
@ -1,46 +0,0 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.Common;
|
||||
|
||||
namespace Wabbajack.BuildServer.BackendServices
|
||||
{
|
||||
public abstract class ABackendService
|
||||
{
|
||||
protected ABackendService(SqlService sql, AppSettings settings, TimeSpan pollRate)
|
||||
{
|
||||
Sql = sql;
|
||||
Settings = settings;
|
||||
PollRate = pollRate;
|
||||
}
|
||||
|
||||
public TimeSpan PollRate { get; }
|
||||
|
||||
public async Task RunLoop(CancellationToken token)
|
||||
{
|
||||
Utils.Log($"Starting loop for {GetType()}");
|
||||
while (!token.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Execute();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Utils.Log($"Error executing {GetType()}");
|
||||
Utils.Log(ex.ToString());
|
||||
}
|
||||
|
||||
await Task.Delay(PollRate);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public abstract Task Execute();
|
||||
|
||||
protected AppSettings Settings { get; set; }
|
||||
|
||||
protected SqlService Sql { get; set; }
|
||||
}
|
||||
}
|
||||
@ -1,77 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO.Compression;
|
||||
using System.Threading.Tasks;
|
||||
using Newtonsoft.Json;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using Wabbajack.Lib.ModListRegistry;
|
||||
|
||||
namespace Wabbajack.BuildServer.BackendServices
|
||||
{
|
||||
public class ListIngest : ABackendService
|
||||
{
|
||||
public ListIngest(SqlService sql, AppSettings settings) : base(sql, settings, TimeSpan.FromMinutes(1))
|
||||
{
|
||||
}
|
||||
|
||||
public override async Task Execute()
|
||||
{
|
||||
var client = new Common.Http.Client();
|
||||
var lists = await client.GetJsonAsync<List<ModlistMetadata>>(Consts.ModlistMetadataURL);
|
||||
bool newData = false;
|
||||
foreach (var list in lists)
|
||||
{
|
||||
if (await Sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
|
||||
continue;
|
||||
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
|
||||
if (list.NeedsDownload(modlistPath))
|
||||
{
|
||||
modlistPath.Delete();
|
||||
|
||||
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
|
||||
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
|
||||
await state.Download(modlistPath);
|
||||
}
|
||||
else
|
||||
{
|
||||
Utils.Log($"No changes detected from downloaded modlist");
|
||||
}
|
||||
|
||||
ModList modlist;
|
||||
await using (var fs = modlistPath.OpenRead())
|
||||
using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
|
||||
await using (var entry = zip.GetEntry("modlist")?.Open())
|
||||
{
|
||||
if (entry == null)
|
||||
{
|
||||
Utils.Log($"Bad Modlist {list.Links.MachineURL}");
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
modlist = entry.FromJson<ModList>();
|
||||
}
|
||||
catch (JsonReaderException ex)
|
||||
{
|
||||
Utils.Log($"Bad JSON format for {list.Links.MachineURL}");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
newData = true;
|
||||
await Sql.IngestModList(list.DownloadMetadata.Hash, list, modlist);
|
||||
}
|
||||
|
||||
if (newData)
|
||||
{
|
||||
var service = new ValidateNonNexusArchives(Sql, Settings);
|
||||
await service.Execute();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,54 +0,0 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Identity;
|
||||
using Wabbajack.BuildServer.Controllers;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
|
||||
namespace Wabbajack.BuildServer.BackendServices
|
||||
{
|
||||
public class ValidateNonNexusArchives : ABackendService
|
||||
{
|
||||
public ValidateNonNexusArchives(SqlService sql, AppSettings settings) : base(sql, settings, TimeSpan.FromHours(2))
|
||||
{
|
||||
}
|
||||
|
||||
public override async Task Execute()
|
||||
{
|
||||
Utils.Log("Updating Non Nexus archives");
|
||||
var archives = await Sql.GetNonNexusModlistArchives();
|
||||
Utils.Log($"Validating {archives.Count} Non-Nexus archives.");
|
||||
using var queue = new WorkQueue();
|
||||
await DownloadDispatcher.PrepareAll(archives.Select(a => a.State));
|
||||
var results = await archives.PMap(queue, async archive =>
|
||||
{
|
||||
try
|
||||
{
|
||||
bool isValid;
|
||||
switch (archive.State)
|
||||
{
|
||||
case GoogleDriveDownloader.State _:
|
||||
case ManualDownloader.State _:
|
||||
case HTTPDownloader.State s when new Uri(s.Url).Host.StartsWith("wabbajackpush"):
|
||||
isValid = true;
|
||||
break;
|
||||
default:
|
||||
isValid = await archive.State.Verify(archive);
|
||||
break;
|
||||
}
|
||||
return (Archive: archive, IsValid: isValid);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Utils.Log($"Got Validation error {ex}");
|
||||
return (Archive: archive, IsValid: false);
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
await Sql.UpdateNonNexusModlistArchivesStatus(results);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.Common;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
[ApiController]
|
||||
public abstract class AControllerBase<T> : ControllerBase
|
||||
{
|
||||
protected readonly ILogger<T> Logger;
|
||||
protected readonly SqlService SQL;
|
||||
|
||||
protected AControllerBase(ILogger<T> logger, SqlService sql)
|
||||
{
|
||||
Logger = logger;
|
||||
SQL = sql;
|
||||
}
|
||||
|
||||
|
||||
protected async Task Metric(string verb, string subject)
|
||||
{
|
||||
await SQL.IngestMetric(new Metric
|
||||
{
|
||||
MetricsKey = Request?.Headers[Consts.MetricsKeyHeader].FirstOrDefault() ?? "",
|
||||
Subject = subject,
|
||||
Action = verb,
|
||||
Timestamp = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,45 +0,0 @@
|
||||
using System.Threading.Tasks;
|
||||
using GraphQL;
|
||||
using GraphQL.Types;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer.GraphQL;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.Common;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
[Route("graphql")]
|
||||
[ApiController]
|
||||
public class GraphQL : AControllerBase<GraphQL>
|
||||
{
|
||||
public GraphQL(ILogger<GraphQL> logger, SqlService sql) : base(logger, sql)
|
||||
{
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> Post([FromBody] GraphQLQuery query)
|
||||
{
|
||||
var inputs = query.Variables.ToInputs();
|
||||
var schema = new Schema {
|
||||
Query = new Query(SQL)
|
||||
};
|
||||
|
||||
var result = await new DocumentExecuter().ExecuteAsync(_ =>
|
||||
{
|
||||
_.Schema = schema;
|
||||
_.Query = query.Query;
|
||||
_.OperationName = query.OperationName;
|
||||
_.Inputs = inputs;
|
||||
});
|
||||
|
||||
if (result.Errors?.Count > 0)
|
||||
{
|
||||
return BadRequest(result.Errors);
|
||||
}
|
||||
|
||||
return Ok(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,81 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Org.BouncyCastle.Asn1.Cms;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
using Wabbajack.Common.StatusFeed;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
[Route("/heartbeat")]
|
||||
public class Heartbeat : AControllerBase<Heartbeat>
|
||||
{
|
||||
static Heartbeat()
|
||||
{
|
||||
_startTime = DateTime.Now;
|
||||
|
||||
}
|
||||
private static DateTime _startTime;
|
||||
|
||||
public Heartbeat(ILogger<Heartbeat> logger, SqlService sql) : base(logger, sql)
|
||||
{
|
||||
}
|
||||
|
||||
private const int MAX_LOG_SIZE = 128;
|
||||
private static List<string> Log = new List<string>();
|
||||
public static void AddToLog(IStatusMessage msg)
|
||||
{
|
||||
lock (Log)
|
||||
{
|
||||
Log.Add(msg.ToString());
|
||||
if (Log.Count > MAX_LOG_SIZE)
|
||||
Log.RemoveAt(0);
|
||||
}
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
public async Task<IActionResult> GetHeartbeat()
|
||||
{
|
||||
return Ok(new HeartbeatResult
|
||||
{
|
||||
Uptime = DateTime.Now - _startTime,
|
||||
LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync,
|
||||
LastListValidation = DateTime.UtcNow - ListValidation.SummariesLastChecked
|
||||
});
|
||||
}
|
||||
|
||||
[JsonName("HeartbeatResult")]
|
||||
public class HeartbeatResult
|
||||
{
|
||||
public TimeSpan Uptime { get; set; }
|
||||
public TimeSpan LastNexusUpdate { get; set; }
|
||||
|
||||
public TimeSpan LastListValidation { get; set; }
|
||||
}
|
||||
|
||||
[HttpGet("only-authenticated")]
|
||||
[Authorize]
|
||||
public IActionResult OnlyAuthenticated()
|
||||
{
|
||||
var message = $"Hello from {nameof(OnlyAuthenticated)}";
|
||||
return new ObjectResult(message);
|
||||
}
|
||||
|
||||
[HttpGet("logs")]
|
||||
[Authorize]
|
||||
public IActionResult GetLogs()
|
||||
{
|
||||
string[] lst;
|
||||
lock (Log)
|
||||
{
|
||||
lst = Log.ToArray();
|
||||
}
|
||||
return Ok(string.Join("\n", lst));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,147 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using IndexedFile = Wabbajack.BuildServer.Models.IndexedFile;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
[Route("/indexed_files")]
|
||||
public class IndexedFiles : AControllerBase<IndexedFiles>
|
||||
{
|
||||
private SqlService _sql;
|
||||
private AppSettings _settings;
|
||||
|
||||
public IndexedFiles(ILogger<IndexedFiles> logger, SqlService sql, AppSettings settings) : base(logger, sql)
|
||||
{
|
||||
_settings = settings;
|
||||
_sql = sql;
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("{xxHashAsBase64}/meta.ini")]
|
||||
public async Task<IActionResult> GetFileMeta(string xxHashAsBase64)
|
||||
{
|
||||
var id = Hash.FromHex(xxHashAsBase64);
|
||||
|
||||
var result = await SQL.GetIniForHash(id);
|
||||
if (result == null)
|
||||
return NotFound();
|
||||
|
||||
Response.ContentType = "text/plain";
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("ingest/{folder}")]
|
||||
[Authorize]
|
||||
public async Task<IActionResult> Ingest(string folder)
|
||||
{
|
||||
var fullPath = folder.RelativeTo((AbsolutePath)_settings.TempFolder);
|
||||
Utils.Log($"Ingesting Inis from {fullPath}");
|
||||
int loadCount = 0;
|
||||
using var queue = new WorkQueue();
|
||||
await fullPath.EnumerateFiles().Where(f => f.Extension == Consts.IniExtension)
|
||||
.PMap(queue, async file => {
|
||||
|
||||
try
|
||||
{
|
||||
var loaded =
|
||||
(AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile(), true));
|
||||
|
||||
var hash = Hash.FromHex(((string)file.FileNameWithoutExtension).Split("_").First());
|
||||
await SQL.AddDownloadState(hash, loaded);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Utils.Log($"Failure for {file}");
|
||||
}
|
||||
|
||||
loadCount += 1;
|
||||
});
|
||||
|
||||
return Ok(loadCount);
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
[Route("notify")]
|
||||
public async Task<IActionResult> Notify()
|
||||
{
|
||||
Utils.Log("Starting ingestion of uploaded INIs");
|
||||
var body = await Request.Body.ReadAllAsync();
|
||||
await using var ms = new MemoryStream(body);
|
||||
using var za = new ZipArchive(ms, ZipArchiveMode.Read);
|
||||
int enqueued = 0;
|
||||
foreach (var entry in za.Entries)
|
||||
{
|
||||
await using var ins = entry.Open();
|
||||
var iniString = Encoding.UTF8.GetString(await ins.ReadAllAsync());
|
||||
var data = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(iniString.LoadIniString(), true));
|
||||
|
||||
if (data == null)
|
||||
{
|
||||
Utils.Log("No valid INI parser for: \n" + iniString);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (data is ManualDownloader.State)
|
||||
continue;
|
||||
|
||||
if (await SQL.HaveIndexedArchivePrimaryKey(data.PrimaryKeyString))
|
||||
continue;
|
||||
|
||||
await SQL.EnqueueJob(new Job
|
||||
{
|
||||
Priority = Job.JobPriority.Low,
|
||||
Payload = new IndexJob
|
||||
{
|
||||
Archive = new Archive(data)
|
||||
{
|
||||
Name = entry.Name,
|
||||
}
|
||||
}
|
||||
});
|
||||
enqueued += 1;
|
||||
}
|
||||
|
||||
Utils.Log($"Enqueued {enqueued} out of {za.Entries.Count} entries from uploaded ini package");
|
||||
|
||||
return Ok(enqueued.ToString());
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("{xxHashAsBase64}")]
|
||||
public async Task<IActionResult> GetFile(string xxHashAsBase64)
|
||||
{
|
||||
var result = await _sql.AllArchiveContents(BitConverter.ToInt64(xxHashAsBase64.FromHex()));
|
||||
if (result == null)
|
||||
return NotFound();
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("/game_files/{game}/{version}")]
|
||||
public async Task<IActionResult> GetGameFiles(string game, string version)
|
||||
{
|
||||
var result = await _sql.GameFiles(GameRegistry.GetByFuzzyName(game).Game, Version.Parse(version));
|
||||
return Ok(result.ToDictionary(k => k.Item1, k => k.Item2));
|
||||
}
|
||||
|
||||
public class TreeResult : IndexedFile
|
||||
{
|
||||
public List<TreeResult> ChildFiles { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,30 +0,0 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
[Authorize]
|
||||
[ApiController]
|
||||
[Route("/jobs")]
|
||||
public class Jobs : AControllerBase<Jobs>
|
||||
{
|
||||
public Jobs(ILogger<Jobs> logger, SqlService sql) : base(logger, sql)
|
||||
{
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("enqueue_job/{JobName}")]
|
||||
public async Task<long> EnqueueJob(string JobName)
|
||||
{
|
||||
var jobtype = AJobPayload.NameToType[JobName];
|
||||
var job = new Job{Priority = Job.JobPriority.High, Payload = (AJobPayload)jobtype.GetConstructor(new Type[0]).Invoke(new object[0])};
|
||||
await SQL.EnqueueJob(job);
|
||||
return job.Id;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,358 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using FluentFTP;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Nettle;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using Wabbajack.Lib.ModListRegistry;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
[ApiController]
|
||||
[Route("/lists")]
|
||||
public class ListValidation : AControllerBase<ListValidation>
|
||||
{
|
||||
public enum ArchiveStatus
|
||||
{
|
||||
Valid,
|
||||
InValid,
|
||||
Updating,
|
||||
Updated,
|
||||
}
|
||||
|
||||
public ListValidation(ILogger<ListValidation> logger, SqlService sql, IMemoryCache cache, AppSettings settings) : base(logger, sql)
|
||||
{
|
||||
_updater = new ModlistUpdater(null, sql, settings);
|
||||
_settings = settings;
|
||||
Cache = cache;
|
||||
_nexusClient = NexusApiClient.Get();
|
||||
|
||||
}
|
||||
|
||||
public static IMemoryCache Cache { get; set; }
|
||||
public const string ModListSummariesKey = "ModListSummaries";
|
||||
|
||||
public static void ResetCache()
|
||||
{
|
||||
SummariesLastChecked = DateTime.UnixEpoch;
|
||||
ModListSummaries = null;
|
||||
}
|
||||
|
||||
private static IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> ModListSummaries = null;
|
||||
public static DateTime SummariesLastChecked = DateTime.UnixEpoch;
|
||||
private static AsyncLock UpdateLock = new AsyncLock();
|
||||
public async Task<IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)>> GetSummaries()
|
||||
{
|
||||
static bool TimesUp()
|
||||
{
|
||||
return DateTime.UtcNow - SummariesLastChecked > TimeSpan.FromMinutes(5);
|
||||
}
|
||||
|
||||
if (ModListSummaries != null && !TimesUp())
|
||||
{
|
||||
return ModListSummaries;
|
||||
}
|
||||
|
||||
var task = Task.Run(async () =>
|
||||
{
|
||||
using var _ = await UpdateLock.WaitAsync();
|
||||
if (ModListSummaries != null && !TimesUp())
|
||||
{
|
||||
return ModListSummaries;
|
||||
}
|
||||
SummariesLastChecked = DateTime.UtcNow;
|
||||
|
||||
|
||||
var data = await SQL.GetValidationData();
|
||||
|
||||
using var queue = new WorkQueue();
|
||||
|
||||
var results = await data.ModLists.PMap(queue, async list =>
|
||||
{
|
||||
var (metadata, modList) = list;
|
||||
var archives = await modList.Archives.PMap(queue, async archive =>
|
||||
{
|
||||
var (_, result) = await ValidateArchive(data, archive);
|
||||
if (result != ArchiveStatus.InValid) return (archive, result);
|
||||
|
||||
return await TryToFix(data, archive);
|
||||
|
||||
});
|
||||
|
||||
var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);
|
||||
var passCount = archives.Count(f =>
|
||||
f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated);
|
||||
var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating);
|
||||
|
||||
var summary = new ModListSummary
|
||||
{
|
||||
Checked = DateTime.UtcNow,
|
||||
Failed = failedCount,
|
||||
Passed = passCount,
|
||||
Updating = updatingCount,
|
||||
MachineURL = metadata.Links.MachineURL,
|
||||
Name = metadata.Title,
|
||||
};
|
||||
|
||||
var detailed = new DetailedStatus
|
||||
{
|
||||
Name = metadata.Title,
|
||||
Checked = DateTime.UtcNow,
|
||||
DownloadMetaData = metadata.DownloadMetadata,
|
||||
HasFailures = failedCount > 0,
|
||||
MachineName = metadata.Links.MachineURL,
|
||||
Archives = archives.Select(a =>
|
||||
{
|
||||
a.Item1.Meta = "";
|
||||
return new DetailedStatusItem
|
||||
{
|
||||
Archive = a.Item1,
|
||||
IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
|
||||
};
|
||||
}).ToList()
|
||||
};
|
||||
|
||||
return (summary, detailed);
|
||||
});
|
||||
|
||||
|
||||
var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1));
|
||||
Cache.Set(ModListSummariesKey, results, cacheOptions);
|
||||
|
||||
ModListSummaries = results;
|
||||
return results;
|
||||
});
|
||||
var data = ModListSummaries;
|
||||
if (data == null)
|
||||
return await task;
|
||||
return data;
|
||||
}
|
||||
|
||||
private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(SqlService.ValidationData data, Archive archive)
|
||||
{
|
||||
switch (archive.State)
|
||||
{
|
||||
case GoogleDriveDownloader.State _:
|
||||
// Disabled for now due to GDrive rate-limiting the build server
|
||||
return (archive, ArchiveStatus.Valid);
|
||||
case NexusDownloader.State nexusState when data.NexusFiles.Contains((
|
||||
nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
|
||||
return (archive, ArchiveStatus.Valid);
|
||||
case NexusDownloader.State ns:
|
||||
return (archive, await FastNexusModStats(ns));
|
||||
case HTTPDownloader.State s when new Uri(s.Url).Host.StartsWith("wabbajackpush"):
|
||||
return (archive, ArchiveStatus.Valid);
|
||||
case ManualDownloader.State _:
|
||||
return (archive, ArchiveStatus.Valid);
|
||||
default:
|
||||
{
|
||||
if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash),
|
||||
out bool isValid))
|
||||
{
|
||||
return isValid ? (archive, ArchiveStatus.Valid) : (archive, ArchiveStatus.InValid);
|
||||
}
|
||||
|
||||
return (archive, ArchiveStatus.InValid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
|
||||
{
|
||||
|
||||
var mod = await SQL.GetNexusModInfoString(ns.Game, ns.ModID);
|
||||
var files = await SQL.GetModFiles(ns.Game, ns.ModID);
|
||||
|
||||
try
|
||||
{
|
||||
if (mod == null)
|
||||
{
|
||||
Utils.Log($"Found missing Nexus mod info {ns.Game} {ns.ModID}");
|
||||
try
|
||||
{
|
||||
mod = await (await _nexusClient).GetModInfo(ns.Game, ns.ModID, false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
mod = new ModInfo
|
||||
{
|
||||
mod_id = ns.ModID.ToString(), game_id = ns.Game.MetaData().NexusGameId, available = false
|
||||
};
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await SQL.AddNexusModInfo(ns.Game, ns.ModID, mod.updated_time, mod);
|
||||
}
|
||||
catch (Exception _)
|
||||
{
|
||||
// Could be a PK constraint failure
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (files == null)
|
||||
{
|
||||
Utils.Log($"Found missing Nexus mod file infos {ns.Game} {ns.ModID}");
|
||||
try
|
||||
{
|
||||
files = await (await _nexusClient).GetModFiles(ns.Game, ns.ModID, false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
files = new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>()};
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await SQL.AddNexusModFiles(ns.Game, ns.ModID, mod.updated_time, files);
|
||||
}
|
||||
catch (Exception _)
|
||||
{
|
||||
// Could be a PK constraint failure
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return ArchiveStatus.InValid;
|
||||
}
|
||||
|
||||
if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID))
|
||||
return ArchiveStatus.Valid;
|
||||
return ArchiveStatus.InValid;
|
||||
|
||||
}
|
||||
|
||||
private static AsyncLock _findPatchLock = new AsyncLock();
|
||||
private async Task<(Archive, ArchiveStatus)> TryToFix(SqlService.ValidationData data, Archive archive)
|
||||
{
|
||||
using var _ = await _findPatchLock.WaitAsync();
|
||||
|
||||
var result = await _updater.GetAlternative(archive.Hash.ToHex());
|
||||
return result switch
|
||||
{
|
||||
OkObjectResult ok => (archive, ArchiveStatus.Updated),
|
||||
OkResult ok => (archive, ArchiveStatus.Updated),
|
||||
AcceptedResult accept => (archive, ArchiveStatus.Updating),
|
||||
_ => (archive, ArchiveStatus.InValid)
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
[HttpGet]
|
||||
[Route("status.json")]
|
||||
public async Task<IEnumerable<ModListSummary>> HandleGetLists()
|
||||
{
|
||||
return (await GetSummaries()).Select(d => d.Summary);
|
||||
}
|
||||
|
||||
private static readonly Func<object, string> HandleGetRssFeedTemplate = NettleEngine.GetCompiler().Compile(@"
|
||||
<?xml version=""1.0""?>
|
||||
<rss version=""2.0"">
|
||||
<channel>
|
||||
<title>{{lst.Name}} - Broken Mods</title>
|
||||
<link>http://build.wabbajack.org/status/{{lst.Name}}.html</link>
|
||||
<description>These are mods that are broken and need updating</description>
|
||||
{{ each $.failed }}
|
||||
<item>
|
||||
<title>{{$.Archive.Name}} {{$.Archive.Hash}} {{$.Archive.State.PrimaryKeyString}}</title>
|
||||
<link>{{$.Archive.Name}}</link>
|
||||
</item>
|
||||
{{/each}}
|
||||
</channel>
|
||||
</rss>
|
||||
");
|
||||
|
||||
[HttpGet]
|
||||
[Route("status/{Name}/broken.rss")]
|
||||
public async Task<ContentResult> HandleGetRSSFeed(string Name)
|
||||
{
|
||||
var lst = await DetailedStatus(Name);
|
||||
var response = HandleGetRssFeedTemplate(new
|
||||
{
|
||||
lst,
|
||||
failed = lst.Archives.Where(a => a.IsFailing).ToList(),
|
||||
passed = lst.Archives.Where(a => !a.IsFailing).ToList()
|
||||
});
|
||||
return new ContentResult
|
||||
{
|
||||
ContentType = "application/rss+xml",
|
||||
StatusCode = (int) HttpStatusCode.OK,
|
||||
Content = response
|
||||
};
|
||||
}
|
||||
|
||||
private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
|
||||
<html><body>
|
||||
<h2>{{lst.Name}} - {{lst.Checked}} - {{ago}}min ago</h2>
|
||||
<h3>Failed ({{failed.Count}}):</h3>
|
||||
<ul>
|
||||
{{each $.failed }}
|
||||
<li>{{$.Archive.Name}}</li>
|
||||
{{/each}}
|
||||
</ul>
|
||||
<h3>Passed ({{passed.Count}}):</h3>
|
||||
<ul>
|
||||
{{each $.passed }}
|
||||
<li>{{$.Archive.Name}}</li>
|
||||
{{/each}}
|
||||
</ul>
|
||||
</body></html>
|
||||
");
|
||||
|
||||
private AppSettings _settings;
|
||||
private ModlistUpdater _updater;
|
||||
private Task<NexusApiClient> _nexusClient;
|
||||
|
||||
[HttpGet]
|
||||
[Route("status/{Name}.html")]
|
||||
public async Task<ContentResult> HandleGetListHtml(string Name)
|
||||
{
|
||||
|
||||
var lst = await DetailedStatus(Name);
|
||||
var response = HandleGetListTemplate(new
|
||||
{
|
||||
lst,
|
||||
ago = (DateTime.UtcNow - lst.Checked).TotalMinutes,
|
||||
failed = lst.Archives.Where(a => a.IsFailing).ToList(),
|
||||
passed = lst.Archives.Where(a => !a.IsFailing).ToList()
|
||||
});
|
||||
return new ContentResult
|
||||
{
|
||||
ContentType = "text/html",
|
||||
StatusCode = (int) HttpStatusCode.OK,
|
||||
Content = response
|
||||
};
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("status/{Name}.json")]
|
||||
public async Task<IActionResult> HandleGetListJson(string Name)
|
||||
{
|
||||
return Ok((await DetailedStatus(Name)).ToJson());
|
||||
}
|
||||
|
||||
|
||||
|
||||
private async Task<DetailedStatus> DetailedStatus(string Name)
|
||||
{
|
||||
return (await GetSummaries())
|
||||
.Select(d => d.Detailed)
|
||||
.FirstOrDefault(d => d.MachineName == Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,43 +0,0 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.Common;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
[ApiController]
|
||||
[Route("/metrics")]
|
||||
public class MetricsController : AControllerBase<MetricsController>
|
||||
{
|
||||
public MetricsController(ILogger<MetricsController> logger, SqlService sql) : base(logger, sql)
|
||||
{
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("{Subject}/{Value}")]
|
||||
public async Task<Result> LogMetricAsync(string Subject, string Value)
|
||||
{
|
||||
var date = DateTime.UtcNow;
|
||||
await Log(date, Subject, Value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault());
|
||||
return new Result { Timestamp = date};
|
||||
}
|
||||
|
||||
private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
|
||||
{
|
||||
Logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}");
|
||||
await SQL.IngestMetric(new Metric
|
||||
{
|
||||
Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey
|
||||
});
|
||||
}
|
||||
|
||||
public class Result
|
||||
{
|
||||
public DateTime Timestamp { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,255 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using Alphaleonis.Win32.Filesystem;
|
||||
using FluentFTP;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
using AlphaFile = Alphaleonis.Win32.Filesystem.File;
|
||||
using Directory = System.IO.Directory;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
[ApiController]
|
||||
[Route("/listupdater")]
|
||||
public class ModlistUpdater : AControllerBase<ModlistUpdater>
|
||||
{
|
||||
private AppSettings _settings;
|
||||
private SqlService _sql;
|
||||
|
||||
public ModlistUpdater(ILogger<ModlistUpdater> logger, SqlService sql, AppSettings settings) : base(logger, sql)
|
||||
{
|
||||
_settings = settings;
|
||||
_sql = sql;
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Authorize]
|
||||
[Route("/delete_updates")]
|
||||
public async Task<IActionResult> DeleteUpdates()
|
||||
{
|
||||
var lists = await SQL.GetDetailedModlistStatuses();
|
||||
var archives = lists.SelectMany(list => list.Archives)
|
||||
.Select(a => a.Archive.Hash.ToHex())
|
||||
.ToHashSet();
|
||||
|
||||
var toDelete = new List<string>();
|
||||
var toSave = new List<string>();
|
||||
using (var client = new FtpClient("storage.bunnycdn.com"))
|
||||
{
|
||||
client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password);
|
||||
await client.ConnectAsync();
|
||||
|
||||
foreach (var file in Directory.GetFiles("updates"))
|
||||
{
|
||||
var relativeName = Path.GetFileName(file);
|
||||
var parts = Path.GetFileName(file).Split('_', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length != 2) continue;
|
||||
|
||||
if (parts[0] == parts[1])
|
||||
{
|
||||
toDelete.Add(relativeName);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!archives.Contains(parts[0]))
|
||||
toDelete.Add(relativeName);
|
||||
else
|
||||
toSave.Add(relativeName);
|
||||
}
|
||||
|
||||
foreach (var delete in toDelete)
|
||||
{
|
||||
Utils.Log($"Deleting update {delete}");
|
||||
if (await client.FileExistsAsync($"updates/{delete}"))
|
||||
await client.DeleteFileAsync($"updates/{delete}");
|
||||
if (AlphaFile.Exists($"updates\\{delete}"))
|
||||
AlphaFile.Delete($"updates\\{delete}");
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return Ok(new {Save = toSave.ToArray(), Delete = toDelete.ToArray()}.ToJson());
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("/alternative/{xxHash}")]
|
||||
public async Task<IActionResult> GetAlternative(string xxHash)
|
||||
{
|
||||
var startingHash = Hash.FromHex(xxHash);
|
||||
await Metric("requested_upgrade", startingHash.ToString());
|
||||
|
||||
var archive = await SQL.GetStateByHash(startingHash);
|
||||
|
||||
if (archive == null)
|
||||
{
|
||||
return NotFound("Original state not found");
|
||||
}
|
||||
|
||||
Archive newArchive;
|
||||
IActionResult result;
|
||||
switch (archive.State)
|
||||
{
|
||||
case NexusDownloader.State _:
|
||||
{
|
||||
(result, newArchive) = await FindNexusAlternative(archive);
|
||||
if (newArchive == null)
|
||||
return result;
|
||||
break;
|
||||
}
|
||||
case HTTPDownloader.State _:
|
||||
(result, newArchive) = await FindHttpAlternative(archive);
|
||||
if (newArchive == null)
|
||||
return result;
|
||||
break;
|
||||
default:
|
||||
return NotFound("No alternative");
|
||||
}
|
||||
|
||||
|
||||
if (newArchive.Hash == Hash.Empty)
|
||||
{
|
||||
await SQL.EnqueueJob(new Job
|
||||
{
|
||||
Payload = new IndexJob
|
||||
{
|
||||
Archive = newArchive
|
||||
},
|
||||
OnSuccess = new Job
|
||||
{
|
||||
Payload = new PatchArchive
|
||||
{
|
||||
Src = startingHash,
|
||||
DestPK = newArchive.State.PrimaryKeyString
|
||||
}
|
||||
}
|
||||
});
|
||||
Utils.Log($"Enqueued Index and Upgrade for {startingHash} -> {newArchive.State.PrimaryKeyString}");
|
||||
return Accepted("Enqueued for Processing");
|
||||
}
|
||||
|
||||
if (startingHash == newArchive.Hash)
|
||||
return NotFound("End hash same as old hash");
|
||||
|
||||
if (!PatchArchive.CdnPath(startingHash, newArchive.Hash).Exists)
|
||||
{
|
||||
await SQL.EnqueueJob(new Job
|
||||
{
|
||||
Priority = Job.JobPriority.High,
|
||||
Payload = new PatchArchive
|
||||
{
|
||||
Src = startingHash,
|
||||
DestPK = newArchive.State.PrimaryKeyString
|
||||
}
|
||||
});
|
||||
Utils.Log($"Enqueued Upgrade for {startingHash} -> {newArchive.State.PrimaryKeyString}");
|
||||
}
|
||||
return Ok(newArchive.ToJson());
|
||||
}
|
||||
|
||||
|
||||
private async Task<(IActionResult, Archive)> FindHttpAlternative(Archive archive)
|
||||
{
|
||||
try
|
||||
{
|
||||
var valid = await archive.State.Verify(archive);
|
||||
|
||||
if (valid)
|
||||
{
|
||||
return (NotFound("Http file still valid"), null);
|
||||
}
|
||||
|
||||
archive.Hash = default;
|
||||
archive.Size = 0;
|
||||
return (Ok("Index"), archive);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Utils.Log($"Http file {archive.Hash} no longer exists");
|
||||
return (NotFound("Http file no longer exists"), null);
|
||||
}
|
||||
}
|
||||
private async Task<(IActionResult, Archive)> FindNexusAlternative(Archive archive)
|
||||
{
|
||||
var nexusState = (NexusDownloader.State)archive.State;
|
||||
var nexusGame = nexusState.Game;
|
||||
var nexusModFiles = await SQL.GetModFiles(nexusGame, nexusState.ModID);
|
||||
if (nexusModFiles == null)
|
||||
{
|
||||
Utils.Log($"No nexus mod files for {archive.Hash}");
|
||||
return (NotFound("No nexus info"), null);
|
||||
}
|
||||
var mod_files = nexusModFiles.files;
|
||||
|
||||
if (mod_files.Any(f => f.category_name != null && f.file_id == nexusState.FileID))
|
||||
{
|
||||
Utils.Log($"No available upgrade required for {nexusState.PrimaryKey}");
|
||||
await Metric("not_required_upgrade", archive.Hash.ToString());
|
||||
return (BadRequest("Upgrade Not Required"), null);
|
||||
}
|
||||
|
||||
Utils.Log($"Found original, looking for alternatives to {archive.Hash}");
|
||||
var newArchive = await FindNexusAlternative(nexusState, archive.Hash);
|
||||
if (newArchive != null)
|
||||
{
|
||||
return (Ok(newArchive), newArchive);
|
||||
}
|
||||
|
||||
Utils.Log($"No available upgrade for {nexusState.PrimaryKey}");
|
||||
return (NotFound("No alternative available"), null);
|
||||
|
||||
}
|
||||
|
||||
private async Task<Archive> FindNexusAlternative(NexusDownloader.State state, Hash srcHash)
|
||||
{
|
||||
var origSize = _settings.PathForArchive(srcHash).Size;
|
||||
var api = await NexusApiClient.Get(Request?.Headers["apikey"].FirstOrDefault());
|
||||
var allMods = await api.GetModFiles(state.Game, state.ModID);
|
||||
var archive = allMods.files.Where(m => !string.IsNullOrEmpty(m.category_name))
|
||||
.OrderBy(s => Math.Abs((long)s.size - origSize))
|
||||
.Select(s =>
|
||||
new Archive(
|
||||
new NexusDownloader.State
|
||||
{
|
||||
Game = state.Game,
|
||||
ModID = state.ModID,
|
||||
FileID = s.file_id
|
||||
})
|
||||
{
|
||||
Name = s.file_name,
|
||||
Size = (long)s.size,
|
||||
})
|
||||
.FirstOrDefault();
|
||||
|
||||
if (archive == null)
|
||||
{
|
||||
Utils.Log($"No alternative for {srcHash}");
|
||||
return null;
|
||||
}
|
||||
|
||||
Utils.Log($"Found alternative for {srcHash}");
|
||||
|
||||
var indexed = await SQL.DownloadStateByPrimaryKey(archive.State.PrimaryKeyString);
|
||||
|
||||
if (indexed == null)
|
||||
{
|
||||
return archive;
|
||||
}
|
||||
|
||||
Utils.Log($"Pre-Indexed alternative {indexed.Hash} found for {srcHash}");
|
||||
archive.Hash = indexed.Hash;
|
||||
return archive;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,154 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
//[Authorize]
|
||||
[ApiController]
|
||||
[Route("/v1/games/")]
|
||||
public class NexusCache : AControllerBase<NexusCache>
|
||||
{
|
||||
private AppSettings _settings;
|
||||
private static long CachedCount = 0;
|
||||
private static long ForwardCount = 0;
|
||||
|
||||
public NexusCache(ILogger<NexusCache> logger, SqlService sql, AppSettings settings) : base(logger, sql)
|
||||
{
|
||||
_settings = settings;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Looks up the mod details for a given Gamename/ModId pair. If the entry is not found in the cache it will
|
||||
/// be requested from the server (using the caller's Nexus API key if provided).
|
||||
/// </summary>
|
||||
/// <param name="db"></param>
|
||||
/// <param name="GameName">The Nexus game name</param>
|
||||
/// <param name="ModId">The Nexus mod id</param>
|
||||
/// <returns>A Mod Info result</returns>
|
||||
[HttpGet]
|
||||
[Route("{GameName}/mods/{ModId}.json")]
|
||||
public async Task<ModInfo> GetModInfo(string GameName, long ModId)
|
||||
{
|
||||
var game = GameRegistry.GetByFuzzyName(GameName).Game;
|
||||
var result = await SQL.GetNexusModInfoString(game, ModId);
|
||||
|
||||
string method = "CACHED";
|
||||
if (result == null)
|
||||
{
|
||||
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
|
||||
result = await api.GetModInfo(game, ModId, false);
|
||||
await SQL.AddNexusModInfo(game, ModId, DateTime.UtcNow, result);
|
||||
|
||||
|
||||
method = "NOT_CACHED";
|
||||
Interlocked.Increment(ref ForwardCount);
|
||||
}
|
||||
else
|
||||
{
|
||||
Interlocked.Increment(ref CachedCount);
|
||||
}
|
||||
|
||||
Response.Headers.Add("x-cache-result", method);
|
||||
return result;
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("{GameName}/mods/{ModId}/files.json")]
|
||||
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, long ModId)
|
||||
{
|
||||
var game = GameRegistry.GetByFuzzyName(GameName).Game;
|
||||
var result = await SQL.GetModFiles(game, ModId);
|
||||
|
||||
string method = "CACHED";
|
||||
if (result == null)
|
||||
{
|
||||
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
|
||||
result = await api.GetModFiles(game, ModId, false);
|
||||
await SQL.AddNexusModFiles(game, ModId, DateTime.UtcNow, result);
|
||||
|
||||
method = "NOT_CACHED";
|
||||
Interlocked.Increment(ref ForwardCount);
|
||||
}
|
||||
else
|
||||
{
|
||||
Interlocked.Increment(ref CachedCount);
|
||||
}
|
||||
Response.Headers.Add("x-cache-result", method);
|
||||
return result;
|
||||
}
|
||||
|
||||
private class NexusIngestHeader
|
||||
{
|
||||
public List<NexusCacheData<ModInfo>> ModInfos { get; set; }
|
||||
public List<NexusCacheData<NexusFileInfo>> FileInfos { get; set; }
|
||||
public List<NexusCacheData<NexusApiClient.GetModFilesResponse>> ModFiles { get; set; }
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("/nexus_cache/ingest")]
|
||||
[Authorize]
|
||||
public async Task<IActionResult> IngestNexusFile()
|
||||
{
|
||||
long totalRows = 0;
|
||||
|
||||
var dataPath = @"nexus_export.json".RelativeTo(_settings.TempPath);
|
||||
|
||||
var data = JsonConvert.DeserializeObject<NexusIngestHeader>(await dataPath.ReadAllTextAsync());
|
||||
|
||||
foreach (var record in data.ModInfos)
|
||||
{
|
||||
if (!GameRegistry.TryGetByFuzzyName(record.Game, out var game)) continue;
|
||||
|
||||
await SQL.AddNexusModInfo(game.Game, record.ModId,
|
||||
record.LastCheckedUTC, record.Data);
|
||||
totalRows += 1;
|
||||
}
|
||||
|
||||
foreach (var record in data.FileInfos)
|
||||
{
|
||||
if (!GameRegistry.TryGetByFuzzyName(record.Game, out var game)) continue;
|
||||
|
||||
await SQL.AddNexusFileInfo(game.Game, record.ModId,
|
||||
long.Parse(record.FileId),
|
||||
record.LastCheckedUTC, record.Data);
|
||||
totalRows += 1;
|
||||
}
|
||||
|
||||
foreach (var record in data.ModFiles)
|
||||
{
|
||||
if (!GameRegistry.TryGetByFuzzyName(record.Game, out var game)) continue;
|
||||
|
||||
await SQL.AddNexusModFiles(game.Game, record.ModId,
|
||||
record.LastCheckedUTC, record.Data);
|
||||
totalRows += 1;
|
||||
}
|
||||
|
||||
return Ok(totalRows);
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("/nexus_cache/stats")]
|
||||
public async Task<IActionResult> NexusCacheStats()
|
||||
{
|
||||
return Ok(new ClientAPI.NexusCacheStats
|
||||
{
|
||||
CachedCount = CachedCount,
|
||||
ForwardCount = ForwardCount,
|
||||
CacheRatio = (double)CachedCount / (ForwardCount == 0 ? 1 : ForwardCount)
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,273 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Security.Claims;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using FluentFTP;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Nettle;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
using Wabbajack.Common;
|
||||
using Path = Alphaleonis.Win32.Filesystem.Path;
|
||||
using AlphaFile = Alphaleonis.Win32.Filesystem.File;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
public class UploadedFiles : AControllerBase<UploadedFiles>
|
||||
{
|
||||
private static ConcurrentDictionary<string, AsyncLock> _writeLocks = new ConcurrentDictionary<string, AsyncLock>();
|
||||
private AppSettings _settings;
|
||||
|
||||
public UploadedFiles(ILogger<UploadedFiles> logger, AppSettings settings, SqlService sql) : base(logger, sql)
|
||||
{
|
||||
_settings = settings;
|
||||
}
|
||||
|
||||
[HttpPut]
|
||||
[Route("upload_file/{Name}/start")]
|
||||
public async Task<IActionResult> UploadFileStreaming(string Name)
|
||||
{
|
||||
var guid = Guid.NewGuid();
|
||||
var key = Encoding.UTF8.GetBytes($"{Path.GetFileNameWithoutExtension(Name)}|{guid.ToString()}|{Path.GetExtension(Name)}").ToHex();
|
||||
|
||||
_writeLocks.GetOrAdd(key, new AsyncLock());
|
||||
|
||||
await using var fs = _settings.TempPath.Combine(key).Create();
|
||||
Utils.Log($"Starting Ingest for {key}");
|
||||
return Ok(key);
|
||||
}
|
||||
|
||||
static private HashSet<char> HexChars = new HashSet<char>("abcdef1234567890");
|
||||
[HttpPut]
|
||||
[Route("upload_file/{Key}/data/{Offset}")]
|
||||
public async Task<IActionResult> UploadFilePart(string Key, long Offset)
|
||||
{
|
||||
if (!Key.All(a => HexChars.Contains(a)))
|
||||
return BadRequest("NOT A VALID FILENAME");
|
||||
|
||||
var ms = new MemoryStream();
|
||||
await Request.Body.CopyToAsync(ms);
|
||||
ms.Position = 0;
|
||||
|
||||
Utils.Log($"Writing {ms.Length} at position {Offset} in ingest file {Key}");
|
||||
|
||||
long position;
|
||||
using (var _ = await _writeLocks[Key].WaitAsync())
|
||||
{
|
||||
await using var file = _settings.TempPath.Combine(Key).WriteShared();
|
||||
file.Position = Offset;
|
||||
await ms.CopyToAsync(file);
|
||||
position = Offset + ms.Length;
|
||||
}
|
||||
|
||||
Utils.Log($"Wrote {ms.Length} as position {Offset} result {position}");
|
||||
|
||||
return Ok(position);
|
||||
}
|
||||
|
||||
[Authorize]
|
||||
[HttpGet]
|
||||
[Route("clean_http_uploads")]
|
||||
public async Task<IActionResult> CleanUploads()
|
||||
{
|
||||
var files = await SQL.AllUploadedFiles();
|
||||
var seen = new HashSet<string>();
|
||||
var duplicate = new List<UploadedFile>();
|
||||
|
||||
foreach (var file in files)
|
||||
{
|
||||
if (seen.Contains(file.Name))
|
||||
duplicate.Add(file);
|
||||
else
|
||||
seen.Add(file.Name);
|
||||
}
|
||||
|
||||
using (var client = new FtpClient("storage.bunnycdn.com"))
|
||||
{
|
||||
client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password);
|
||||
await client.ConnectAsync();
|
||||
|
||||
foreach (var dup in duplicate)
|
||||
{
|
||||
var final_path = Path.Combine("public", "files", dup.MungedName);
|
||||
Utils.Log($"Cleaning upload {final_path}");
|
||||
|
||||
if (AlphaFile.Exists(final_path))
|
||||
AlphaFile.Delete(final_path);
|
||||
|
||||
if (await client.FileExistsAsync(dup.MungedName))
|
||||
await client.DeleteFileAsync(dup.MungedName);
|
||||
await SQL.DeleteUploadedFile(dup.Id);
|
||||
}
|
||||
}
|
||||
|
||||
return Ok(new {Remain = seen.ToArray(), Deleted = duplicate.ToArray()}.ToJson());
|
||||
}
|
||||
|
||||
|
||||
[HttpPut]
|
||||
[Route("upload_file/{Key}/finish/{xxHashAsHex}")]
|
||||
public async Task<IActionResult> UploadFileFinish(string Key, string xxHashAsHex)
|
||||
{
|
||||
var expectedHash = Hash.FromHex(xxHashAsHex);
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
if (!Key.All(a => HexChars.Contains(a)))
|
||||
return BadRequest("NOT A VALID FILENAME");
|
||||
var parts = Encoding.UTF8.GetString(Key.FromHex()).Split('|');
|
||||
var finalName = $"{parts[0]}-{parts[1]}{parts[2]}";
|
||||
var originalName = $"{parts[0]}{parts[2]}";
|
||||
|
||||
var finalPath = "public".RelativeTo(AbsolutePath.EntryPoint).Combine("files", finalName);
|
||||
await _settings.TempPath.Combine(Key).MoveToAsync(finalPath);
|
||||
|
||||
var hash = await finalPath.FileHashAsync();
|
||||
|
||||
if (expectedHash != hash)
|
||||
{
|
||||
finalPath.Delete();
|
||||
return BadRequest($"Bad Hash, Expected: {expectedHash} Got: {hash}");
|
||||
}
|
||||
|
||||
_writeLocks.TryRemove(Key, out var _);
|
||||
var record = new UploadedFile
|
||||
{
|
||||
Id = Guid.Parse(parts[1]),
|
||||
Hash = hash,
|
||||
Name = originalName,
|
||||
Uploader = user,
|
||||
Size = finalPath.Size,
|
||||
CDNName = "wabbajackpush"
|
||||
};
|
||||
await SQL.AddUploadedFile(record);
|
||||
await SQL.EnqueueJob(new Job
|
||||
{
|
||||
Priority = Job.JobPriority.High, Payload = new UploadToCDN {FileId = record.Id}
|
||||
});
|
||||
|
||||
|
||||
return Ok(record.Uri);
|
||||
}
|
||||
|
||||
|
||||
private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
|
||||
<html><body>
|
||||
<table>
|
||||
{{each $.files }}
|
||||
<tr><td><a href='{{$.Link}}'>{{$.Name}}</a></td><td>{{$.Size}}</td><td>{{$.Date}}</td><td>{{$.Uploader}}</td></tr>
|
||||
{{/each}}
|
||||
</table>
|
||||
</body></html>
|
||||
");
|
||||
|
||||
|
||||
[HttpGet]
|
||||
[Route("uploaded_files")]
|
||||
public async Task<ContentResult> UploadedFilesGet()
|
||||
{
|
||||
var files = await SQL.AllUploadedFiles();
|
||||
var response = HandleGetListTemplate(new
|
||||
{
|
||||
files = files.Select(file => new
|
||||
{
|
||||
Link = file.Uri,
|
||||
Size = file.Size.ToFileSizeString(),
|
||||
file.Name,
|
||||
Date = file.UploadDate,
|
||||
file.Uploader
|
||||
})
|
||||
|
||||
});
|
||||
return new ContentResult
|
||||
{
|
||||
ContentType = "text/html",
|
||||
StatusCode = (int) HttpStatusCode.OK,
|
||||
Content = response
|
||||
};
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("uploaded_files/list")]
|
||||
[Authorize]
|
||||
public async Task<IActionResult> ListMyFiles()
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
Utils.Log($"List Uploaded Files {user}");
|
||||
var files = await SQL.AllUploadedFilesForUser(user);
|
||||
return Ok(files.OrderBy(f => f.UploadDate).Select(f => f.MungedName ).ToArray().ToJson());
|
||||
}
|
||||
|
||||
[HttpDelete]
|
||||
[Route("uploaded_files/{name}")]
|
||||
[Authorize]
|
||||
public async Task<IActionResult> DeleteMyFile(string name)
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
Utils.Log($"Delete Uploaded File {user} {name}");
|
||||
var files = await SQL.AllUploadedFilesForUser(user);
|
||||
|
||||
var to_delete = files.First(f => f.MungedName == name);
|
||||
|
||||
if (AlphaFile.Exists(Path.Combine("public", "files", to_delete.MungedName)))
|
||||
AlphaFile.Delete(Path.Combine("public", "files", to_delete.MungedName));
|
||||
|
||||
|
||||
if (_settings.BunnyCDN_User != "TEST" || _settings.BunnyCDN_Password != "TEST")
|
||||
{
|
||||
using (var client = new FtpClient("storage.bunnycdn.com"))
|
||||
{
|
||||
client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password);
|
||||
await client.ConnectAsync();
|
||||
if (await client.FileExistsAsync(to_delete.MungedName))
|
||||
await client.DeleteFileAsync(to_delete.MungedName);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
await SQL.DeleteUploadedFile(to_delete.Id);
|
||||
return Ok($"Deleted {to_delete.MungedName}");
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("ingest/uploaded_files/{name}")]
|
||||
[Authorize]
|
||||
public async Task<IActionResult> IngestMongoDB(string name)
|
||||
{
|
||||
var fullPath = name.RelativeTo((AbsolutePath)_settings.TempFolder);
|
||||
await using var fs = fullPath.OpenRead();
|
||||
|
||||
var files = new List<UploadedFile>();
|
||||
using var rdr = new JsonTextReader(new StreamReader(fs)) {SupportMultipleContent = true};
|
||||
|
||||
while (await rdr.ReadAsync())
|
||||
{
|
||||
dynamic obj = await JObject.LoadAsync(rdr);
|
||||
|
||||
|
||||
var uf = new UploadedFile
|
||||
{
|
||||
Id = Guid.Parse((string)obj._id),
|
||||
Name = obj.Name,
|
||||
Size = long.Parse((string)(obj.Size["$numberLong"] ?? obj.Size["$numberInt"])),
|
||||
Hash = Hash.FromBase64((string)obj.Hash),
|
||||
Uploader = obj.Uploader,
|
||||
UploadDate = long.Parse(((string)obj.UploadDate["$date"]["$numberLong"]).Substring(0, 10)).AsUnixTime(),
|
||||
CDNName = obj.CDNName
|
||||
};
|
||||
files.Add(uf);
|
||||
await SQL.AddUploadedFile(uf);
|
||||
}
|
||||
return Ok(files.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,47 +0,0 @@
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.Common;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
[Authorize]
|
||||
[Route("/users")]
|
||||
public class Users : AControllerBase<Users>
|
||||
{
|
||||
private AppSettings _settings;
|
||||
|
||||
public Users(ILogger<Users> logger, SqlService sql, AppSettings settings) : base(logger, sql)
|
||||
{
|
||||
_settings = settings;
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("add/{Name}")]
|
||||
public async Task<string> AddUser(string Name)
|
||||
{
|
||||
return await SQL.AddLogin(Name);
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("export")]
|
||||
public async Task<string> Export()
|
||||
{
|
||||
var mainFolder = _settings.TempPath.Combine("exported_users");
|
||||
mainFolder.CreateDirectory();
|
||||
|
||||
foreach (var (owner, key) in await SQL.GetAllUserKeys())
|
||||
{
|
||||
var folder = mainFolder.Combine(owner);
|
||||
folder.CreateDirectory();
|
||||
await folder.Combine(Consts.AuthorAPIKeyFile).WriteAllTextAsync(key);
|
||||
}
|
||||
|
||||
return "done";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,57 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Authentication;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Wabbajack.Common;
|
||||
using File = Alphaleonis.Win32.Filesystem.File;
|
||||
|
||||
namespace Wabbajack.BuildServer
|
||||
{
|
||||
public static class Extensions
|
||||
{
|
||||
public static void UseJobManager(this IApplicationBuilder b)
|
||||
{
|
||||
var manager = (JobManager)b.ApplicationServices.GetService(typeof(JobManager));
|
||||
var tsk = manager.JobScheduler();
|
||||
|
||||
manager.StartJobRunners();
|
||||
}
|
||||
|
||||
public static async Task CopyFileAsync(string sourcePath, string destinationPath)
|
||||
{
|
||||
using (Stream source = File.OpenRead(sourcePath))
|
||||
{
|
||||
using(Stream destination = File.Create(destinationPath))
|
||||
{
|
||||
await source.CopyToAsync(destination);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static AuthenticationBuilder AddApiKeySupport(this AuthenticationBuilder authenticationBuilder, Action<ApiKeyAuthenticationOptions> options)
|
||||
{
|
||||
return authenticationBuilder.AddScheme<ApiKeyAuthenticationOptions, ApiKeyAuthenticationHandler>(ApiKeyAuthenticationOptions.DefaultScheme, options);
|
||||
}
|
||||
|
||||
private static readonly ConcurrentDictionary<Hash, AbsolutePath> PathForArchiveHash = new ConcurrentDictionary<Hash, AbsolutePath>();
|
||||
public static AbsolutePath PathForArchive(this AppSettings settings, Hash hash)
|
||||
{
|
||||
if (PathForArchiveHash.TryGetValue(hash, out AbsolutePath result))
|
||||
return result;
|
||||
|
||||
var hexHash = hash.ToHex();
|
||||
|
||||
var ends = "_" + hexHash + "_";
|
||||
var file = settings.ArchivePath.EnumerateFiles()
|
||||
.FirstOrDefault(f => ((string)f.FileNameWithoutExtension).EndsWith(ends));
|
||||
|
||||
if (file != default)
|
||||
PathForArchiveHash.TryAdd(hash, file);
|
||||
return file;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,28 +0,0 @@
|
||||
Deployment Plan for 2.0 go-live
|
||||
|
||||
1. Release 2.0 to authors and let them rebuild their lists
|
||||
1. Save old configs so the don't get overwritten
|
||||
1. Backup SQL server data
|
||||
1. Update SQL Tables
|
||||
1. Nexus Mod Files
|
||||
1. Nexus Mod Infos
|
||||
1. Job Queue
|
||||
1. Api Keys
|
||||
1. Mod Lists
|
||||
1. Download States
|
||||
1. Uploaded Files
|
||||
1. Export Download Inis from server
|
||||
1. Export all cache files from server
|
||||
1. Hand insert all API keys
|
||||
1. Copy over new server binaries
|
||||
1. Disable background jobs on server
|
||||
1. Start new server
|
||||
1. Load data
|
||||
1. Import downloaded Inis
|
||||
1. Import all cache files
|
||||
1. Stop server
|
||||
1. Enable backend jobs
|
||||
1. Start server
|
||||
1. Verify that list validation triggers
|
||||
1. ???
|
||||
1. Profit?
|
||||
@ -1,15 +0,0 @@
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace Wabbajack.BuildServer.GraphQL
|
||||
{
|
||||
public class GraphQLQuery
|
||||
{
|
||||
|
||||
public string OperationName { get; set; }
|
||||
|
||||
public string NamedQuery { get; set; }
|
||||
public string Query { get; set; }
|
||||
public JObject Variables { get; set; }
|
||||
}
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
using GraphQL.Types;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
|
||||
namespace Wabbajack.BuildServer.GraphQL
|
||||
{
|
||||
public class JobType : ObjectGraphType<Job>
|
||||
{
|
||||
public JobType()
|
||||
{
|
||||
Name = "Job";
|
||||
Field(x => x.Id, type: typeof(IdGraphType)).Description("Unique Id of the Job");
|
||||
Field(x => x.Payload.Description).Description("Description of the job's behavior");
|
||||
Field(x => x.Created, type: typeof(DateTimeGraphType)).Description("Creation time of the Job");
|
||||
Field(x => x.Started, type: typeof(DateTimeGraphType)).Description("Started time of the Job");
|
||||
Field(x => x.Ended, type: typeof(DateTimeGraphType)).Description("Ended time of the Job");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using GraphQL.Types;
|
||||
|
||||
namespace Wabbajack.BuildServer.GraphQL
|
||||
{
|
||||
public class MetricEnum : EnumerationGraphType
|
||||
{
|
||||
public MetricEnum()
|
||||
{
|
||||
Name = "MetricType";
|
||||
Description = "The metric grouping";
|
||||
AddValue("BEGIN_INSTALL", "Installation of a modlist started", "begin_install");
|
||||
AddValue("FINISHED_INSTALL", "Installation of a modlist finished", "finish_install");
|
||||
AddValue("BEGIN_DOWNLOAD", "Downloading of a modlist begain started", "downloading");
|
||||
}
|
||||
}
|
||||
|
||||
public class MetricResultType : ObjectGraphType<MetricResult>
|
||||
{
|
||||
public MetricResultType()
|
||||
{
|
||||
Name = "MetricResult";
|
||||
Description =
|
||||
"A single line of data from a metrics graph. For example, the number of unique downloads each day.";
|
||||
Field(x => x.SeriesName).Description("The name of the data series");
|
||||
Field(x => x.Labels).Description("The name for each plot of data (for example the date for each value");
|
||||
Field(x => x.Values).Description("The value for each plot of data");
|
||||
}
|
||||
}
|
||||
public class MetricResult
|
||||
{
|
||||
public string SeriesName { get; set; }
|
||||
public List<string> Labels { get; set; }
|
||||
public List<int> Values { get; set; }
|
||||
}
|
||||
}
|
||||
@ -1,67 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using GraphQL.Types;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.Lib.ModListRegistry;
|
||||
|
||||
namespace Wabbajack.BuildServer.GraphQL
|
||||
{
|
||||
public class ModListStatusType : ObjectGraphType<ModListStatus>
|
||||
{
|
||||
public ModListStatusType()
|
||||
{
|
||||
Name = "ModlistSummary";
|
||||
Description = "Short summary of a modlist status";
|
||||
Field(x => x.Id).Description("Name of the modlist");
|
||||
Field(x => x.Metadata.Title).Description("Human-friendly name of the modlist");
|
||||
Field<ListGraphType<ModListArchiveType>>("Archives",
|
||||
arguments: new QueryArguments(new QueryArgument<ArchiveEnumFilterType>
|
||||
{
|
||||
Name = "filter", Description = "Type of archives to return"
|
||||
}),
|
||||
resolve: context =>
|
||||
{
|
||||
var arg = context.GetArgument<string>("filter");
|
||||
var archives = (IEnumerable<DetailedStatusItem>)context.Source.DetailedStatus.Archives;
|
||||
switch (arg)
|
||||
{
|
||||
case "FAILED":
|
||||
archives = archives.Where(a => a.IsFailing);
|
||||
break;
|
||||
case "PASSED":
|
||||
archives = archives.Where(a => !a.IsFailing);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return archives;
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public class ModListArchiveType : ObjectGraphType<DetailedStatusItem>
|
||||
{
|
||||
public ModListArchiveType()
|
||||
{
|
||||
Field(x => x.IsFailing).Description("Is this archive failing validation?");
|
||||
Field(x => x.Archive.Name).Description("Name of the archive");
|
||||
Field(x => x.Archive.Hash).Description("Hash of the archive");
|
||||
Field(x => x.Archive.Size).Description("Size of the archive");
|
||||
}
|
||||
}
|
||||
|
||||
public class ArchiveEnumFilterType : EnumerationGraphType
|
||||
{
|
||||
public ArchiveEnumFilterType()
|
||||
{
|
||||
Name = "ArchiveFilterEnum";
|
||||
Description = "What archives should be returned from a sublist";
|
||||
AddValue("ALL", "All archives are returned", "ALL");
|
||||
AddValue("FAILED", "All archives are returned", "FAILED");
|
||||
AddValue("PASSED", "All archives are returned", "PASSED");
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,52 +0,0 @@
|
||||
using System.Linq;
|
||||
using GraphQL.Types;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
|
||||
namespace Wabbajack.BuildServer.GraphQL
|
||||
{
|
||||
public class Query : ObjectGraphType
|
||||
{
|
||||
public Query(SqlService sql)
|
||||
{
|
||||
/*
|
||||
FieldAsync<ListGraphType<ModListStatusType>>("modLists",
|
||||
arguments: new QueryArguments(new QueryArgument<ArchiveEnumFilterType>
|
||||
{
|
||||
Name = "filter", Description = "Filter lists to those that only have these archive classifications"
|
||||
}),
|
||||
resolve: async context =>
|
||||
{
|
||||
var arg = context.GetArgument<string>("filter");
|
||||
var lists = await sql.GetDetailedModlistStatuses();
|
||||
switch (arg)
|
||||
{
|
||||
case "FAILED":
|
||||
return lists.Where(l => l.HasFailures);
|
||||
case "PASSED":
|
||||
return lists.Where(l => !l.HasFailures);
|
||||
default:
|
||||
return lists;
|
||||
}
|
||||
|
||||
});
|
||||
*/
|
||||
FieldAsync<ListGraphType<MetricResultType>>("dailyUniqueMetrics",
|
||||
arguments: new QueryArguments(
|
||||
new QueryArgument<MetricEnum> {Name = "metric_type", Description = "The grouping of metric data to query"}
|
||||
),
|
||||
resolve: async context =>
|
||||
{
|
||||
var group = context.GetArgument<string>("metric_type");
|
||||
var data = (await sql.MetricsReport(group))
|
||||
.GroupBy(m => m.Subject)
|
||||
.Select(g => new MetricResult
|
||||
{
|
||||
SeriesName = g.Key,
|
||||
Labels = g.Select(m => m.Date.ToString()).ToList(),
|
||||
Values = g.Select(m => m.Count).ToList()
|
||||
});
|
||||
return data;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,22 +0,0 @@
|
||||
using GraphQL.Types;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
|
||||
namespace Wabbajack.BuildServer.GraphQL
|
||||
{
|
||||
public class UploadedFileType : ObjectGraphType<UploadedFile>
|
||||
{
|
||||
public UploadedFileType()
|
||||
{
|
||||
Name = "UploadedFile";
|
||||
Description = "A file uploaded for hosting on Wabbajack's static file hosting";
|
||||
Field(x => x.Id, type: typeof(IdGraphType)).Description("Unique Id of the Job");
|
||||
Field(x => x.Name).Description("Non-unique name of the file");
|
||||
Field(x => x.MungedName, type: typeof(IdGraphType)).Description("Unique file name");
|
||||
Field(x => x.UploadDate, type: typeof(DateGraphType)).Description("Date of the file upload");
|
||||
Field(x => x.Uploader, type: typeof(IdGraphType)).Description("Uploader of the file");
|
||||
Field(x => x.Uri, type: typeof(UriGraphType)).Description("URI of the file");
|
||||
Field(x => x.Hash).Description("xxHash64 of the file");
|
||||
Field(x => x.Size).Description("Size of the file");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
using GraphQL.Types;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
|
||||
namespace Wabbajack.BuildServer.GraphQL
|
||||
{
|
||||
public class VirtualFileType : ObjectGraphType<IndexedFileWithChildren>
|
||||
{
|
||||
public VirtualFileType()
|
||||
{
|
||||
Name = "VirtualFile";
|
||||
Field(x => x.Hash, type: typeof(IdGraphType)).Description("xxHash64 of the file, in Base64 encoding");
|
||||
Field(x => x.Size, type: typeof(LongGraphType)).Description("Size of the file");
|
||||
Field(x => x.IsArchive).Description("True if this file is an archive (BSA, zip, 7z, etc.)");
|
||||
Field(x => x.SHA256).Description("SHA256 hash of the file, in hexidecimal encoding");
|
||||
Field(x => x.SHA1).Description("SHA1 hash of the file, in hexidecimal encoding");
|
||||
Field(x => x.MD5).Description("MD5 hash of the file, in hexidecimal encoding");
|
||||
Field(x => x.CRC).Description("CRC32 hash of the file, in hexidecimal encoding");
|
||||
Field(x => x.Children, type: typeof(ChildFileType)).Description("Metadata for the files in this archive (if any)");
|
||||
}
|
||||
}
|
||||
|
||||
public class ChildFileType : ObjectGraphType<ChildFile>
|
||||
{
|
||||
public ChildFileType()
|
||||
{
|
||||
Name = "ChildFile";
|
||||
Field(x => x.Name).Description("The relative path to the file inside the parent archive");
|
||||
Field(x => x.Hash).Description("The hash (xxHash64, Base64 ecoded) of the child file");
|
||||
Field(x => x.Extension).Description("File extension of the child file");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,159 +0,0 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Reactive.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Nettle;
|
||||
using Wabbajack.BuildServer.BackendServices;
|
||||
using Wabbajack.BuildServer.Controllers;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
|
||||
namespace Wabbajack.BuildServer
|
||||
{
|
||||
public class JobManager
|
||||
{
|
||||
protected readonly ILogger<JobManager> Logger;
|
||||
protected readonly AppSettings Settings;
|
||||
protected SqlService Sql;
|
||||
|
||||
public JobManager(ILogger<JobManager> logger, SqlService sql, AppSettings settings)
|
||||
{
|
||||
Logger = logger;
|
||||
Settings = settings;
|
||||
Sql = sql;
|
||||
}
|
||||
|
||||
|
||||
public void StartJobRunners()
|
||||
{
|
||||
if (!Settings.JobRunner) return;
|
||||
for (var idx = 0; idx < Settings.MaxJobs; idx++)
|
||||
{
|
||||
Task.Run(async () =>
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
try
|
||||
{
|
||||
var job = await Sql.GetJob();
|
||||
if (job == null)
|
||||
{
|
||||
await Task.Delay(5000);
|
||||
continue;
|
||||
}
|
||||
|
||||
Logger.Log(LogLevel.Information, $"Starting job: {job.Payload.Description}");
|
||||
try
|
||||
{
|
||||
job.Result = await job.Payload.Execute(Sql, Settings);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Log(LogLevel.Error, ex, $"Error while running job: {job.Payload.Description}");
|
||||
job.Result = JobResult.Error(ex);
|
||||
}
|
||||
|
||||
await Sql.FinishJob(job);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Log(LogLevel.Error, ex, $"Error getting or updating job");
|
||||
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public async Task JobScheduler()
|
||||
{
|
||||
Utils.LogMessages.Subscribe(msg => Logger.Log(LogLevel.Information, msg.ToString()));
|
||||
Utils.LogMessages.Subscribe(Heartbeat.AddToLog);
|
||||
Utils.LogMessages.OfType<IUserIntervention>().Subscribe(u => u.Cancel());
|
||||
if (!Settings.JobScheduler) return;
|
||||
|
||||
var token = new CancellationTokenSource();
|
||||
var task = RunNexusCacheLoop();
|
||||
var listIngest = (new ListIngest(Sql, Settings)).RunLoop(token.Token);
|
||||
var nonNexus = (new ValidateNonNexusArchives(Sql, Settings)).RunLoop(token.Token);
|
||||
|
||||
while (true)
|
||||
{
|
||||
await KillOrphanedJobs();
|
||||
await ScheduledJob<GetNexusUpdatesJob>(TimeSpan.FromHours(1), Job.JobPriority.High);
|
||||
//await ScheduledJob<UpdateModLists>(TimeSpan.FromMinutes(30), Job.JobPriority.High);
|
||||
//await ScheduledJob<EnqueueAllArchives>(TimeSpan.FromHours(2), Job.JobPriority.Low);
|
||||
//await ScheduledJob<EnqueueAllGameFiles>(TimeSpan.FromHours(24), Job.JobPriority.High);
|
||||
await ScheduledJob<IndexDynDOLOD>(TimeSpan.FromHours(1), Job.JobPriority.Normal);
|
||||
await Task.Delay(10000);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunNexusCacheLoop()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
await GetNexusUpdatesJob.UpdateNexusCacheFast(Sql);
|
||||
await Task.Delay(TimeSpan.FromMinutes(1));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private async Task KillOrphanedJobs()
|
||||
{
|
||||
try
|
||||
{
|
||||
var started = await Sql.GetRunningJobs();
|
||||
foreach (var job in started)
|
||||
{
|
||||
var runtime = DateTime.Now - job.Started;
|
||||
|
||||
if (!(runtime > TimeSpan.FromMinutes(30))) continue;
|
||||
|
||||
job.Result = JobResult.Error(new Exception($"Timeout after {runtime.Value.TotalMinutes}"));
|
||||
await Sql.FinishJob(job);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.Log(LogLevel.Error, ex, "Error in JobScheduler when scheduling KillOrphanedJobs");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ScheduledJob<T>(TimeSpan span, Job.JobPriority priority) where T : AJobPayload, new()
|
||||
{
|
||||
if (!Settings.RunBackEndJobs && typeof(T).ImplementsInterface(typeof(IBackEndJob))) return;
|
||||
if (!Settings.RunFrontEndJobs && typeof(T).ImplementsInterface(typeof(IFrontEndJob))) return;
|
||||
try
|
||||
{
|
||||
var jobs = (await Sql.GetAllJobs(span))
|
||||
.Where(j => j.Payload is T)
|
||||
.OrderByDescending(j => j.Created)
|
||||
.Take(10);
|
||||
|
||||
foreach (var job in jobs)
|
||||
{
|
||||
if (job.Started == null || job.Ended == null) return;
|
||||
if (DateTime.UtcNow - job.Ended < span) return;
|
||||
}
|
||||
await Sql.EnqueueJob(new Job
|
||||
{
|
||||
Priority = priority,
|
||||
Payload = new T()
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
Logger.Log(LogLevel.Error, ex, $"Error in JobScheduler when scheduling {typeof(T).Name}");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using Wabbajack.Common;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models
|
||||
{
|
||||
public class IndexedFile
|
||||
{
|
||||
public Hash Hash { get; set; }
|
||||
public string SHA256 { get; set; }
|
||||
public string SHA1 { get; set; }
|
||||
public string MD5 { get; set; }
|
||||
public string CRC { get; set; }
|
||||
public long Size { get; set; }
|
||||
public bool IsArchive { get; set; }
|
||||
public List<ChildFile> Children { get; set; } = new List<ChildFile>();
|
||||
}
|
||||
|
||||
public class ChildFile
|
||||
{
|
||||
public string Name;
|
||||
public string Extension;
|
||||
public Hash Hash;
|
||||
}
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models
|
||||
{
|
||||
public class IndexedFileWithChildren : IndexedFile
|
||||
{
|
||||
}
|
||||
}
|
||||
@ -1,45 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.JobQueue
|
||||
{
|
||||
public abstract class AJobPayload
|
||||
{
|
||||
public static List<Type> KnownSubTypes = new List<Type>
|
||||
{
|
||||
typeof(IndexJob),
|
||||
typeof(GetNexusUpdatesJob),
|
||||
typeof(UpdateModLists),
|
||||
typeof(EnqueueAllArchives),
|
||||
typeof(EnqueueAllGameFiles),
|
||||
typeof(UploadToCDN),
|
||||
typeof(IndexDynDOLOD),
|
||||
typeof(ReindexArchives),
|
||||
typeof(PatchArchive)
|
||||
};
|
||||
public static Dictionary<Type, string> TypeToName { get; set; }
|
||||
public static Dictionary<string, Type> NameToType { get; set; }
|
||||
|
||||
|
||||
public abstract string Description { get; }
|
||||
|
||||
public virtual bool UsesNexus { get; } = false;
|
||||
|
||||
public abstract Task<JobResult> Execute(SqlService sql,AppSettings settings);
|
||||
|
||||
protected abstract IEnumerable<object> PrimaryKey { get; }
|
||||
|
||||
public string PrimaryKeyString => string.Join("|", PrimaryKey.Cons(this.GetType().Name).Select(i => i.ToString()));
|
||||
|
||||
static AJobPayload()
|
||||
{
|
||||
NameToType = KnownSubTypes.ToDictionary(t => t.FullName.Substring(t.Namespace.Length + 1), t => t);
|
||||
TypeToName = NameToType.ToDictionary(k => k.Value, k => k.Key);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,28 +0,0 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.JobQueue
|
||||
{
|
||||
[JsonName("Job")]
|
||||
public class Job
|
||||
{
|
||||
public enum JobPriority : int
|
||||
{
|
||||
Low,
|
||||
Normal,
|
||||
High,
|
||||
}
|
||||
|
||||
public long Id { get; set; }
|
||||
public DateTime? Started { get; set; }
|
||||
public DateTime? Ended { get; set; }
|
||||
public DateTime Created { get; set; } = DateTime.Now;
|
||||
public JobPriority Priority { get; set; } = JobPriority.Normal;
|
||||
public JobResult Result { get; set; }
|
||||
public bool RequiresNexus { get; set; } = true;
|
||||
public AJobPayload Payload { get; set; }
|
||||
|
||||
public Job OnSuccess { get; set; }
|
||||
}
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
using System;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.JobQueue
|
||||
{
|
||||
[JsonName("JobResult")]
|
||||
public class JobResult
|
||||
{
|
||||
public JobResultType ResultType { get; set; }
|
||||
|
||||
public string Message { get; set; }
|
||||
|
||||
public string Stacktrace { get; set; }
|
||||
|
||||
public static JobResult Success()
|
||||
{
|
||||
return new JobResult { ResultType = JobResultType.Success };
|
||||
}
|
||||
|
||||
public static JobResult Error(Exception ex)
|
||||
{
|
||||
return new JobResult {ResultType = JobResultType.Error, Stacktrace = ex.ToString()};
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public enum JobResultType
|
||||
{
|
||||
Success,
|
||||
Error
|
||||
}
|
||||
}
|
||||
@ -1,88 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using System.Linq;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using Wabbajack.Lib.ModListRegistry;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.Jobs
|
||||
{
|
||||
[JsonName("EnqueueAllArchives")]
|
||||
public class EnqueueAllArchives : AJobPayload, IBackEndJob
|
||||
{
|
||||
public override string Description => "Add missing modlist archives to indexer";
|
||||
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
|
||||
{
|
||||
Utils.Log("Starting ModList indexing");
|
||||
var modlists = await ModlistMetadata.LoadFromGithub();
|
||||
|
||||
using (var queue = new WorkQueue())
|
||||
{
|
||||
foreach (var list in modlists)
|
||||
{
|
||||
try
|
||||
{
|
||||
await EnqueueFromList(sql, list, queue);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Utils.Log(ex.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return JobResult.Success();
|
||||
}
|
||||
|
||||
protected override IEnumerable<object> PrimaryKey => new object[0];
|
||||
|
||||
private static async Task EnqueueFromList(SqlService sql, ModlistMetadata list, WorkQueue queue)
|
||||
{
|
||||
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
|
||||
|
||||
if (list.NeedsDownload(modlistPath))
|
||||
{
|
||||
modlistPath.Delete();
|
||||
|
||||
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
|
||||
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
|
||||
await state.Download(modlistPath);
|
||||
}
|
||||
else
|
||||
{
|
||||
Utils.Log($"No changes detected from downloaded ModList");
|
||||
}
|
||||
|
||||
Utils.Log($"Loading {modlistPath}");
|
||||
|
||||
var installer = AInstaller.LoadFromFile(modlistPath);
|
||||
|
||||
var archives = installer.Archives;
|
||||
|
||||
Utils.Log($"Found {archives.Count} archives in {installer.Name} to index");
|
||||
var searching = archives.Select(a => a.Hash).ToHashSet();
|
||||
|
||||
Utils.Log($"Looking for missing archives");
|
||||
var knownArchives = await sql.FilterByExistingIndexedArchives(searching);
|
||||
|
||||
Utils.Log($"Found {knownArchives.Count} pre-existing archives");
|
||||
var missing = archives.Where(a => !knownArchives.Contains(a.Hash)).ToList();
|
||||
|
||||
Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs");
|
||||
|
||||
var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}, Priority = Job.JobPriority.Low});
|
||||
|
||||
Utils.Log($"Writing jobs to the database");
|
||||
|
||||
foreach (var job in jobs)
|
||||
await sql.EnqueueJob(job);
|
||||
|
||||
Utils.Log($"Done adding archives for {installer.Name}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,70 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using System.IO;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.Jobs
|
||||
{
|
||||
[JsonName("EnqueueAllGameFiles")]
|
||||
public class EnqueueAllGameFiles : AJobPayload, IBackEndJob
|
||||
{
|
||||
public override string Description { get => $"Enqueue all game files for indexing"; }
|
||||
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
|
||||
{
|
||||
using (var queue = new WorkQueue(4))
|
||||
{
|
||||
Utils.Log($"Finding game files to Index game files");
|
||||
var states = GameRegistry.Games.Values
|
||||
.Where(game => game.TryGetGameLocation() != default && game.MainExecutable != null)
|
||||
.SelectMany(game => game.GameLocation().EnumerateFiles()
|
||||
.Select(file => new GameFileSourceDownloader.State(game.InstalledVersion)
|
||||
{
|
||||
Game = game.Game,
|
||||
GameFile = file.RelativeTo(game.GameLocation()),
|
||||
}))
|
||||
.ToList();
|
||||
|
||||
var pks = states.Select(s => s.PrimaryKeyString).ToHashSet();
|
||||
Utils.Log($"Found {pks.Count} archives to cross-reference with the database");
|
||||
|
||||
var notFound = await sql.FilterByExistingPrimaryKeys(pks);
|
||||
|
||||
states = states.Where(s => notFound.Contains(s.PrimaryKeyString)).ToList();
|
||||
Utils.Log($"Found {states.Count} archives to index");
|
||||
|
||||
await states.PMap(queue, async state =>
|
||||
{
|
||||
var path = state.Game.MetaData().GameLocation().Combine(state.GameFile);
|
||||
Utils.Log($"Hashing Game file {path}");
|
||||
try
|
||||
{
|
||||
state.Hash = await path.FileHashAsync();
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
Utils.Log($"Unable to hash {path}");
|
||||
}
|
||||
});
|
||||
|
||||
var with_hash = states.Where(state => state.Hash != default).ToList();
|
||||
Utils.Log($"Inserting {with_hash.Count} jobs.");
|
||||
var jobs = states.Select(state => new IndexJob {Archive = new Archive(state) { Name = state.GameFile.FileName.ToString()}})
|
||||
.Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus})
|
||||
.ToList();
|
||||
|
||||
foreach (var job in jobs)
|
||||
await sql.EnqueueJob(job);
|
||||
|
||||
return JobResult.Success();
|
||||
}
|
||||
}
|
||||
|
||||
protected override IEnumerable<object> PrimaryKey => new object[0];
|
||||
}
|
||||
}
|
||||
@ -1,122 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.Jobs
|
||||
{
|
||||
[JsonName("GetNexusUpdatesJob")]
|
||||
public class GetNexusUpdatesJob : AJobPayload, IFrontEndJob
|
||||
{
|
||||
public override string Description => "Poll the Nexus for updated mods, and clean any references to those mods";
|
||||
|
||||
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
|
||||
{
|
||||
var api = await NexusApiClient.Get();
|
||||
|
||||
var gameTasks = GameRegistry.Games.Values
|
||||
.Where(game => game.NexusName != null)
|
||||
.Select(async game =>
|
||||
{
|
||||
var mods = await api.Get<List<NexusUpdateEntry>>(
|
||||
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m");
|
||||
|
||||
var entry = new NexusCacheData<List<NexusUpdateEntry>>();
|
||||
entry.Game = game.NexusName;
|
||||
entry.Path = $"/v1/games/{game.NexusName}/mods/updated.json?period=1m";
|
||||
entry.Data = mods;
|
||||
|
||||
return (game, mods);
|
||||
})
|
||||
.Select(async rTask =>
|
||||
{
|
||||
var (game, mods) = await rTask;
|
||||
return mods.Select(mod => new { game = game, mod = mod });
|
||||
}).ToList();
|
||||
|
||||
Utils.Log($"Getting update list for {gameTasks.Count} games");
|
||||
|
||||
var purge = (await Task.WhenAll(gameTasks))
|
||||
.SelectMany(i => i)
|
||||
.ToList();
|
||||
|
||||
Utils.Log($"Found {purge.Count} updated mods in the last month");
|
||||
using (var queue = new WorkQueue())
|
||||
{
|
||||
var collected = purge.Select(d =>
|
||||
{
|
||||
var a = d.mod.LatestFileUpdate.AsUnixTime();
|
||||
// Mod activity could hide files
|
||||
var b = d.mod.LastestModActivity.AsUnixTime();
|
||||
|
||||
return new {Game = d.game.Game, Date = (a > b ? a : b), ModId = d.mod.ModId};
|
||||
});
|
||||
|
||||
var purged = await collected.PMap(queue, async t =>
|
||||
{
|
||||
var resultA = await sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date);
|
||||
var resultB = await sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date);
|
||||
return resultA + resultB;
|
||||
});
|
||||
|
||||
Utils.Log($"Purged {purged.Sum()} cache entries");
|
||||
}
|
||||
|
||||
return JobResult.Success();
|
||||
}
|
||||
|
||||
protected override IEnumerable<object> PrimaryKey => new object[0];
|
||||
|
||||
public static DateTime LastNexusSync { get; set; } = DateTime.Now;
|
||||
public static async Task<long> UpdateNexusCacheFast(SqlService sql)
|
||||
{
|
||||
var results = await NexusUpdatesFeeds.GetUpdates();
|
||||
NexusApiClient client = null;
|
||||
long updated = 0;
|
||||
foreach (var result in results)
|
||||
{
|
||||
try
|
||||
{
|
||||
var purgedMods =
|
||||
await sql.DeleteNexusModFilesUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
|
||||
var purgedFiles =
|
||||
await sql.DeleteNexusModInfosUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
|
||||
|
||||
var totalPurged = purgedFiles + purgedMods;
|
||||
if (totalPurged > 0)
|
||||
Utils.Log($"Purged {totalPurged} cache items");
|
||||
|
||||
if (await sql.GetNexusModInfoString(result.Game, result.ModId) != null) continue;
|
||||
|
||||
// Lazily create the client
|
||||
client ??= await NexusApiClient.Get();
|
||||
|
||||
// Cache the info
|
||||
var files = await client.GetModFiles(result.Game, result.ModId, false);
|
||||
await sql.AddNexusModFiles(result.Game, result.ModId, result.TimeStamp, files);
|
||||
|
||||
var modInfo = await client.GetModInfo(result.Game, result.ModId);
|
||||
await sql.AddNexusModInfo(result.Game, result.ModId, result.TimeStamp, modInfo);
|
||||
updated++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Utils.Log($"Failed Nexus update for {result.Game} - {result.ModId} - {result.TimeStamp}");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (updated > 0)
|
||||
Utils.Log($"Primed {updated} nexus cache entries");
|
||||
|
||||
LastNexusSync = DateTime.Now;
|
||||
return updated;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
namespace Wabbajack.BuildServer.Models.Jobs
|
||||
{
|
||||
public interface IBackEndJob
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
namespace Wabbajack.BuildServer.Models.Jobs
|
||||
{
|
||||
public interface IFrontEndJob
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,67 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using HtmlAgilityPack;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.Jobs
|
||||
{
|
||||
/// <summary>
|
||||
/// DynDOLOD is really hosted on a STEP Forum post as a series of MEGA links. The Nexus URLs come and go
|
||||
/// but the real releases are on STEP. So let's keep that data fresh.
|
||||
/// </summary>
|
||||
[JsonName("IndexDynDOLOD")]
|
||||
public class IndexDynDOLOD : AJobPayload
|
||||
{
|
||||
public override string Description => "Queue MEGA URLs from the DynDOLOD Post";
|
||||
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
|
||||
{
|
||||
var doc = new HtmlDocument();
|
||||
var body = await new HttpClient().GetStringAsync(new Uri(
|
||||
"https://forum.step-project.com/topic/13894-dyndolod-beta-for-skyrim-special-edition-and-skyrim-vr-279/"));
|
||||
doc.LoadHtml(body);
|
||||
|
||||
var matches =
|
||||
doc.DocumentNode
|
||||
.Descendants()
|
||||
.Where(d=> d.NodeType == HtmlNodeType.Element && d.Attributes.Contains("href"))
|
||||
.Select(d => d.Attributes["href"].Value)
|
||||
.Select(m => Uri.TryCreate(m.ToString(), UriKind.Absolute, out var result) ? result : null)
|
||||
.Where(uri => uri != null && uri.Host == "mega.nz")
|
||||
.Select(url => new Job()
|
||||
{
|
||||
Payload = new IndexJob
|
||||
{
|
||||
Archive = new Archive(new MegaDownloader.State(url.ToString()))
|
||||
{
|
||||
Name = Guid.NewGuid() + ".7z",
|
||||
}
|
||||
}
|
||||
})
|
||||
.ToList();
|
||||
|
||||
|
||||
foreach (var job in matches)
|
||||
{
|
||||
var key = ((MegaDownloader.State)((IndexJob)job.Payload).Archive.State).PrimaryKeyString;
|
||||
var found = await sql.DownloadStateByPrimaryKey(key);
|
||||
if (found != null) continue;
|
||||
|
||||
Utils.Log($"Queuing {key} for indexing");
|
||||
await sql.EnqueueJob(job);
|
||||
}
|
||||
|
||||
return JobResult.Success();
|
||||
|
||||
}
|
||||
|
||||
protected override IEnumerable<object> PrimaryKey => new object[0];
|
||||
}
|
||||
}
|
||||
@ -1,76 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Alphaleonis.Win32.Filesystem;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using Wabbajack.VirtualFileSystem;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.Jobs
|
||||
{
|
||||
|
||||
[JsonName("IndexJob")]
|
||||
public class IndexJob : AJobPayload, IBackEndJob
|
||||
{
|
||||
public Archive Archive { get; set; }
|
||||
|
||||
public bool ForceIndex { get; set; }
|
||||
public override string Description => $"Index ${Archive.State.PrimaryKeyString} and save the download/file state";
|
||||
public override bool UsesNexus { get => Archive.State is NexusDownloader.State; }
|
||||
public Hash DownloadedHash { get; set; }
|
||||
|
||||
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
|
||||
{
|
||||
if (Archive.State is ManualDownloader.State)
|
||||
return JobResult.Success();
|
||||
|
||||
var pk = new List<object>();
|
||||
pk.Add(AbstractDownloadState.TypeToName[Archive.State.GetType()]);
|
||||
pk.AddRange(Archive.State.PrimaryKey);
|
||||
var pkStr = string.Join("|",pk.Select(p => p.ToString()));
|
||||
|
||||
var found = await sql.DownloadStateByPrimaryKey(pkStr);
|
||||
if (found != null && !ForceIndex)
|
||||
return JobResult.Success();
|
||||
|
||||
string fileName = Archive.Name ?? Guid.NewGuid().ToString();
|
||||
string folder = Guid.NewGuid().ToString();
|
||||
Utils.Log($"Indexer is downloading {fileName}");
|
||||
var downloadDest = settings.DownloadPath.Combine(folder, fileName);
|
||||
await Archive.State.Download(downloadDest);
|
||||
|
||||
using (var queue = new WorkQueue())
|
||||
{
|
||||
var vfs = new Context(queue, true);
|
||||
await vfs.AddRoot(settings.DownloadPath.Combine(folder));
|
||||
var archive = vfs.Index.ByRootPath.First().Value;
|
||||
|
||||
DownloadedHash = archive.Hash;
|
||||
|
||||
await sql.MergeVirtualFile(archive);
|
||||
|
||||
await sql.AddDownloadState(archive.Hash, Archive.State);
|
||||
|
||||
var to_path = settings.ArchivePath.Combine(
|
||||
$"{Path.GetFileName(fileName)}_{archive.Hash.ToHex()}_{Path.GetExtension(fileName)}");
|
||||
|
||||
if (to_path.Exists)
|
||||
downloadDest.Delete();
|
||||
else
|
||||
downloadDest.MoveTo(to_path);
|
||||
await settings.DownloadPath.Combine(folder).DeleteDirectory();
|
||||
|
||||
}
|
||||
return JobResult.Success();
|
||||
}
|
||||
|
||||
|
||||
protected override IEnumerable<object> PrimaryKey => Archive.State.PrimaryKey;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,75 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Alphaleonis.Win32.Filesystem;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
using Wabbajack.VirtualFileSystem;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.Jobs
|
||||
{
|
||||
[JsonName("ReindexArchives")]
|
||||
public class ReindexArchives : AJobPayload
|
||||
{
|
||||
public override string Description => "Reindex all files in the mod archive folder";
|
||||
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
|
||||
{
|
||||
using (var queue = new WorkQueue())
|
||||
{
|
||||
var files = settings.ArchivePath.EnumerateFiles()
|
||||
.Where(f => f.Extension != Consts.HashFileExtension)
|
||||
.ToList();
|
||||
var total_count = files.Count;
|
||||
int completed = 0;
|
||||
|
||||
|
||||
await files.PMap(queue, async file =>
|
||||
{
|
||||
try
|
||||
{
|
||||
Interlocked.Increment(ref completed);
|
||||
|
||||
if (await sql.HaveIndexdFile(await file.FileHashCachedAsync()))
|
||||
{
|
||||
Utils.Log($"({completed}/{total_count}) Skipping {file.FileName}, it's already indexed");
|
||||
return;
|
||||
}
|
||||
|
||||
var sub_folder = Guid.NewGuid().ToString();
|
||||
var folder = settings.DownloadPath.Combine(sub_folder);
|
||||
|
||||
Utils.Log($"({completed}/{total_count}) Copying {file}");
|
||||
folder.CreateDirectory();
|
||||
|
||||
Utils.Log($"({completed}/{total_count}) Copying {file}");
|
||||
file.CopyTo(folder.Combine(file.FileName));
|
||||
|
||||
Utils.Log($"({completed}/{total_count}) Analyzing {file}");
|
||||
var vfs = new Context(queue, true);
|
||||
await vfs.AddRoot(folder);
|
||||
|
||||
var root = vfs.Index.ByRootPath.First().Value;
|
||||
|
||||
Utils.Log($"({completed}/{total_count}) Ingesting {root.ThisAndAllChildren.Count()} files");
|
||||
|
||||
await sql.MergeVirtualFile(root);
|
||||
Utils.Log($"({completed}/{total_count}) Cleaning up {file}");
|
||||
await Utils.DeleteDirectory(folder);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Utils.Log(ex.ToString());
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
return JobResult.Success();
|
||||
}
|
||||
|
||||
protected override IEnumerable<object> PrimaryKey => new object[0];
|
||||
}
|
||||
}
|
||||
@ -1,193 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using Wabbajack.Lib.ModListRegistry;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
using Wabbajack.Lib.Validation;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.Jobs
|
||||
{
|
||||
[JsonName("UpdateModLists")]
|
||||
public class UpdateModLists : AJobPayload, IFrontEndJob
|
||||
{
|
||||
public override string Description => "Validate curated modlists";
|
||||
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
|
||||
{
|
||||
Utils.Log("Starting Modlist Validation");
|
||||
var modlists = await ModlistMetadata.LoadFromGithub();
|
||||
|
||||
using (var queue = new WorkQueue())
|
||||
{
|
||||
|
||||
var whitelists = new ValidateModlist();
|
||||
await whitelists.LoadListsFromGithub();
|
||||
|
||||
foreach (var list in modlists)
|
||||
{
|
||||
try
|
||||
{
|
||||
await ValidateList(sql, list, queue, whitelists);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Utils.Log(ex.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return JobResult.Success();
|
||||
}
|
||||
|
||||
protected override IEnumerable<object> PrimaryKey => new object[0];
|
||||
|
||||
private async Task ValidateList(SqlService sql, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
|
||||
{
|
||||
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
|
||||
|
||||
if (list.NeedsDownload(modlistPath))
|
||||
{
|
||||
modlistPath.Delete();
|
||||
|
||||
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
|
||||
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
|
||||
await state.Download(modlistPath);
|
||||
}
|
||||
else
|
||||
{
|
||||
Utils.Log($"No changes detected from downloaded modlist");
|
||||
}
|
||||
|
||||
|
||||
Utils.Log($"Loading {modlistPath}");
|
||||
|
||||
var installer = AInstaller.LoadFromFile(modlistPath);
|
||||
|
||||
Utils.Log($"{installer.Archives.Count} archives to validate");
|
||||
|
||||
await DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
|
||||
|
||||
|
||||
var validated = (await installer.Archives
|
||||
.PMap(queue, async archive =>
|
||||
{
|
||||
var isValid = await IsValid(sql, whitelists, archive);
|
||||
|
||||
return new DetailedStatusItem {IsFailing = !isValid, Archive = archive};
|
||||
}))
|
||||
.ToList();
|
||||
|
||||
|
||||
var status = new DetailedStatus
|
||||
{
|
||||
Name = list.Title,
|
||||
Archives = validated.OrderBy(v => v.Archive.Name).ToList(),
|
||||
DownloadMetaData = list.DownloadMetadata,
|
||||
HasFailures = validated.Any(v => v.IsFailing)
|
||||
};
|
||||
|
||||
var dto = new ModListStatus
|
||||
{
|
||||
Id = list.Links.MachineURL,
|
||||
Summary = new ModListSummary
|
||||
{
|
||||
Name = status.Name,
|
||||
MachineURL = list.Links?.MachineURL ?? status.Name,
|
||||
Checked = status.Checked,
|
||||
Failed = status.Archives.Count(a => a.IsFailing),
|
||||
Passed = status.Archives.Count(a => !a.IsFailing),
|
||||
},
|
||||
DetailedStatus = status,
|
||||
Metadata = list
|
||||
};
|
||||
Utils.Log(
|
||||
$"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed");
|
||||
await sql.UpdateModListStatus(dto);
|
||||
Utils.Log(
|
||||
$"Done updating {dto.Summary.Name}");
|
||||
|
||||
}
|
||||
|
||||
private async Task<bool> IsValid(SqlService sql, ValidateModlist whitelists, Archive archive)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!archive.State.IsWhitelisted(whitelists.ServerWhitelist)) return false;
|
||||
|
||||
try
|
||||
{
|
||||
if (archive.State is NexusDownloader.State state)
|
||||
{
|
||||
if (await ValidateNexusFast(sql, state)) return true;
|
||||
|
||||
}
|
||||
else if (archive.State is GoogleDriveDownloader.State)
|
||||
{
|
||||
// Disabled for now
|
||||
return true;
|
||||
}
|
||||
else if (archive.State is HTTPDownloader.State hstate &&
|
||||
hstate.Url.StartsWith("https://wabbajack"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (await archive.State.Verify(archive)) return true;
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// ignored
|
||||
}
|
||||
|
||||
Utils.Log($"{archive.State.PrimaryKeyString} is broken, looking for upgrade: {archive.Name}");
|
||||
var result = await ClientAPI.GetModUpgrade(archive.Hash);
|
||||
|
||||
if (result != null)
|
||||
{
|
||||
Utils.Log($"{archive.State.PrimaryKeyString} is broken, upgraded to {result.State.PrimaryKeyString} {result.Name}");
|
||||
return true;
|
||||
}
|
||||
|
||||
Utils.Log($"{archive.State.PrimaryKeyString} is broken, no alternative found");
|
||||
return false;
|
||||
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Utils.Log(ex.ToString());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> ValidateNexusFast(SqlService sql, NexusDownloader.State state)
|
||||
{
|
||||
try
|
||||
{
|
||||
var modFiles = await sql.GetModFiles(state.Game, state.ModID);
|
||||
|
||||
if (modFiles == null)
|
||||
{
|
||||
Utils.Log($"No Cache for {state.PrimaryKeyString} falling back to HTTP");
|
||||
var nexusApi = await NexusApiClient.Get();
|
||||
modFiles = await nexusApi.GetModFiles(state.Game, state.ModID);
|
||||
}
|
||||
|
||||
var found = modFiles.files
|
||||
.FirstOrDefault(file => file.file_id == state.FileID && file.category_name != null);
|
||||
return found != null;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,93 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using Alphaleonis.Win32.Filesystem;
|
||||
using FluentFTP;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using File = System.IO.File;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.Jobs
|
||||
{
|
||||
[JsonName("UploadToCDN")]
|
||||
public class UploadToCDN : AJobPayload
|
||||
{
|
||||
public override string Description => $"Push an uploaded file ({FileId}) to the CDN";
|
||||
|
||||
public Guid FileId { get; set; }
|
||||
|
||||
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
|
||||
{
|
||||
int retries = 0;
|
||||
TOP:
|
||||
var file = await sql.UploadedFileById(FileId);
|
||||
|
||||
if (settings.BunnyCDN_User == "TEST" && settings.BunnyCDN_Password == "TEST")
|
||||
{
|
||||
return JobResult.Success();
|
||||
}
|
||||
|
||||
using (var client = new FtpClient("storage.bunnycdn.com"))
|
||||
{
|
||||
client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);
|
||||
await client.ConnectAsync();
|
||||
using (var stream = File.OpenRead(Path.Combine("public", "files", file.MungedName)))
|
||||
{
|
||||
try
|
||||
{
|
||||
await client.UploadAsync(stream, file.MungedName, progress: new Progress((RelativePath)file.MungedName));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (retries > 10) throw;
|
||||
Utils.Log(ex.ToString());
|
||||
Utils.Log("Retrying FTP Upload");
|
||||
retries++;
|
||||
goto TOP;
|
||||
}
|
||||
}
|
||||
|
||||
await sql.EnqueueJob(new Job
|
||||
{
|
||||
Priority = Job.JobPriority.High,
|
||||
Payload = new IndexJob
|
||||
{
|
||||
Archive = new Archive(new HTTPDownloader.State(file.Uri))
|
||||
{
|
||||
Name = file.MungedName,
|
||||
Size = file.Size,
|
||||
Hash = file.Hash,
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
return JobResult.Success();
|
||||
}
|
||||
|
||||
protected override IEnumerable<object> PrimaryKey => new object[] {FileId};
|
||||
|
||||
public class Progress : IProgress<FluentFTP.FtpProgress>
|
||||
{
|
||||
private RelativePath _name;
|
||||
private DateTime LastUpdate = DateTime.UnixEpoch;
|
||||
|
||||
public Progress(RelativePath name)
|
||||
{
|
||||
_name = name;
|
||||
}
|
||||
public void Report(FtpProgress value)
|
||||
{
|
||||
if (DateTime.Now - LastUpdate <= TimeSpan.FromSeconds(5)) return;
|
||||
|
||||
Utils.Log($"Uploading {_name} - {value.Progress}% {(int)((value.TransferSpeed + 1) / 1024 / 1024)} MB/sec ETA: {value.ETA}");
|
||||
LastUpdate = DateTime.Now;
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.BuildServer.GraphQL;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
|
||||
|
||||
namespace Wabbajack.BuildServer.Models
|
||||
{
|
||||
public class Metric
|
||||
{
|
||||
public DateTime Timestamp { get; set; }
|
||||
public string Action { get; set; }
|
||||
public string Subject { get; set; }
|
||||
public string MetricsKey { get; set; }
|
||||
}
|
||||
}
|
||||
@ -1,45 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.ModListRegistry;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models
|
||||
{
|
||||
public class ModListStatus
|
||||
{
|
||||
public string Id { get; set; }
|
||||
public ModListSummary Summary { get; set; }
|
||||
|
||||
public ModlistMetadata Metadata { get; set; }
|
||||
public DetailedStatus DetailedStatus { get; set; }
|
||||
|
||||
public static IQueryable<ModListStatus> AllSummaries
|
||||
{
|
||||
get
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[JsonName("DetailedStatus")]
|
||||
public class DetailedStatus
|
||||
{
|
||||
public string Name { get; set; }
|
||||
public DateTime Checked { get; set; } = DateTime.UtcNow;
|
||||
public List<DetailedStatusItem> Archives { get; set; }
|
||||
public DownloadMetadata DownloadMetaData { get; set; }
|
||||
public bool HasFailures { get; set; }
|
||||
public string MachineName { get; set; }
|
||||
}
|
||||
|
||||
[JsonName("DetailedStatusItem")]
|
||||
public class DetailedStatusItem
|
||||
{
|
||||
public bool IsFailing { get; set; }
|
||||
public Archive Archive { get; set; }
|
||||
}
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models
|
||||
{
|
||||
public class NexusCacheData<T>
|
||||
{
|
||||
public string Path { get; set; }
|
||||
public T Data { get; set; }
|
||||
public string Game { get; set; }
|
||||
|
||||
public long ModId { get; set; }
|
||||
|
||||
public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow;
|
||||
|
||||
public string FileId { get; set; }
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models
|
||||
{
|
||||
public class NexusUpdateEntry
|
||||
{
|
||||
[JsonProperty("mod_id")]
|
||||
public long ModId { get; set; }
|
||||
[JsonProperty("latest_file_update")]
|
||||
public long LatestFileUpdate { get; set; }
|
||||
[JsonProperty("latest_mod_activity")]
|
||||
public long LastestModActivity { get; set; }
|
||||
}
|
||||
}
|
||||
@ -1,85 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using FluentFTP;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models
|
||||
{
|
||||
[JsonName("PatchArchive")]
|
||||
public class PatchArchive : AJobPayload
|
||||
{
|
||||
public override string Description => "Create a archive update patch";
|
||||
public Hash Src { get; set; }
|
||||
public string DestPK { get; set; }
|
||||
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
|
||||
{
|
||||
var srcPath = settings.PathForArchive(Src);
|
||||
var destHash = (await sql.DownloadStateByPrimaryKey(DestPK)).Hash;
|
||||
var destPath = settings.PathForArchive(destHash);
|
||||
|
||||
if (Src == destHash)
|
||||
return JobResult.Success();
|
||||
|
||||
Utils.Log($"Creating Patch ({Src} -> {DestPK})");
|
||||
var cdnPath = CdnPath(Src, destHash);
|
||||
cdnPath.Parent.CreateDirectory();
|
||||
|
||||
if (cdnPath.Exists)
|
||||
return JobResult.Success();
|
||||
|
||||
Utils.Log($"Calculating Patch ({Src} -> {DestPK})");
|
||||
await using var fs = cdnPath.Create();
|
||||
await using (var srcStream = srcPath.OpenRead())
|
||||
await using (var destStream = destPath.OpenRead())
|
||||
await using (var sigStream = cdnPath.WithExtension(Consts.OctoSig).Create())
|
||||
{
|
||||
OctoDiff.Create(destStream, srcStream, sigStream, fs);
|
||||
}
|
||||
fs.Position = 0;
|
||||
|
||||
Utils.Log($"Uploading Patch ({Src} -> {DestPK})");
|
||||
|
||||
int retries = 0;
|
||||
|
||||
if (settings.BunnyCDN_User == "TEST" && settings.BunnyCDN_Password == "TEST")
|
||||
{
|
||||
return JobResult.Success();
|
||||
}
|
||||
|
||||
TOP:
|
||||
using (var client = new FtpClient("storage.bunnycdn.com"))
|
||||
{
|
||||
client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);
|
||||
await client.ConnectAsync();
|
||||
try
|
||||
{
|
||||
await client.UploadAsync(fs, $"updates/{Src.ToHex()}_{destHash.ToHex()}", progress: new UploadToCDN.Progress(cdnPath.FileName));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (retries > 10) throw;
|
||||
Utils.Log(ex.ToString());
|
||||
Utils.Log("Retrying FTP Upload");
|
||||
retries++;
|
||||
goto TOP;
|
||||
}
|
||||
}
|
||||
|
||||
return JobResult.Success();
|
||||
|
||||
}
|
||||
|
||||
protected override IEnumerable<object> PrimaryKey => new object[] {Src, DestPK};
|
||||
|
||||
public static AbsolutePath CdnPath(Hash srcHash, Hash destHash)
|
||||
{
|
||||
return $"updates/{srcHash.ToHex()}_{destHash.ToHex()}".RelativeTo(AbsolutePath.EntryPoint);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using Wabbajack.Common;
|
||||
|
||||
namespace Wabbajack.BuildServer.Model.Models
|
||||
{
|
||||
public partial class ArchiveContent
|
||||
{
|
||||
public long Parent { get; set; }
|
||||
public long Child { get; set; }
|
||||
public RelativePath Path { get; set; }
|
||||
public byte[] PathHash { get; set; }
|
||||
}
|
||||
}
|
||||
@ -1,40 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.ComponentModel;
|
||||
using System.Data;
|
||||
|
||||
namespace Wabbajack.BuildServer.Model.Models
|
||||
{
|
||||
public static class Extensions
|
||||
{
|
||||
|
||||
public static DataTable ToDataTable(this IEnumerable<IndexedFile> coll)
|
||||
{
|
||||
var ut = new DataTable("dbo.IndexedFileType");
|
||||
ut.Columns.Add("Hash", typeof(long));
|
||||
ut.Columns.Add("Sha256", typeof(byte[]));
|
||||
ut.Columns.Add("Sha1", typeof(byte[]));
|
||||
ut.Columns.Add("Md5", typeof(byte[]));
|
||||
ut.Columns.Add("Crc32", typeof(int));
|
||||
ut.Columns.Add("Size", typeof(long));
|
||||
|
||||
foreach (var itm in coll)
|
||||
ut.Rows.Add(itm.Hash, itm.Sha256, itm.Sha1, itm.Md5, itm.Crc32, itm.Size);
|
||||
|
||||
return ut;
|
||||
}
|
||||
|
||||
public static DataTable ToDataTable(this IEnumerable<ArchiveContent> coll)
|
||||
{
|
||||
var ut = new DataTable("dbo.ArchiveContentType");
|
||||
ut.Columns.Add("Parent", typeof(long));
|
||||
ut.Columns.Add("Child", typeof(long));
|
||||
ut.Columns.Add("Path", typeof(string));
|
||||
|
||||
foreach (var itm in coll)
|
||||
ut.Rows.Add(itm.Parent, itm.Child, itm.Path);
|
||||
|
||||
return ut;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,15 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace Wabbajack.BuildServer.Model.Models
|
||||
{
|
||||
public class IndexedFile
|
||||
{
|
||||
public long Hash { get; set; }
|
||||
public byte[] Sha256 { get; set; }
|
||||
public byte[] Sha1 { get; set; }
|
||||
public byte[] Md5 { get; set; }
|
||||
public int Crc32 { get; set; }
|
||||
public long Size { get; set; }
|
||||
}
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
using System;
|
||||
|
||||
namespace Wabbajack.BuildServer.Model.Models.Results
|
||||
{
|
||||
public class AggregateMetric
|
||||
{
|
||||
public DateTime Date { get; set; }
|
||||
public string Subject { get; set; }
|
||||
public int Count { get; set; }
|
||||
}
|
||||
}
|
||||
@ -1,937 +0,0 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Data.SqlClient;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Dapper;
|
||||
using Microsoft.AspNetCore.Mvc.ModelBinding.Metadata;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Newtonsoft.Json;
|
||||
using Wabbajack.BuildServer.Model.Models.Results;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using Wabbajack.Lib.ModListRegistry;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
using Wabbajack.VirtualFileSystem;
|
||||
|
||||
namespace Wabbajack.BuildServer.Model.Models
|
||||
{
|
||||
public class SqlService
|
||||
{
|
||||
private AppSettings _settings;
|
||||
|
||||
public SqlService(AppSettings settings)
|
||||
{
|
||||
_settings = settings;
|
||||
|
||||
}
|
||||
|
||||
public async Task<SqlConnection> Open()
|
||||
{
|
||||
var conn = new SqlConnection(_settings.SqlConnection);
|
||||
await conn.OpenAsync();
|
||||
return conn;
|
||||
}
|
||||
|
||||
public async Task MergeVirtualFile(VirtualFile vfile)
|
||||
{
|
||||
var files = new List<IndexedFile>();
|
||||
var contents = new List<ArchiveContent>();
|
||||
|
||||
IngestFile(vfile, files, contents);
|
||||
|
||||
files = files.DistinctBy(f => f.Hash).ToList();
|
||||
contents = contents.DistinctBy(c => (c.Parent, c.Path)).ToList();
|
||||
|
||||
await using var conn = await Open();
|
||||
await conn.ExecuteAsync("dbo.MergeIndexedFiles", new {Files = files.ToDataTable(), Contents = contents.ToDataTable()},
|
||||
commandType: CommandType.StoredProcedure);
|
||||
}
|
||||
|
||||
private static void IngestFile(VirtualFile root, ICollection<IndexedFile> files, ICollection<ArchiveContent> contents)
|
||||
{
|
||||
files.Add(new IndexedFile
|
||||
{
|
||||
Hash = (long)root.Hash,
|
||||
Sha256 = root.ExtendedHashes.SHA256.FromHex(),
|
||||
Sha1 = root.ExtendedHashes.SHA1.FromHex(),
|
||||
Md5 = root.ExtendedHashes.MD5.FromHex(),
|
||||
Crc32 = BitConverter.ToInt32(root.ExtendedHashes.CRC.FromHex()),
|
||||
Size = root.Size
|
||||
});
|
||||
|
||||
if (root.Children == null) return;
|
||||
|
||||
foreach (var child in root.Children)
|
||||
{
|
||||
IngestFile(child, files, contents);
|
||||
|
||||
contents.Add(new ArchiveContent
|
||||
{
|
||||
Parent = (long)root.Hash,
|
||||
Child = (long)child.Hash,
|
||||
Path = (RelativePath)child.Name
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public async Task<bool> HaveIndexdFile(Hash hash)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var row = await conn.QueryAsync(@"SELECT * FROM IndexedFile WHERE Hash = @Hash",
|
||||
new {Hash = (long)hash});
|
||||
return row.Any();
|
||||
}
|
||||
|
||||
|
||||
|
||||
class ArchiveContentsResult
|
||||
{
|
||||
public long Parent { get; set; }
|
||||
public long Hash { get; set; }
|
||||
public long Size { get; set; }
|
||||
public string Path { get; set; }
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Get the name, path, hash and size of the file with the provided hash, and all files perhaps
|
||||
/// contained inside this file. Note: files themselves do not have paths, so the top level result
|
||||
/// will have a null path
|
||||
/// </summary>
|
||||
/// <param name="hash">The xxHash64 of the file to look up</param>
|
||||
/// <returns></returns>
|
||||
public async Task<IndexedVirtualFile> AllArchiveContents(long hash)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var files = await conn.QueryAsync<ArchiveContentsResult>(@"
|
||||
SELECT 0 as Parent, i.Hash, i.Size, null as Path FROM IndexedFile i WHERE Hash = @Hash
|
||||
UNION ALL
|
||||
SELECT a.Parent, i.Hash, i.Size, a.Path FROM AllArchiveContent a
|
||||
LEFT JOIN IndexedFile i ON i.Hash = a.Child
|
||||
WHERE TopParent = @Hash",
|
||||
new {Hash = hash});
|
||||
|
||||
var grouped = files.GroupBy(f => f.Parent).ToDictionary(f => f.Key, f=> (IEnumerable<ArchiveContentsResult>)f);
|
||||
|
||||
List<IndexedVirtualFile> Build(long parent)
|
||||
{
|
||||
if (grouped.TryGetValue(parent, out var children))
|
||||
{
|
||||
return children.Select(f => new IndexedVirtualFile
|
||||
{
|
||||
Name = (RelativePath)f.Path,
|
||||
Hash = Hash.FromLong(f.Hash),
|
||||
Size = f.Size,
|
||||
Children = Build(f.Hash)
|
||||
}).ToList();
|
||||
}
|
||||
return new List<IndexedVirtualFile>();
|
||||
}
|
||||
return Build(0).FirstOrDefault();
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<(RelativePath, Hash)>> GameFiles(Game game, Version version)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var files = await conn.QueryAsync<(RelativePath, Hash)>(
|
||||
@"SELECT Path, Hash FROM dbo.GameFiles where Game = @Game AND GameVersion = @GameVersion",
|
||||
new {Game = game.ToString(), GameVersion = version});
|
||||
|
||||
return files;
|
||||
|
||||
}
|
||||
|
||||
public async Task IngestAllMetrics(IEnumerable<Metric> allMetrics)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", allMetrics);
|
||||
}
|
||||
public async Task IngestMetric(Metric metric)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", metric);
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<AggregateMetric>> MetricsReport(string action)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
return (await conn.QueryAsync<AggregateMetric>(@"
|
||||
SELECT d.Date, d.GroupingSubject as Subject, Count(*) as Count FROM
|
||||
(select DISTINCT CONVERT(date, Timestamp) as Date, GroupingSubject, Action, MetricsKey from dbo.Metrics) m
|
||||
RIGHT OUTER JOIN
|
||||
(SELECT CONVERT(date, DATEADD(DAY, number + 1, dbo.MinMetricDate())) as Date, GroupingSubject, Action
|
||||
FROM master..spt_values
|
||||
CROSS JOIN (
|
||||
SELECT DISTINCT GroupingSubject, Action FROM dbo.Metrics
|
||||
WHERE MetricsKey is not null
|
||||
AND Subject != 'Default'
|
||||
AND TRY_CONVERT(uniqueidentifier, Subject) is null) as keys
|
||||
WHERE type = 'P'
|
||||
AND DATEADD(DAY, number+1, dbo.MinMetricDate()) <= dbo.MaxMetricDate()) as d
|
||||
ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action
|
||||
WHERE d.Action = @action
|
||||
AND d.Date >= DATEADD(month, -1, GETUTCDATE())
|
||||
group by d.Date, d.GroupingSubject, d.Action
|
||||
ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action}))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
#region JobRoutines
|
||||
|
||||
/// <summary>
|
||||
/// Enqueue a Job into the Job queue to be run at a later time
|
||||
/// </summary>
|
||||
/// <param name="job"></param>
|
||||
/// <returns></returns>
|
||||
public async Task EnqueueJob(Job job)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
await conn.ExecuteAsync(
|
||||
@"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETUTCDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)",
|
||||
new {
|
||||
job.Priority,
|
||||
PrimaryKeyString = job.Payload.PrimaryKeyString,
|
||||
Payload = job.Payload.ToJson(),
|
||||
OnSuccess = job.OnSuccess?.ToJson() ?? null});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Enqueue a Job into the Job queue to be run at a later time
|
||||
/// </summary>
|
||||
/// <param name="job"></param>
|
||||
/// <returns></returns>
|
||||
public async Task FinishJob(Job job)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
await conn.ExecuteAsync(
|
||||
@"UPDATE dbo.Jobs SET Ended = GETUTCDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id",
|
||||
new {
|
||||
job.Id,
|
||||
Success = job.Result.ResultType == JobResultType.Success,
|
||||
ResultContent = job.Result
|
||||
});
|
||||
|
||||
if (job.OnSuccess != null)
|
||||
await EnqueueJob(job.OnSuccess);
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Get a Job from the Job queue to run.
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public async Task<Job> GetJob()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var result = await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>(
|
||||
@"UPDATE jobs SET Started = GETUTCDATE(), RunBy = @RunBy
|
||||
WHERE ID in (SELECT TOP(1) ID FROM Jobs
|
||||
WHERE Started is NULL
|
||||
AND PrimaryKeyString NOT IN (SELECT PrimaryKeyString from jobs WHERE Started IS NOT NULL and Ended IS NULL)
|
||||
ORDER BY Priority DESC, Created);
|
||||
SELECT TOP(1) Id, Started, Ended, Created, Payload, Priority FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC",
|
||||
new {RunBy = Guid.NewGuid().ToString()});
|
||||
return result.Select(k =>
|
||||
new Job {
|
||||
Id = k.Item1,
|
||||
Started = k.Item2,
|
||||
Ended = k.Item3,
|
||||
Created = k.Item4,
|
||||
Payload = k.Item5,
|
||||
Priority = (Job.JobPriority)k.Item6
|
||||
}).FirstOrDefault();
|
||||
}
|
||||
|
||||
|
||||
public async Task<IEnumerable<Job>> GetRunningJobs()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var results =
|
||||
await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority FROM dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL ");
|
||||
return results.Select(k =>
|
||||
new Job {
|
||||
Id = k.Item1,
|
||||
Started = k.Item2,
|
||||
Ended = k.Item3,
|
||||
Created = k.Item4,
|
||||
Payload = k.Item5,
|
||||
Priority = (Job.JobPriority)k.Item6
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public async Task<IEnumerable<Job>> GetUnfinishedJobs()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var results =
|
||||
await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority from dbo.Jobs WHERE Ended IS NULL ");
|
||||
return results.Select(k =>
|
||||
new Job {
|
||||
Id = k.Item1,
|
||||
Started = k.Item2,
|
||||
Ended = k.Item3,
|
||||
Created = k.Item4,
|
||||
Payload = k.Item5,
|
||||
Priority = (Job.JobPriority)k.Item6
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
#region TypeMappers
|
||||
|
||||
static SqlService()
|
||||
{
|
||||
SqlMapper.AddTypeHandler(new HashMapper());
|
||||
SqlMapper.AddTypeHandler(new RelativePathMapper());
|
||||
SqlMapper.AddTypeHandler(new JsonMapper<AbstractDownloadState>());
|
||||
SqlMapper.AddTypeHandler(new JsonMapper<AJobPayload>());
|
||||
SqlMapper.AddTypeHandler(new JsonMapper<JobResult>());
|
||||
SqlMapper.AddTypeHandler(new JsonMapper<Job>());
|
||||
SqlMapper.AddTypeHandler(new VersionMapper());
|
||||
SqlMapper.AddTypeHandler(new GameMapper());
|
||||
}
|
||||
|
||||
public class JsonMapper<T> : SqlMapper.TypeHandler<T>
|
||||
{
|
||||
public override void SetValue(IDbDataParameter parameter, T value)
|
||||
{
|
||||
parameter.Value = value.ToJson();
|
||||
}
|
||||
|
||||
public override T Parse(object value)
|
||||
{
|
||||
return ((string)value).FromJsonString<T>();
|
||||
}
|
||||
}
|
||||
|
||||
public class RelativePathMapper : SqlMapper.TypeHandler<RelativePath>
|
||||
{
|
||||
public override void SetValue(IDbDataParameter parameter, RelativePath value)
|
||||
{
|
||||
parameter.Value = value.ToJson();
|
||||
}
|
||||
|
||||
public override RelativePath Parse(object value)
|
||||
{
|
||||
return (RelativePath)(string)value;
|
||||
}
|
||||
}
|
||||
|
||||
class HashMapper : SqlMapper.TypeHandler<Hash>
|
||||
{
|
||||
public override void SetValue(IDbDataParameter parameter, Hash value)
|
||||
{
|
||||
parameter.Value = (long)value;
|
||||
}
|
||||
|
||||
public override Hash Parse(object value)
|
||||
{
|
||||
return Hash.FromLong((long)value);
|
||||
}
|
||||
}
|
||||
|
||||
class VersionMapper : SqlMapper.TypeHandler<Version>
|
||||
{
|
||||
public override void SetValue(IDbDataParameter parameter, Version value)
|
||||
{
|
||||
parameter.Value = value.ToString();
|
||||
}
|
||||
|
||||
public override Version Parse(object value)
|
||||
{
|
||||
return Version.Parse((string)value);
|
||||
}
|
||||
}
|
||||
|
||||
class GameMapper : SqlMapper.TypeHandler<Game>
|
||||
{
|
||||
public override void SetValue(IDbDataParameter parameter, Game value)
|
||||
{
|
||||
parameter.Value = value.ToString();
|
||||
}
|
||||
|
||||
public override Game Parse(object value)
|
||||
{
|
||||
return GameRegistry.GetByFuzzyName((string)value).Game;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#endregion
|
||||
|
||||
public async Task AddUploadedFile(UploadedFile uf)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
await conn.ExecuteAsync(
|
||||
"INSERT INTO dbo.UploadedFiles (Id, Name, Size, UploadedBy, Hash, UploadDate, CDNName) VALUES " +
|
||||
"(@Id, @Name, @Size, @UploadedBy, @Hash, @UploadDate, @CDNName)",
|
||||
new
|
||||
{
|
||||
Id = uf.Id.ToString(),
|
||||
uf.Name,
|
||||
uf.Size,
|
||||
UploadedBy = uf.Uploader,
|
||||
Hash = (long)uf.Hash,
|
||||
uf.UploadDate,
|
||||
uf.CDNName
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public async Task<UploadedFile> UploadedFileById(Guid fileId)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
return await conn.QueryFirstAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles WHERE Id = @Id",
|
||||
new {Id = fileId.ToString()});
|
||||
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<UploadedFile>> AllUploadedFilesForUser(string user)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
return await conn.QueryAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles WHERE UploadedBy = @uploadedBy",
|
||||
new {UploadedBy = user});
|
||||
}
|
||||
|
||||
|
||||
public async Task<IEnumerable<UploadedFile>> AllUploadedFiles()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
return await conn.QueryAsync<UploadedFile>("SELECT Id, Name, Size, UploadedBy as Uploader, Hash, UploadDate, CDNName FROM dbo.UploadedFiles ORDER BY UploadDate DESC");
|
||||
}
|
||||
|
||||
public async Task DeleteUploadedFile(Guid dupId)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
await conn.ExecuteAsync("SELECT * FROM dbo.UploadedFiles WHERE Id = @id",
|
||||
new
|
||||
{
|
||||
Id = dupId.ToString()
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public async Task AddDownloadState(Hash hash, AbstractDownloadState state)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
await conn.ExecuteAsync("INSERT INTO dbo.DownloadStates (Id, Hash, PrimaryKey, IniState, JsonState) " +
|
||||
"VALUES (@Id, @Hash, @PrimaryKey, @IniState, @JsonState)",
|
||||
new
|
||||
{
|
||||
Id = state.PrimaryKeyString.StringSha256Hex().FromHex(),
|
||||
Hash = hash,
|
||||
PrimaryKey = state.PrimaryKeyString,
|
||||
IniState = string.Join("\n", state.GetMetaIni()),
|
||||
JsonState = state.ToJson()
|
||||
});
|
||||
}
|
||||
|
||||
public async Task<string> GetIniForHash(Hash id)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var results = await conn.QueryAsync<string>("SELECT IniState FROM dbo.DownloadStates WHERE Hash = @Hash",
|
||||
new {
|
||||
Hash = id
|
||||
});
|
||||
|
||||
return results.FirstOrDefault();
|
||||
|
||||
}
|
||||
|
||||
public async Task<bool> HaveIndexedArchivePrimaryKey(string key)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var results = await conn.QueryFirstOrDefaultAsync<string>(
|
||||
"SELECT PrimaryKey FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey",
|
||||
new {PrimaryKey = key});
|
||||
return results != null;
|
||||
}
|
||||
|
||||
public async Task AddNexusFileInfo(Game game, long modId, long fileId, DateTime lastCheckedUtc, NexusFileInfo data)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
|
||||
await conn.ExecuteAsync("INSERT INTO dbo.NexusFileInfos (Game, ModId, FileId, LastChecked, Data) VALUES " +
|
||||
"(@Game, @ModId, @FileId, @LastChecked, @Data)",
|
||||
new
|
||||
{
|
||||
Game = game.MetaData().NexusGameId,
|
||||
ModId = modId,
|
||||
FileId = fileId,
|
||||
LastChecked = lastCheckedUtc,
|
||||
Data = JsonConvert.SerializeObject(data)
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
public async Task AddNexusModInfo(Game game, long modId, DateTime lastCheckedUtc, ModInfo data)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
|
||||
await conn.ExecuteAsync(
|
||||
@"MERGE dbo.NexusModInfos AS Target
|
||||
USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
|
||||
ON Target.Game = Source.Game AND Target.ModId = Source.ModId
|
||||
WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
|
||||
WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
|
||||
new
|
||||
{
|
||||
Game = game.MetaData().NexusGameId,
|
||||
ModId = modId,
|
||||
LastChecked = lastCheckedUtc,
|
||||
Data = JsonConvert.SerializeObject(data)
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
public async Task AddNexusModFiles(Game game, long modId, DateTime lastCheckedUtc, NexusApiClient.GetModFilesResponse data)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
|
||||
await conn.ExecuteAsync(
|
||||
@"MERGE dbo.NexusModFiles AS Target
|
||||
USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
|
||||
ON Target.Game = Source.Game AND Target.ModId = Source.ModId
|
||||
WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
|
||||
WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
|
||||
new
|
||||
{
|
||||
Game = game.MetaData().NexusGameId,
|
||||
ModId = modId,
|
||||
LastChecked = lastCheckedUtc,
|
||||
Data = JsonConvert.SerializeObject(data)
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
public async Task<ModInfo> GetNexusModInfoString(Game game, long modId)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var result = await conn.QueryFirstOrDefaultAsync<string>(
|
||||
"SELECT Data FROM dbo.NexusModInfos WHERE Game = @Game AND @ModId = ModId",
|
||||
new {Game = game.MetaData().NexusGameId, ModId = modId});
|
||||
return result == null ? null : JsonConvert.DeserializeObject<ModInfo>(result);
|
||||
}
|
||||
|
||||
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(Game game, long modId)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var result = await conn.QueryFirstOrDefaultAsync<string>(
|
||||
"SELECT Data FROM dbo.NexusModFiles WHERE Game = @Game AND @ModId = ModId",
|
||||
new {Game = game.MetaData().NexusGameId, ModId = modId});
|
||||
return result == null ? null : JsonConvert.DeserializeObject<NexusApiClient.GetModFilesResponse>(result);
|
||||
}
|
||||
|
||||
#region ModLists
|
||||
public async Task<IEnumerable<ModListSummary>> GetModListSummaries()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var results = await conn.QueryAsync<string>("SELECT Summary from dbo.ModLists");
|
||||
return results.Select(s => s.FromJsonString<ModListSummary>()).ToList();
|
||||
}
|
||||
|
||||
public async Task<DetailedStatus> GetDetailedModlistStatus(string machineUrl)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var result = await conn.QueryFirstOrDefaultAsync<string>("SELECT DetailedStatus from dbo.ModLists WHERE MachineURL = @MachineURL",
|
||||
new
|
||||
{
|
||||
machineUrl
|
||||
});
|
||||
return result.FromJsonString<DetailedStatus>();
|
||||
}
|
||||
public async Task<List<DetailedStatus>> GetDetailedModlistStatuses()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var results = await conn.QueryAsync<string>("SELECT DetailedStatus from dbo.ModLists");
|
||||
return results.Select(s => s.FromJsonString<DetailedStatus>()).ToList();
|
||||
}
|
||||
|
||||
|
||||
|
||||
#endregion
|
||||
|
||||
#region Logins
|
||||
public async Task<string> AddLogin(string name)
|
||||
{
|
||||
var key = NewAPIKey();
|
||||
await using var conn = await Open();
|
||||
|
||||
|
||||
await conn.ExecuteAsync("INSERT INTO dbo.ApiKeys (Owner, ApiKey) VALUES (@Owner, @ApiKey)",
|
||||
new {Owner = name, ApiKey = key});
|
||||
return key;
|
||||
}
|
||||
|
||||
public static string NewAPIKey()
|
||||
{
|
||||
var arr = new byte[128];
|
||||
new Random().NextBytes(arr);
|
||||
return arr.ToHex();
|
||||
}
|
||||
|
||||
public async Task<string> LoginByAPIKey(string key)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var result = await conn.QueryAsync<string>(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @ApiKey",
|
||||
new {ApiKey = key});
|
||||
return result.FirstOrDefault();
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<(string Owner, string Key)>> GetAllUserKeys()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var result = await conn.QueryAsync<(string Owner, string Key)>("SELECT Owner, ApiKey FROM dbo.ApiKeys");
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
#endregion
|
||||
|
||||
#region Auto-healing routines
|
||||
|
||||
public async Task<Archive> GetNexusStateByHash(Hash startingHash)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var result = await conn.QueryFirstOrDefaultAsync<string>(@"SELECT JsonState FROM dbo.DownloadStates
|
||||
WHERE Hash = @hash AND PrimaryKey like 'NexusDownloader+State|%'",
|
||||
new {Hash = (long)startingHash});
|
||||
return result == null ? null : new Archive(result.FromJsonString<AbstractDownloadState>())
|
||||
{
|
||||
Hash = startingHash
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<Archive> GetStateByHash(Hash startingHash)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var result = await conn.QueryFirstOrDefaultAsync<(string, long)>(@"SELECT JsonState, indexed.Size FROM dbo.DownloadStates state
|
||||
LEFT JOIN dbo.IndexedFile indexed ON indexed.Hash = state.Hash
|
||||
WHERE state.Hash = @hash",
|
||||
new {Hash = (long)startingHash});
|
||||
return result == default ? null : new Archive(result.Item1.FromJsonString<AbstractDownloadState>())
|
||||
{
|
||||
Hash = startingHash,
|
||||
Size = result.Item2
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<Archive> DownloadStateByPrimaryKey(string primaryKey)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var result = await conn.QueryFirstOrDefaultAsync<(long Hash, string State)>(@"SELECT Hash, JsonState FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey",
|
||||
new {PrimaryKey = primaryKey});
|
||||
return result == default ? null : new Archive(result.State.FromJsonString<AbstractDownloadState>())
|
||||
{
|
||||
Hash = Hash.FromLong(result.Hash)
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Returns a hashset the only contains hashes from the input that do not exist in IndexedArchives
|
||||
/// </summary>
|
||||
/// <param name="searching"></param>
|
||||
/// <returns></returns>
|
||||
/// <exception cref="NotImplementedException"></exception>
|
||||
public async Task<HashSet<Hash>> FilterByExistingIndexedArchives(HashSet<Hash> searching)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var found = await conn.QueryAsync<long>("SELECT Hash from dbo.IndexedFile WHERE Hash in @Hashes",
|
||||
new {Hashes = searching.Select(h => (long)h)});
|
||||
return searching.Except(found.Select(h => Hash.FromLong(h)).ToHashSet()).ToHashSet();
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Returns a hashset the only contains primary keys from the input that do not exist in IndexedArchives
|
||||
/// </summary>
|
||||
/// <param name="searching"></param>
|
||||
/// <returns></returns>
|
||||
/// <exception cref="NotImplementedException"></exception>
|
||||
public async Task<HashSet<string>> FilterByExistingPrimaryKeys(HashSet<string> pks)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var results = new List<string>();
|
||||
|
||||
foreach (var partition in pks.Partition(512))
|
||||
{
|
||||
var found = await conn.QueryAsync<string>(
|
||||
"SELECT Hash from dbo.DownloadStates WHERE PrimaryKey in @PrimaryKeys",
|
||||
new {PrimaryKeys = partition.ToList()});
|
||||
results.AddRange(found);
|
||||
}
|
||||
|
||||
return pks.Except(results.ToHashSet()).ToHashSet();
|
||||
}
|
||||
|
||||
public async Task<long> DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var deleted = await conn.ExecuteScalarAsync<long>(
|
||||
@"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
|
||||
SELECT @@ROWCOUNT AS Deleted",
|
||||
new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
|
||||
return deleted;
|
||||
}
|
||||
|
||||
public async Task<long> DeleteNexusModFilesUpdatedBeforeDate(Game game, long modId, DateTime date)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var deleted = await conn.ExecuteScalarAsync<long>(
|
||||
@"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
|
||||
SELECT @@ROWCOUNT AS Deleted",
|
||||
new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
|
||||
return deleted;
|
||||
}
|
||||
|
||||
public async Task UpdateModListStatus(ModListStatus dto)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
await using var tran = await conn.BeginTransactionAsync();
|
||||
|
||||
await conn.ExecuteAsync(@"DELETE FROM dbo.ModLists Where MachineUrl = @MachineUrl",
|
||||
new {MachineUrl = metadata.Links.MachineURL}, tran);
|
||||
|
||||
await conn.ExecuteAsync(
|
||||
@"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList) VALUES (@MachineUrl, @Hash, @Metadata, @ModList)",
|
||||
new
|
||||
{
|
||||
MachineUrl = metadata.Links.MachineURL,
|
||||
Hash = hash,
|
||||
MetaData = metadata.ToJson(),
|
||||
ModList = modlist.ToJson()
|
||||
}, tran);
|
||||
|
||||
var entries = modlist.Archives.Select(a =>
|
||||
new
|
||||
{
|
||||
MachineUrl = metadata.Links.MachineURL,
|
||||
Hash = a.Hash,
|
||||
Size = a.Size,
|
||||
State = a.State.ToJson(),
|
||||
PrimaryKeyString = a.State.PrimaryKeyString
|
||||
}).ToArray();
|
||||
|
||||
await conn.ExecuteAsync(@"DELETE FROM dbo.ModListArchives WHERE MachineURL = @machineURL",
|
||||
new {MachineUrl = metadata.Links.MachineURL}, tran);
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
await conn.ExecuteAsync(
|
||||
"INSERT INTO dbo.ModListArchives (MachineURL, Hash, Size, PrimaryKeyString, State) VALUES (@MachineURL, @Hash, @Size, @PrimaryKeyString, @State)",
|
||||
entry, tran);
|
||||
}
|
||||
|
||||
await tran.CommitAsync();
|
||||
}
|
||||
|
||||
public async Task<bool> HaveIndexedModlist(string machineUrl, Hash hash)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var result = await conn.QueryFirstOrDefaultAsync<string>(
|
||||
"SELECT MachineURL from dbo.Modlists WHERE MachineURL = @MachineUrl AND Hash = @Hash",
|
||||
new {MachineUrl = machineUrl, Hash = hash});
|
||||
return result != null;
|
||||
}
|
||||
|
||||
public async Task<List<Archive>> GetNonNexusModlistArchives()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var results = await conn.QueryAsync<(Hash Hash, long Size, string State)>(
|
||||
@"SELECT Hash, Size, State FROM dbo.ModListArchives WHERE PrimaryKeyString NOT LIKE 'NexusDownloader+State|%'");
|
||||
return results.Select(r => new Archive (r.State.FromJsonString<AbstractDownloadState>())
|
||||
{
|
||||
Size = r.Size,
|
||||
Hash = r.Hash,
|
||||
|
||||
}).ToList();}
|
||||
|
||||
public async Task UpdateNonNexusModlistArchivesStatus(IEnumerable<(Archive Archive, bool IsValid)> results)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var trans = await conn.BeginTransactionAsync();
|
||||
await conn.ExecuteAsync("DELETE FROM dbo.ModlistArchiveStatus;", transaction:trans);
|
||||
|
||||
foreach (var itm in results.DistinctBy(itm => (itm.Archive.Hash, itm.Archive.State.PrimaryKeyString)))
|
||||
{
|
||||
await conn.ExecuteAsync(
|
||||
@"INSERT INTO dbo.ModlistArchiveStatus (PrimaryKeyStringHash, PrimaryKeyString, Hash, IsValid)
|
||||
VALUES (HASHBYTES('SHA2_256', @PrimaryKeyString), @PrimaryKeyString, @Hash, @IsValid)", new
|
||||
{
|
||||
PrimaryKeyString = itm.Archive.State.PrimaryKeyString,
|
||||
Hash = itm.Archive.Hash,
|
||||
IsValid = itm.IsValid
|
||||
}, trans);
|
||||
}
|
||||
|
||||
await trans.CommitAsync();
|
||||
}
|
||||
|
||||
public async Task<ValidationData> GetValidationData()
|
||||
{
|
||||
var nexusFiles = AllNexusFiles();
|
||||
var archiveStatus = AllModListArchivesStatus();
|
||||
var modLists = AllModLists();
|
||||
var archivePatches = AllArchivePatches();
|
||||
|
||||
return new ValidationData
|
||||
{
|
||||
NexusFiles = await nexusFiles,
|
||||
ArchiveStatus = await archiveStatus,
|
||||
ModLists = await modLists,
|
||||
ArchivePatches = await archivePatches
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<Dictionary<(string PrimaryKeyString, Hash Hash), bool>> AllModListArchivesStatus()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var results =
|
||||
await conn.QueryAsync<(string, Hash, bool)>(
|
||||
@"SELECT PrimaryKeyString, Hash, IsValid FROM dbo.ModListArchiveStatus");
|
||||
return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3);
|
||||
}
|
||||
|
||||
public async Task<HashSet<(long NexusGameId, long ModId, long FileId)>> AllNexusFiles()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var results = await conn.QueryAsync<(long, long, long)>(@"SELECT Game, ModId, p.file_id
|
||||
FROM [NexusModFiles] files
|
||||
CROSS APPLY
|
||||
OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p
|
||||
WHERE p.category is not null");
|
||||
return results.ToHashSet();
|
||||
}
|
||||
|
||||
public async Task<List<(ModlistMetadata, ModList)>> AllModLists()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var results = await conn.QueryAsync<(string, string)>(@"SELECT Metadata, ModList FROM dbo.ModLists");
|
||||
return results.Select(m => (m.Item1.FromJsonString<ModlistMetadata>(), m.Item2.FromJsonString<ModList>())).ToList();
|
||||
}
|
||||
|
||||
public class ValidationData
|
||||
{
|
||||
public HashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; }
|
||||
public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
|
||||
public List<(ModlistMetadata Metadata, ModList ModList)> ModLists { get; set; }
|
||||
public List<ArchivePatch> ArchivePatches { get; set; }
|
||||
}
|
||||
|
||||
|
||||
#region ArchivePatches
|
||||
|
||||
public class ArchivePatch
|
||||
{
|
||||
public Hash SrcHash { get; set; }
|
||||
public AbstractDownloadState SrcState { get; set; }
|
||||
public Hash DestHash { get; set; }
|
||||
public AbstractDownloadState DestState { get; set; }
|
||||
|
||||
public RelativePath DestDownload { get; set; }
|
||||
public RelativePath SrcDownload { get; set; }
|
||||
public Uri CDNPath { get; set; }
|
||||
}
|
||||
|
||||
public async Task UpsertArchivePatch(ArchivePatch patch)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
|
||||
await using var trans = conn.BeginTransaction();
|
||||
await conn.ExecuteAsync(@"DELETE FROM dbo.ArchivePatches
|
||||
WHERE SrcHash = @SrcHash
|
||||
AND DestHash = @DestHash
|
||||
AND SrcPrimaryKeyStringHash = HASHBYTES('SHA2-256', @SrcPrimaryKeyString)
|
||||
AND DestPrimaryKeyStringHash = HASHBYTES('SHA2-256', @DestPrimaryKeyString)",
|
||||
new
|
||||
{
|
||||
SrcHash = patch.SrcHash,
|
||||
DestHash = patch.DestHash,
|
||||
SrcPrimaryKeyString = patch.SrcState.PrimaryKeyString,
|
||||
DestPrimaryKeyString = patch.DestState.PrimaryKeyString
|
||||
}, trans);
|
||||
|
||||
await conn.ExecuteAsync(@"INSERT INTO dbo.ArchivePatches
|
||||
(SrcHash, SrcPrimaryKeyString, SrcPrimaryKeyStringHash, SrcState,
|
||||
DestHash, DestPrimaryKeyString, DestPrimaryKeyStringHash, DestState,
|
||||
|
||||
SrcDownload, DestDownload, CDNPath)
|
||||
VALUES (@SrcHash, @SrcPrimaryKeyString, HASHBYTES('SHA2-256', @SrcPrimaryKeyString), @SrcState,
|
||||
@DestHash, @DestPrimaryKeyString, HASHBYTES('SHA2-256', @DestPrimaryKeyString), @DestState,
|
||||
@SrcDownload, @DestDownload, @CDNPAth)",
|
||||
new
|
||||
{
|
||||
SrcHash = patch.SrcHash,
|
||||
DestHash = patch.DestHash,
|
||||
SrcPrimaryKeyString = patch.SrcState.PrimaryKeyString,
|
||||
DestPrimaryKeyString = patch.DestState.PrimaryKeyString,
|
||||
SrcState = patch.SrcState.ToJson(),
|
||||
DestState = patch.DestState.ToString(),
|
||||
DestDownload = patch.DestDownload,
|
||||
SrcDownload = patch.SrcDownload,
|
||||
CDNPath = patch.CDNPath
|
||||
}, trans);
|
||||
|
||||
await trans.CommitAsync();
|
||||
}
|
||||
|
||||
public async Task<List<ArchivePatch>> AllArchivePatches()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
|
||||
var results =
|
||||
await conn.QueryAsync<(Hash, AbstractDownloadState, Hash, AbstractDownloadState, RelativePath, RelativePath, Uri)>(
|
||||
@"SELECT SrcHash, SrcState, DestHash, DestState, SrcDownload, DestDownload, CDNPath FROM dbo.ArchivePatches");
|
||||
return results.Select(a => new ArchivePatch
|
||||
{
|
||||
SrcHash = a.Item1,
|
||||
SrcState = a.Item2,
|
||||
DestHash = a.Item3,
|
||||
DestState = a.Item4,
|
||||
SrcDownload = a.Item5,
|
||||
DestDownload = a.Item6,
|
||||
CDNPath = a.Item7
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
|
||||
#endregion
|
||||
|
||||
public async Task<IEnumerable<Job>> GetAllJobs(TimeSpan from)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var results =
|
||||
await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority from dbo.Jobs WHERE Created >= @FromTime ",
|
||||
new {FromTime = DateTime.UtcNow - from});
|
||||
return results.Select(k =>
|
||||
new Job {
|
||||
Id = k.Item1,
|
||||
Started = k.Item2,
|
||||
Ended = k.Item3,
|
||||
Created = k.Item4,
|
||||
Payload = k.Item5,
|
||||
Priority = (Job.JobPriority)k.Item6
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,22 +0,0 @@
|
||||
using System;
|
||||
using Wabbajack.Common;
|
||||
using Path = Alphaleonis.Win32.Filesystem.Path;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models
|
||||
{
|
||||
public class UploadedFile
|
||||
{
|
||||
public Guid Id { get; set; }
|
||||
public string Name { get; set; }
|
||||
public long Size { get; set; }
|
||||
public Hash Hash { get; set; }
|
||||
public string Uploader { get; set; }
|
||||
public DateTime UploadDate { get; set; } = DateTime.UtcNow;
|
||||
|
||||
public string CDNName { get; set; }
|
||||
|
||||
public string MungedName => $"{Path.GetFileNameWithoutExtension(Name)}-{Id}{Path.GetExtension(Name)}";
|
||||
|
||||
public string Uri => CDNName == null ? $"https://wabbajack.b-cdn.net/{MungedName}" : $"https://{CDNName}.b-cdn.net/{MungedName}";
|
||||
}
|
||||
}
|
||||
@ -1,47 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Wabbajack.BuildServer
|
||||
{
|
||||
public class Program
|
||||
{
|
||||
public static void Main(string[] args)
|
||||
{
|
||||
CreateHostBuilder(args, false).Build().Run();
|
||||
}
|
||||
|
||||
public static IHostBuilder CreateHostBuilder(string[] args, bool testMode) =>
|
||||
Host.CreateDefaultBuilder(args)
|
||||
.ConfigureWebHostDefaults(webBuilder =>
|
||||
{
|
||||
webBuilder.UseStartup<Startup>()
|
||||
.UseKestrel(options =>
|
||||
{
|
||||
options.Listen(IPAddress.Any, testMode ? 8080 : 80);
|
||||
if (!testMode)
|
||||
{
|
||||
options.Listen(IPAddress.Any, 443, listenOptions =>
|
||||
{
|
||||
using (var store = new X509Store(StoreName.My))
|
||||
{
|
||||
store.Open(OpenFlags.ReadOnly);
|
||||
var cert = store.Certificates.Find(X509FindType.FindBySubjectName,
|
||||
"build.wabbajack.org", true)[0];
|
||||
listenOptions.UseHttps(cert);
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
options.Limits.MaxRequestBodySize = null;
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -1,27 +0,0 @@
|
||||
{
|
||||
"iisSettings": {
|
||||
"windowsAuthentication": false,
|
||||
"anonymousAuthentication": true,
|
||||
"iisExpress": {
|
||||
"applicationUrl": "http://localhost:51578/",
|
||||
"sslPort": 0
|
||||
}
|
||||
},
|
||||
"profiles": {
|
||||
"IIS Express": {
|
||||
"commandName": "IISExpress",
|
||||
"launchBrowser": true,
|
||||
"environmentVariables": {
|
||||
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||
}
|
||||
},
|
||||
"Wabbajack.BuildServer": {
|
||||
"commandName": "Project",
|
||||
"launchBrowser": true,
|
||||
"environmentVariables": {
|
||||
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||
},
|
||||
"applicationUrl": "http://localhost:51579/"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,155 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using Alphaleonis.Win32.Filesystem;
|
||||
using GraphiQl;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.HttpsPolicy;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.AspNetCore.Authentication;
|
||||
using Microsoft.AspNetCore.Authentication.Cookies;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Http.Features;
|
||||
using Microsoft.AspNetCore.Mvc.Authorization;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.OpenApi.Models;
|
||||
using Newtonsoft.Json;
|
||||
using Swashbuckle.AspNetCore.Swagger;
|
||||
using Wabbajack.BuildServer.Controllers;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Microsoft.AspNetCore.Mvc.NewtonsoftJson;
|
||||
using Microsoft.AspNetCore.StaticFiles;
|
||||
using Microsoft.Extensions.FileProviders;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Directory = System.IO.Directory;
|
||||
|
||||
|
||||
namespace Wabbajack.BuildServer
|
||||
{
|
||||
public class TestStartup : Startup
|
||||
{
|
||||
public TestStartup(IConfiguration configuration) : base(configuration)
|
||||
{
|
||||
}
|
||||
}
|
||||
public class Startup
|
||||
{
|
||||
public Startup(IConfiguration configuration)
|
||||
{
|
||||
Configuration = configuration;
|
||||
}
|
||||
|
||||
public IConfiguration Configuration { get; }
|
||||
|
||||
// This method gets called by the runtime. Use this method to add services to the container.
|
||||
public void ConfigureServices(IServiceCollection services)
|
||||
{
|
||||
services.AddSwaggerGen(c =>
|
||||
{
|
||||
c.SwaggerDoc("v1", new OpenApiInfo {Title = "Wabbajack Build API", Version = "v1"});
|
||||
});
|
||||
|
||||
services.AddAuthentication(options =>
|
||||
{
|
||||
options.DefaultAuthenticateScheme = ApiKeyAuthenticationOptions.DefaultScheme;
|
||||
options.DefaultChallengeScheme = ApiKeyAuthenticationOptions.DefaultScheme;
|
||||
})
|
||||
.AddApiKeySupport(options => {});
|
||||
|
||||
services.Configure<FormOptions>(x =>
|
||||
{
|
||||
x.ValueLengthLimit = int.MaxValue;
|
||||
x.MultipartBodyLengthLimit = int.MaxValue;
|
||||
});
|
||||
|
||||
services.AddSingleton<JobManager>();
|
||||
services.AddSingleton<AppSettings>();
|
||||
services.AddSingleton<SqlService>();
|
||||
services.AddMvc();
|
||||
services.AddControllers()
|
||||
.AddNewtonsoftJson(o =>
|
||||
{
|
||||
|
||||
o.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore;
|
||||
});
|
||||
|
||||
|
||||
}
|
||||
|
||||
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
|
||||
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
|
||||
{
|
||||
if (env.IsDevelopment())
|
||||
{
|
||||
app.UseDeveloperExceptionPage();
|
||||
}
|
||||
|
||||
if (!(this is TestStartup))
|
||||
app.UseHttpsRedirection();
|
||||
|
||||
app.UseGraphiQl();
|
||||
app.UseDeveloperExceptionPage();
|
||||
|
||||
var provider = new FileExtensionContentTypeProvider();
|
||||
provider.Mappings[".rar"] = "application/x-rar-compressed";
|
||||
provider.Mappings[".7z"] = "application/x-7z-compressed";
|
||||
provider.Mappings[".zip"] = "application/zip";
|
||||
provider.Mappings[".wabbajack"] = "application/zip";
|
||||
app.UseStaticFiles();
|
||||
|
||||
app.UseSwagger();
|
||||
app.UseSwaggerUI(c =>
|
||||
{
|
||||
c.SwaggerEndpoint("/swagger/v1/swagger.json", "Wabbajack Build API");
|
||||
c.RoutePrefix = string.Empty;
|
||||
});
|
||||
app.UseRouting();
|
||||
|
||||
app.UseJobManager();
|
||||
app.UseAuthentication();
|
||||
app.UseAuthorization();
|
||||
|
||||
app.Use(next =>
|
||||
{
|
||||
return async context =>
|
||||
{
|
||||
var stopWatch = new Stopwatch();
|
||||
stopWatch.Start();
|
||||
context.Response.OnStarting(() =>
|
||||
{
|
||||
stopWatch.Stop();
|
||||
var headers = context.Response.Headers;
|
||||
headers.Add("Access-Control-Allow-Origin", "*");
|
||||
headers.Add("Access-Control-Allow-Methods", "POST, GET");
|
||||
headers.Add("Access-Control-Allow-Headers", "Accept, Origin, Content-type");
|
||||
headers.Add("X-ResponseTime-Ms", stopWatch.ElapsedMilliseconds.ToString());
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
await next(context);
|
||||
};
|
||||
});
|
||||
|
||||
app.UseFileServer(new FileServerOptions
|
||||
{
|
||||
FileProvider = new PhysicalFileProvider(
|
||||
Path.Combine(Directory.GetCurrentDirectory(), "public")),
|
||||
StaticFileOptions = {ServeUnknownFileTypes = true},
|
||||
|
||||
});
|
||||
|
||||
app.UseEndpoints(endpoints =>
|
||||
{
|
||||
endpoints.MapControllers();
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,97 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<UserSecretsId>aspnet-Wabbajack.BuildServer-6E798B30-DB04-4436-BE65-F043AF37B314</UserSecretsId>
|
||||
<WebProject_DirectoryAccessLevelKey>0</WebProject_DirectoryAccessLevelKey>
|
||||
<PublishReadyToRun>true</PublishReadyToRun>
|
||||
<PublishSingleFile>true</PublishSingleFile>
|
||||
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
|
||||
<AssemblyVersion>2.0.3.0</AssemblyVersion>
|
||||
<FileVersion>2.0.3.0</FileVersion>
|
||||
<Copyright>Copyright © 2019-2020</Copyright>
|
||||
<Description>Server component for Wabbajack</Description>
|
||||
<AssemblyName>BuildServer</AssemblyName>
|
||||
<RootNamespace>BuildServer</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="CsvHelper" Version="15.0.5" />
|
||||
<PackageReference Include="Dapper" Version="2.0.35" />
|
||||
<PackageReference Include="FluentFTP" Version="32.4.0" />
|
||||
<PackageReference Include="graphiql" Version="2.0.0" />
|
||||
<PackageReference Include="GraphQL" Version="3.0.0-preview-1352" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.1" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
|
||||
<PackageReference Include="Microsoft.OpenApi" Version="1.1.4" />
|
||||
<PackageReference Include="Nettle" Version="1.3.0" />
|
||||
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.4.1" />
|
||||
<PackageReference Include="System.Data.SqlClient" Version="4.8.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
|
||||
<ProjectReference Include="..\Wabbajack.Lib\Wabbajack.Lib.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Remove="chrome_elf.dll" />
|
||||
<None Remove="d3dcompiler_47.dll" />
|
||||
<None Remove="libGLESv2.dll" />
|
||||
<None Remove="CefSharp.dll" />
|
||||
<None Remove="v8_context_snapshot.bin" />
|
||||
<None Remove="CefSharp.Core.dll" />
|
||||
<None Remove="icudtl.dat" />
|
||||
<None Remove="innounp.exe" />
|
||||
<None Remove="CefSharp.Wpf.dll" />
|
||||
<None Remove="snapshot_blob.bin" />
|
||||
<None Remove="libEGL.dll" />
|
||||
<None Remove="libcef.dll" />
|
||||
<None Remove="natives_blob.bin" />
|
||||
<None Remove="CefSharp.OffScreen.dll" />
|
||||
<None Remove="devtools_resources.pak" />
|
||||
<None Remove="CefSharp.BrowserSubprocess.Core.dll" />
|
||||
<None Remove="CefSharp.BrowserSubprocess.exe" />
|
||||
<None Remove="cefsharp.7z" />
|
||||
<None Remove="cef_extensions.pak" />
|
||||
<None Remove="cef_200_percent.pak" />
|
||||
<None Remove="cef_100_percent.pak" />
|
||||
<None Remove="cef.pak" />
|
||||
<None Remove="7z.exe" />
|
||||
<None Remove="7z.dll" />
|
||||
<None Remove="swiftshader\**" />
|
||||
<None Update="public\metrics.html">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="public\files\placeholder">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<_ContentIncludedByDefault Remove="Views\MetricsDashboard.cshtml" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Compile Remove="swiftshader\**" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<EmbeddedResource Remove="swiftshader\**" />
|
||||
<None Remove="sql\wabbajack_db.sql" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Remove="swiftshader\**" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Data.SqlClient, Version=1.0.19269.1, Culture=neutral, PublicKeyToken=23ec7fc2d6eaa4a5">
|
||||
<HintPath>..\Wabbajack.MassImport\bin\Release\netcoreapp3.1\Microsoft.Data.SqlClient.dll</HintPath>
|
||||
</Reference>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@ -1,9 +0,0 @@
|
||||
{
|
||||
"Logging": {
|
||||
"LogLevel": {
|
||||
"Default": "Information",
|
||||
"Microsoft": "Warning",
|
||||
"Microsoft.Hosting.Lifetime": "Information"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,45 +0,0 @@
|
||||
{
|
||||
"AzureAd": {
|
||||
"Instance": "https://login.microsoftonline.com/",
|
||||
"Domain": "qualified.domain.name",
|
||||
"TenantId": "22222222-2222-2222-2222-222222222222",
|
||||
"ClientId": "11111111-1111-1111-11111111111111111"
|
||||
},
|
||||
"Logging": {
|
||||
"LogLevel": {
|
||||
"Default": "Information",
|
||||
"Microsoft": "Warning",
|
||||
"Microsoft.Hosting.Lifetime": "Information"
|
||||
}
|
||||
},
|
||||
"MongoDB": {
|
||||
"Host": "foo.bar.baz",
|
||||
"Database": "wabbajack",
|
||||
"Collections": {
|
||||
"NexusModInfos": "nexus_mod_infos",
|
||||
"NexusModFiles": "nexus_mod_files",
|
||||
"NexusFileInfos": "nexus_file_infos",
|
||||
"NexusUpdates": "nexus_updates",
|
||||
"ModListStatus": "mod_lists",
|
||||
"JobQueue": "job_queue",
|
||||
"DownloadStates": "download_states",
|
||||
"IndexedFiles": "indexed_files",
|
||||
"Metrics": "metrics",
|
||||
"ApiKeys": "api_keys",
|
||||
"UploadedFiles": "uploaded_files"
|
||||
}
|
||||
},
|
||||
"WabbajackSettings": {
|
||||
"DownloadDir": "c:\\tmp\\downloads",
|
||||
"ArchiveDir": "w:\\archives",
|
||||
"TempFolder": "c:\\tmp",
|
||||
"JobRunner": true,
|
||||
"JobScheduler": false,
|
||||
"RunFrontEndJobs": true,
|
||||
"RunBackEndJobs": true,
|
||||
"BunnyCDN_User": "wabbajackcdn",
|
||||
"BunnyCDN_Password": "XXXX",
|
||||
"SQLConnection": "Data Source=_,1433;Initial Catalog=wabbajack_dev;User ID=wabbajack;Password=wabbajack;MultipleActiveResultSets=true"
|
||||
},
|
||||
"AllowedHosts": "*"
|
||||
}
|
||||
@ -1 +0,0 @@
|
||||
This file exists to make sure this folder exists
|
||||
@ -1,126 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Wabbajack Metrics</title>
|
||||
<script src="//cdn.jsdelivr.net/npm/graphql.js@0.6.6/graphql.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js@2.8.0"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-colorschemes"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/underscore.js/1.9.1/underscore-min.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<h2>Finished Install Counts</h2>
|
||||
<canvas id="finished_install_count" width="800" height="600"></canvas>
|
||||
<hr/>
|
||||
|
||||
<h2>Begin Download</h2>
|
||||
<canvas id="begin_download_chart" width="800" height="600"></canvas>
|
||||
<hr/>
|
||||
|
||||
<h2>Begin Install</h2>
|
||||
<canvas id="begin_install_chart" width="800" height="600"></canvas>
|
||||
<hr/>
|
||||
|
||||
<h2>Finished Install</h2>
|
||||
<canvas id="finished_install_chart" width="800" height="600"></canvas>
|
||||
<hr/>
|
||||
|
||||
|
||||
|
||||
|
||||
<script>
|
||||
var makeChart = function(ele, group) {
|
||||
var graph = graphql("/graphql",
|
||||
{
|
||||
method: "POST",
|
||||
asJSON: true,
|
||||
headers: {
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
});
|
||||
var metrics = graph.query(`($type: MetricType) {
|
||||
dailyUniqueMetrics(metric_type: $type)
|
||||
{
|
||||
seriesName,
|
||||
labels,
|
||||
values
|
||||
}
|
||||
}`);
|
||||
|
||||
var result = metrics({type: group})
|
||||
.then(function (data) {
|
||||
var data = _.filter(data.dailyUniqueMetrics, series => _.some(series.values, v => v > 1));
|
||||
var labels = _.uniq(_.flatten(_.map(data, series => series.labels)));
|
||||
var datasets = _.map(data, series => {
|
||||
return {
|
||||
label: series.seriesName,
|
||||
fill: false,
|
||||
data: _.last(series.values, 30)
|
||||
}});
|
||||
var ctx = document.getElementById(ele).getContext('2d');
|
||||
var chart = new Chart(ctx, {
|
||||
// The type of chart we want to create
|
||||
type: 'line',
|
||||
|
||||
// The data for our dataset
|
||||
data: {
|
||||
labels: _.last(labels, 30),
|
||||
datasets: datasets},
|
||||
|
||||
// Configuration options go here
|
||||
options: {}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
var makePieChart = function(ele, group) {
|
||||
var graph = graphql("/graphql",
|
||||
{
|
||||
method: "POST",
|
||||
asJSON: true,
|
||||
headers: {
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
});
|
||||
var metrics = graph.query(`($type: MetricType) {
|
||||
dailyUniqueMetrics(metric_type: $type)
|
||||
{
|
||||
seriesName,
|
||||
labels,
|
||||
values
|
||||
}
|
||||
}`);
|
||||
|
||||
var result = metrics({type: group})
|
||||
.then(function (data) {
|
||||
var data = _.filter(data.dailyUniqueMetrics, series => _.some(series.values, v => v > 2));
|
||||
var labels = _.map(data, series => series.seriesName);
|
||||
var datasets = {data : _.map(data, series => {
|
||||
return _.reduce(series.values, (x, y) => x + y, 0)})};
|
||||
console.log(datasets);
|
||||
console.log(labels);
|
||||
var ctx = document.getElementById(ele).getContext('2d');
|
||||
var chart = new Chart(ctx, {
|
||||
// The type of chart we want to create
|
||||
type: 'pie',
|
||||
|
||||
// The data for our dataset
|
||||
data: {
|
||||
labels: labels,
|
||||
datasets: [datasets]},
|
||||
|
||||
// Configuration options go here
|
||||
options: {}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
makeChart("begin_download_chart", "BEGIN_DOWNLOAD");
|
||||
makeChart("begin_install_chart", "BEGIN_INSTALL");
|
||||
makeChart("finished_install_chart", "FINISHED_INSTALL");
|
||||
makePieChart("finished_install_count", "FINISHED_INSTALL");
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@ -32,7 +32,7 @@
|
||||
<PackageReference Include="Microsoft.Win32.Registry" Version="4.7.0" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
|
||||
<PackageReference Include="Octodiff" Version="1.2.1" />
|
||||
<PackageReference Include="ReactiveUI" Version="11.3.8" />
|
||||
<PackageReference Include="ReactiveUI" Version="11.4.1" />
|
||||
<PackageReference Include="SharpZipLib" Version="1.2.0" />
|
||||
<PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
|
||||
<PackageReference Include="System.Net.Http" Version="4.3.4" />
|
||||
|
||||
@ -64,8 +64,12 @@ namespace Wabbajack.Lib
|
||||
|
||||
public static async Task<Dictionary<RelativePath, Hash>> GetGameFiles(Game game, Version version)
|
||||
{
|
||||
// TODO: Disabled for now
|
||||
return new Dictionary<RelativePath, Hash>();
|
||||
/*
|
||||
return await GetClient()
|
||||
.GetJsonAsync<Dictionary<RelativePath, Hash>>($"{Consts.WabbajackBuildServerUri}game_files/{game}/{version}");
|
||||
*/
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -35,10 +35,10 @@
|
||||
<Version>2.1.0</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="ReactiveUI">
|
||||
<Version>11.3.8</Version>
|
||||
<Version>11.4.1</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="ReactiveUI.Fody">
|
||||
<Version>11.3.8</Version>
|
||||
<Version>11.4.1</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="SharpCompress">
|
||||
<Version>0.25.0</Version>
|
||||
|
||||
@ -1,15 +1,14 @@
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Reactive.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Common.Http;
|
||||
using Wabbajack.Common.StatusFeed;
|
||||
using Wabbajack.Lib.FileUploader;
|
||||
using Wabbajack.Server;
|
||||
using Wabbajack.Server.DataLayer;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
@ -167,32 +166,6 @@ namespace Wabbajack.BuildServer.Test
|
||||
return arr;
|
||||
}
|
||||
|
||||
protected async Task ClearJobQueue()
|
||||
{
|
||||
var sql = Fixture.GetService<SqlService>();
|
||||
while (true)
|
||||
{
|
||||
var job = await sql.GetJob();
|
||||
if (job == null) break;
|
||||
|
||||
job.Result = JobResult.Success();
|
||||
await sql.FinishJob(job);
|
||||
}
|
||||
}
|
||||
|
||||
protected async Task RunAllJobs()
|
||||
{
|
||||
var sql = Fixture.GetService<SqlService>();
|
||||
var settings = Fixture.GetService<AppSettings>();
|
||||
while (true)
|
||||
{
|
||||
var job = await sql.GetJob();
|
||||
if (job == null) break;
|
||||
|
||||
job.Result = await job.Payload.Execute(sql, settings);
|
||||
await sql.FinishJob(job);
|
||||
}
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
@ -8,11 +8,9 @@ using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
using Dapper;
|
||||
using Wabbajack.BuildServer.Controllers;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.Server.DataLayer;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace Wabbajack.BuildServer.Test
|
||||
{
|
||||
@ -29,7 +27,7 @@ namespace Wabbajack.BuildServer.Test
|
||||
{
|
||||
DBName = "test_db" + Guid.NewGuid().ToString().Replace("-", "_");
|
||||
User = Guid.NewGuid().ToString().Replace("-", "");
|
||||
APIKey = SqlService.NewAPIKey();
|
||||
//APIKey = SqlService.NewAPIKey();
|
||||
}
|
||||
|
||||
public string APIKey { get; }
|
||||
@ -50,7 +48,7 @@ namespace Wabbajack.BuildServer.Test
|
||||
await KillTestDatabases(conn);
|
||||
//await new SqlCommand($"CREATE DATABASE {DBName};", conn).ExecuteNonQueryAsync();
|
||||
|
||||
await using var schemaStream = Assembly.GetExecutingAssembly().GetManifestResourceStream("Wabbajack.BuildServer.Test.sql.wabbajack_db.sql");
|
||||
await using var schemaStream = Assembly.GetExecutingAssembly().GetManifestResourceStream("Wabbajack.Server.Test.sql.wabbajack_db.sql");
|
||||
await using var ms = new MemoryStream();
|
||||
await schemaStream.CopyToAsync(ms);
|
||||
var schemaString = Encoding.UTF8.GetString(ms.ToArray()).Replace("wabbajack_prod", $"{DBName}");
|
||||
@ -1,10 +1,8 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Dapper;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Server.DataLayer;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
@ -30,12 +28,5 @@ namespace Wabbajack.BuildServer.Test
|
||||
|
||||
Assert.Equal(subject, result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanLoadMetricsFromSQL()
|
||||
{
|
||||
var sql = Fixture.GetService<SqlService>();
|
||||
var results = await sql.MetricsReport("finish_install");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,15 +1,12 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.BuildServer.Model.Models;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
using Wabbajack.Server.DataLayer;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
using Xunit.Priority;
|
||||
|
||||
namespace Wabbajack.BuildServer.Test
|
||||
{
|
||||
@ -19,15 +16,6 @@ namespace Wabbajack.BuildServer.Test
|
||||
{
|
||||
}
|
||||
|
||||
[Fact, Priority(2)]
|
||||
public async Task CanIngestNexusCacheExports()
|
||||
{
|
||||
await @"sql\nexus_export.json".RelativeTo(AbsolutePath.EntryPoint).CopyToAsync("nexus_export.json".RelativeTo(Fixture.ServerTempFolder));
|
||||
var result = await _authedClient.GetStringAsync(MakeURL("nexus_cache/ingest"));
|
||||
|
||||
Assert.Equal("15024", result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TestCanGetModInfo()
|
||||
{
|
||||
59
Wabbajack.Server/Controllers/Heartbeat.cs
Normal file
59
Wabbajack.Server/Controllers/Heartbeat.cs
Normal file
@ -0,0 +1,59 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.Common.StatusFeed;
|
||||
using Wabbajack.Server;
|
||||
using Wabbajack.Server.DataLayer;
|
||||
using Wabbajack.Server.DTOs;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
[Route("/heartbeat")]
|
||||
public class Heartbeat : ControllerBase
|
||||
{
|
||||
static Heartbeat()
|
||||
{
|
||||
_startTime = DateTime.Now;
|
||||
|
||||
}
|
||||
private static DateTime _startTime;
|
||||
|
||||
public Heartbeat(ILogger<Heartbeat> logger, SqlService sql, GlobalInformation globalInformation)
|
||||
{
|
||||
_globalInformation = globalInformation;
|
||||
_sql = sql;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
private const int MAX_LOG_SIZE = 128;
|
||||
private static List<string> Log = new List<string>();
|
||||
private GlobalInformation _globalInformation;
|
||||
private SqlService _sql;
|
||||
private ILogger<Heartbeat> _logger;
|
||||
|
||||
public static void AddToLog(IStatusMessage msg)
|
||||
{
|
||||
lock (Log)
|
||||
{
|
||||
Log.Add(msg.ToString());
|
||||
if (Log.Count > MAX_LOG_SIZE)
|
||||
Log.RemoveAt(0);
|
||||
}
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
public async Task<IActionResult> GetHeartbeat()
|
||||
{
|
||||
return Ok(new HeartbeatResult
|
||||
{
|
||||
Uptime = DateTime.Now - _startTime,
|
||||
LastNexusUpdate = _globalInformation.TimeSinceLastNexusSync,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
14
Wabbajack.Server/DTOs/HeartbeatResult.cs
Normal file
14
Wabbajack.Server/DTOs/HeartbeatResult.cs
Normal file
@ -0,0 +1,14 @@
|
||||
using System;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
|
||||
namespace Wabbajack.Server.DTOs
|
||||
{
|
||||
[JsonName("HeartbeatResult")]
|
||||
public class HeartbeatResult
|
||||
{
|
||||
public TimeSpan Uptime { get; set; }
|
||||
public TimeSpan LastNexusUpdate { get; set; }
|
||||
|
||||
public TimeSpan LastListValidation { get; set; }
|
||||
}
|
||||
}
|
||||
@ -528,6 +528,7 @@ namespace Wabbajack.Test
|
||||
return state.ToJson().FromJsonString<T>();
|
||||
}
|
||||
|
||||
/* TODO : Disabled for now
|
||||
[Fact]
|
||||
public async Task TestUpgrading()
|
||||
{
|
||||
@ -547,7 +548,7 @@ namespace Wabbajack.Test
|
||||
Utils.Log($"Getting Hash for {(long)archive.Hash}");
|
||||
Assert.True(await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, dest));
|
||||
Assert.Equal(Hash.FromBase64("gCRVrvzDNH0="), await dest.FileHashCachedAsync());
|
||||
}
|
||||
}*/
|
||||
|
||||
class TestInstaller : AInstaller
|
||||
{
|
||||
|
||||
@ -398,6 +398,7 @@ namespace Wabbajack.Test
|
||||
|
||||
}
|
||||
|
||||
/* TODO : Disabled For Now
|
||||
[Fact]
|
||||
public async Task CanSourceFilesFromStockGameFiles()
|
||||
{
|
||||
@ -431,6 +432,7 @@ namespace Wabbajack.Test
|
||||
|
||||
Consts.TestMode = true;
|
||||
}
|
||||
*/
|
||||
|
||||
[Fact]
|
||||
public async Task NoMatchIncludeIncludesNonMatchingFiles()
|
||||
|
||||
@ -24,8 +24,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Common.CSP", "Wab
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.VirtualFileSystem", "Wabbajack.VirtualFileSystem\Wabbajack.VirtualFileSystem.csproj", "{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.BuildServer", "Wabbajack.BuildServer\Wabbajack.BuildServer.csproj", "{DE18D89E-39C5-48FD-8E42-16235E3C4593}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack", "Wabbajack\Wabbajack.csproj", "{6ED08CFB-B879-4B55-8741-663A4A3491CE}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Compression.BSA.Test", "Compression.BSA.Test\Compression.BSA.Test.csproj", "{89281BA1-67C8-48D2-9D6E-0F5CC85AD8C9}"
|
||||
@ -42,10 +40,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Common.Test", "Wa
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.App.Test", "Wabbajack.App.Test\Wabbajack.App.Test.csproj", "{44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.BuildServer.Test", "Wabbajack.BuildServer.Test\Wabbajack.BuildServer.Test.csproj", "{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Server", "Wabbajack.Server\Wabbajack.Server.csproj", "{3E11B700-8405-433D-BF47-6C356087A7C2}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Server.Test", "Wabbajack.Server.Test\Wabbajack.Server.Test.csproj", "{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
@ -89,13 +87,6 @@ Global
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x64.ActiveCfg = Release|x64
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x64.Build.0 = Release|x64
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|Any CPU.Build.0 = Debug|x64
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.Build.0 = Release|Any CPU
|
||||
{6ED08CFB-B879-4B55-8741-663A4A3491CE}.Debug|Any CPU.ActiveCfg = Debug|x64
|
||||
{6ED08CFB-B879-4B55-8741-663A4A3491CE}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{6ED08CFB-B879-4B55-8741-663A4A3491CE}.Debug|x64.Build.0 = Debug|x64
|
||||
@ -153,14 +144,6 @@ Global
|
||||
{44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}.Release|x64.Build.0 = Release|Any CPU
|
||||
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|x64.Build.0 = Release|Any CPU
|
||||
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
@ -169,6 +152,14 @@ Global
|
||||
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|x64.Build.0 = Release|Any CPU
|
||||
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|x64.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
||||
@ -57,7 +57,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="CefSharp.Wpf" Version="79.1.360" />
|
||||
<PackageReference Include="DynamicData" Version="6.14.14" />
|
||||
<PackageReference Include="DynamicData" Version="6.14.18" />
|
||||
<PackageReference Include="Extended.Wpf.Toolkit" Version="3.8.1" />
|
||||
<PackageReference Include="Fody" Version="6.1.1">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
@ -72,9 +72,9 @@
|
||||
<PackageReference Include="MahApps.Metro.IconPacks" Version="4.0.0" />
|
||||
<PackageReference Include="PInvoke.Gdi32" Version="0.6.6" />
|
||||
<PackageReference Include="PInvoke.User32" Version="0.6.6" />
|
||||
<PackageReference Include="ReactiveUI" Version="11.3.8" />
|
||||
<PackageReference Include="ReactiveUI.Fody" Version="11.3.8" />
|
||||
<PackageReference Include="ReactiveUI.WPF" Version="11.3.8" />
|
||||
<PackageReference Include="ReactiveUI" Version="11.4.1" />
|
||||
<PackageReference Include="ReactiveUI.Fody" Version="11.4.1" />
|
||||
<PackageReference Include="ReactiveUI.WPF" Version="11.4.1" />
|
||||
<PackageReference Include="SharpDX.DXGI" Version="4.2.0" />
|
||||
<PackageReference Include="WindowsAPICodePack-Shell" Version="1.1.1" />
|
||||
<PackageReference Include="WPFThemes.DarkBlend" Version="1.0.8" />
|
||||
|
||||
Loading…
Reference in New Issue
Block a user