Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement repositories

Browse Source
This commit is contained in:
Timothy Baldridge
2022-03-29 21:59:21 -06:00
parent d9c099273a
commit fd2b523493
95 changed files with 44 additions and 9714 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
Wabbajack.Lib/ModListRegistry/ModListMetadata.cs
View File

@ -58,6 +58,8 @@ namespace Wabbajack.Lib.ModListRegistry
[JsonIgnore]
public ModListSummary ValidationSummary { get; set; } = new ModListSummary();
[JsonName("repositoryName")] public string RepositoryName { get; set; } = string.Empty;
[JsonIgnore] public string NamespacedName => $"{RepositoryName}/{Links.MachineURL}";
[JsonName("Links")]
public class LinksObject
@ -72,7 +74,7 @@ namespace Wabbajack.Lib.ModListRegistry
public string Download { get; set; } = string.Empty;
[JsonProperty("machineURL")]
public string MachineURL { get; set; } = string.Empty;
internal string MachineURL { get; set; } = string.Empty;
[JsonProperty("discordURL")]
public string DiscordURL { get; set; } = string.Empty;
@ -82,18 +84,18 @@ namespace Wabbajack.Lib.ModListRegistry
{
var client = new Http.Client();
Utils.Log("Loading ModLists from GitHub");
var metadataResult = client.GetStringAsync(Consts.ModlistMetadataURL);
var utilityResult = client.GetStringAsync(Consts.UtilityModlistMetadataURL);
var summaryResult = client.GetStringAsync(Consts.ModlistSummaryURL);
var metadata = (await metadataResult).FromJsonString<List<ModlistMetadata>>();
metadata = metadata.Concat((await utilityResult).FromJsonString<List<ModlistMetadata>>()).ToList();
var metadata = await LoadModlists();
try
{
var summaries = (await summaryResult).FromJsonString<List<ModListSummary>>().ToDictionary(d => d.MachineURL);
foreach (var data in metadata)
if (summaries.TryGetValue(data.Links.MachineURL, out var summary))
if (summaries.TryGetValue(data.NamespacedName, out var summary))
data.ValidationSummary = summary;
}
catch (Exception)
@ -109,6 +111,34 @@ namespace Wabbajack.Lib.ModListRegistry
.OrderBy(m => (m.ValidationSummary?.HasFailures ?? false ? 1 : 0))
.ToList();
}
public static async Task<Dictionary<string, Uri>> LoadRepositories()
{
var client = new Http.Client();
var repositories = (await client.GetStringAsync("https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/repositories.json"))
.FromJsonString<Dictionary<string, Uri>>();
return repositories!;
}
public static async Task<ModlistMetadata[]> LoadModlists()
{
var repos = await LoadRepositories();
List<ModlistMetadata> metadatas = new();
var client = new Http.Client();
foreach (var repo in repos)
{
var newData = (await client.GetStringAsync(repo.Value))
.FromJsonString<ModlistMetadata[]>()
.Select(meta =>
{
meta.RepositoryName = repo.Key;
return meta;
});
metadatas.AddRange(newData);
}
return metadatas.ToArray();
}
public static async Task<List<ModlistMetadata>> LoadUnlistedFromGithub()
{

285
Wabbajack.Server.Test/ABuildServerSystemTest.cs
View File

@ -1,285 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO.Compression;
using System.Reactive.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.FileUploader;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class BuildServerFixture : ADBTest, IDisposable
{
private IHost _host;
private CancellationTokenSource _token;
private Task _task;
public readonly TempFolder _severTempFolder = TempFolder.Create().Result;
private bool _disposed = false;
public AbsolutePath ServerTempFolder => _severTempFolder.Dir;
public AbsolutePath ServerPublicFolder => "public".RelativeTo(AbsolutePath.EntryPoint);
public AbsolutePath ServerArchivesFolder => "archives".RelativeTo(AbsolutePath.EntryPoint);
public AbsolutePath ServerUpdatesFolder => "updates".RelativeTo(AbsolutePath.EntryPoint);
public override async Task InitializeAsync()
{
await base.InitializeAsync();
ServerArchivesFolder.DeleteDirectory().Wait();
ServerArchivesFolder.CreateDirectory();
var builder = Program.CreateHostBuilder(
new[]
{
$"WabbajackSettings:DownloadDir={"tmp".RelativeTo(AbsolutePath.EntryPoint)}",
$"WabbajackSettings:ArchiveDir={"archives".RelativeTo(AbsolutePath.EntryPoint)}",
$"WabbajackSettings:TempFolder={ServerTempFolder}",
$"WabbajackSettings:SQLConnection={PublicConnStr}",
$"WabbajackSettings:BunnyCDN_User=TEST",
$"WabbajackSettings:BunnyCDN_Password=TEST",
"WabbajackSettings:JobScheduler=false",
"WabbajackSettings:JobRunner=false",
"WabbajackSettings:RunBackEndJobs=false",
"WabbajackSettings:RunFrontEndJobs=false",
"WabbajackSettinss:DisableNexusForwarding=true"
}, true);
_host = builder.Build();
_token = new CancellationTokenSource();
_task = _host.RunAsync(_token.Token);
Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080");
Consts.TestMode = true;
await "ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllTextAsync(
"GoogleIDs:\nAllowedPrefixes:\n - http://localhost");
var sql = GetService<SqlService>();
await sql.IngestMetric(new Metric
{
Action = "start",
Subject = "tests",
Timestamp = DateTime.UtcNow,
MetricsKey = await Metrics.GetMetricsKey()
});
}
~BuildServerFixture()
{
Dispose();
}
public T GetService<T>()
{
var result = (T)_host.Services.GetService(typeof(T));
if (result == null)
throw new Exception($"Service {typeof(T)} not found in configuration");
return result;
}
public void Dispose()
{
if (_disposed) return;
_disposed = true;
if (!_token.IsCancellationRequested)
_token.Cancel();
try
{
_task.Wait();
}
catch (Exception)
{
//
}
_severTempFolder.DisposeAsync().AsTask().Wait();
}
}
/// <summary>
/// Bit of a hack to get around that we don't want the system starting and stopping our
/// HTTP server for each class its testing.
/// </summary>
/// <typeparam name="T"></typeparam>
public class SingletonAdaptor<T> where T : new()
{
private static T _singleton = default;
private static object _lock = new object();
public SingletonAdaptor()
{
}
public T Deref()
{
lock (this)
{
if (_singleton == null)
{
_singleton = new T();
if (_singleton is IAsyncLifetime d)
{
d.InitializeAsync().Wait();
}
}
return _singleton;
}
}
}
[Collection("ServerTests")]
public class ABuildServerSystemTest : XunitContextBase, IClassFixture<SingletonAdaptor<BuildServerFixture>>
{
protected readonly Wabbajack.Lib.Http.Client _client;
private readonly IDisposable _unsubMsgs;
private readonly IDisposable _unsubErr;
protected Wabbajack.Lib.Http.Client _authedClient;
protected WorkQueue _queue;
protected Random Random;
public ABuildServerSystemTest(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output)
{
Filters.Clear();
_unsubMsgs = Utils.LogMessages.OfType<IInfo>().Subscribe(onNext: msg => XunitContext.WriteLine(msg.ShortDescription));
_unsubErr = Utils.LogMessages.OfType<IUserIntervention>().Subscribe(msg =>
XunitContext.WriteLine("ERROR: User intervention required: " + msg.ShortDescription));
_client = new Wabbajack.Lib.Http.Client();
_authedClient = new Wabbajack.Lib.Http.Client();
Fixture = fixture.Deref();
var cache = Fixture.GetService<MetricsKeyCache>();
cache.AddKey(Metrics.GetMetricsKey().Result).Wait();
_authedClient.Headers.Add(("x-api-key", Fixture.APIKey));
AuthorAPI.ApiKeyOverride = Fixture.APIKey;
_queue = new WorkQueue();
Queue = new WorkQueue();
Random = new Random();
Consts.ModlistSummaryURL = MakeURL("lists/status.json");
Consts.ServerWhitelistURL = MakeURL("ServerWhitelist.yaml");
Consts.UnlistedModlistMetadataURL = MakeURL("lists/none.json");
}
public WorkQueue Queue { get; set; }
public BuildServerFixture Fixture { get; set; }
protected string MakeURL(string path)
{
return "http://localhost:8080/" + path;
}
protected byte[] RandomData(long? size = null)
{
var arr = new byte[size ?? Random.Next(1024)];
Random.NextBytes(arr);
return arr;
}
public override void Dispose()
{
Queue.Dispose();
base.Dispose();
_unsubMsgs.Dispose();
_unsubErr.Dispose();
}
protected async Task<Uri> MakeModList(string modFileName)
{
var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!");
var test_archive_path = modFileName.RelativeTo(Fixture.ServerPublicFolder);
await test_archive_path.WriteAllBytesAsync(archive_data);
ModListData = new ModList();
ModListData.Archives.Add(
new Archive(new HTTPDownloader.State(MakeURL(modFileName)))
{
Hash = await test_archive_path.FileHashAsync() ?? Hash.Empty,
Name = "test_archive",
Size = test_archive_path.Size,
});
var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder);
await using (var fs = await modListPath.Create())
{
using var za = new ZipArchive(fs, ZipArchiveMode.Create);
var entry = za.CreateEntry("modlist");
await using var es = entry.Open();
ModListData.ToJson(es);
}
ModListMetaData = new List<ModlistMetadata>
{
new ModlistMetadata
{
Official = false,
Author = "Test Suite",
Description = "A test",
DownloadMetadata = new DownloadMetadata
{
Hash = await modListPath.FileHashAsync() ?? Hash.Empty,
Size = modListPath.Size
},
Links = new ModlistMetadata.LinksObject
{
MachineURL = "test_list",
Download = MakeURL("test_modlist.wabbajack")
}
},
new ModlistMetadata
{
Official = true,
Author = "Test Suite",
Description = "A list with a broken hash",
DownloadMetadata = new DownloadMetadata()
{
Hash = Hash.FromLong(42),
Size = 42
},
Links = new ModlistMetadata.LinksObject
{
MachineURL = "broken_list",
Download = MakeURL("test_modlist.wabbajack")
}
}
};
var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder);
await ModListMetaData.ToJsonAsync(metadataPath);
return new Uri(MakeURL("test_mod_list_metadata.json"));
}
public ModList ModListData { get; set; }
public List<ModlistMetadata> ModListMetaData { get; set; }
}
}

145
Wabbajack.Server.Test/ADBTest.cs
View File

@ -1,145 +0,0 @@
using System;
using System.Collections.Generic;
using System.Data.SqlClient;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
using Xunit;
namespace Wabbajack.BuildServer.Test
{
public class ADBTest : IAsyncLifetime
{
private static string CONN_STR = @"Data Source=.\SQLEXPRESS;Integrated Security=True;";
public string PublicConnStr => CONN_STR + $";Initial Catalog={DBName}";
protected SqlService _sqlService;
private bool _finishedSchema;
private string DBName { get; }
public ADBTest()
{
DBName = "test_db" + Guid.NewGuid().ToString().Replace("-", "_");
User = Guid.NewGuid().ToString().Replace("-", "");
APIKey = SqlService.NewAPIKey();
}
public string APIKey { get; }
public string User { get; }
public virtual async Task InitializeAsync()
{
await CreateSchema();
}
private async Task CreateSchema()
{
Utils.Log($"Creating Database {DBName}");
await using var conn = new SqlConnection(CONN_STR);
await conn.OpenAsync();
await KillTestDatabases(conn);
//await new SqlCommand($"CREATE DATABASE {DBName};", conn).ExecuteNonQueryAsync();
await using var schemaStream = Assembly.GetExecutingAssembly().GetManifestResourceStream("Wabbajack.Server.Test.sql.wabbajack_db.sql");
await using var ms = new MemoryStream();
await schemaStream.CopyToAsync(ms);
var schemaString = Encoding.UTF8.GetString(ms.ToArray()).Replace("wabbajack_prod", $"{DBName}");
foreach (var statement in SplitSqlStatements(schemaString))
{
await new SqlCommand(statement, conn).ExecuteNonQueryAsync();
}
await new SqlCommand($"USE {DBName}", conn).ExecuteNonQueryAsync();
await new SqlCommand($"INSERT INTO dbo.ApiKeys (APIKey, Owner) VALUES ('{APIKey}', '{User}');", conn).ExecuteNonQueryAsync();
_finishedSchema = true;
Utils.Log($"Finished creating database {DBName}");
}
private static IEnumerable<string> SplitSqlStatements(string sqlScript)
{
// Split by "GO" statements
var statements = Regex.Split(
sqlScript,
@"^[\t \r\n]*GO[\t \r\n]*\d*[\t ]*(?:--.*)?$",
RegexOptions.Multiline |
RegexOptions.IgnorePatternWhitespace |
RegexOptions.IgnoreCase);
// Remove empties, trim, and return
return statements
.Where(x => !string.IsNullOrWhiteSpace(x))
.Select(x => x.Trim(' ', '\r', '\n'));
}
async Task IAsyncLifetime.DisposeAsync()
{
// Don't delete it if the setup failed, so we can debug the issue
if (!_finishedSchema) return;
Utils.Log("Deleting Database");
await using var conn = new SqlConnection(CONN_STR);
await conn.OpenAsync();
await KillTestDatabases(conn);
}
private async Task KillTestDatabases(SqlConnection conn)
{
await KillAll(conn);
var dbs = await conn.QueryAsync<string>("SELECT name from [master].[sys].[databases]");
foreach (var db in dbs.Where(name => name.StartsWith("test_")))
{
await new SqlCommand(
$"DROP DATABASE {db};",
conn)
.ExecuteNonQueryAsync();
}
}
private async Task KillAll(SqlConnection conn)
{
await new SqlCommand($@"
DECLARE @Spid INT
DECLARE @ExecSQL VARCHAR(255)
DECLARE KillCursor CURSOR LOCAL STATIC READ_ONLY FORWARD_ONLY
FOR
SELECT DISTINCT SPID
FROM MASTER..SysProcesses
WHERE DBID = DB_ID('{DBName}')
OPEN KillCursor
-- Grab the first SPID
FETCH NEXT
FROM KillCursor
INTO @Spid
WHILE @@FETCH_STATUS = 0
BEGIN
SET @ExecSQL = 'KILL ' + CAST(@Spid AS VARCHAR(50))
EXEC (@ExecSQL)
-- Pull the next SPID
FETCH NEXT
FROM KillCursor
INTO @Spid
END
CLOSE KillCursor
DEALLOCATE KillCursor", conn).ExecuteNonQueryAsync();
}
}
}

78
Wabbajack.Server.Test/ArchiveDownloadsTests.cs
View File

@ -1,78 +0,0 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DataLayer;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class ArchiveDownloadsTests : ABuildServerSystemTest
{
public ArchiveDownloadsTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanEnqueueDequeueAndUpdateDownloads()
{
await ClearDownloaderQueue();
var state = new HTTPDownloader.State("http://www.google.com");
var archive = new Archive(state);
var service = Fixture.GetService<SqlService>();
var id = await service.EnqueueDownload(archive);
var toRun = await service.GetNextPendingDownload();
Assert.Equal(id, toRun.Id);
await toRun.Finish(service);
await service.UpdatePendingDownload(toRun);
toRun = await service.GetNextPendingDownload();
Assert.Null(toRun);
var allStates = await service.GetAllArchiveDownloads();
Assert.Contains(state.PrimaryKeyString, allStates.Select(s => s.PrimaryKeyString));
}
[Fact]
public async Task DontReenqueueDownloadedfiles()
{
var hash = Hash.FromLong(Random.Next(int.MinValue, int.MaxValue));
await ClearDownloaderQueue();
var _sql = Fixture.GetService<SqlService>();
var archive = new Archive(new HTTPDownloader.State("http://www.google.com")) {Size = 42, Hash = hash,};
await _sql.EnqueueDownload(archive);
var download = await _sql.GetNextPendingDownload();
await download.Finish(_sql);
Assert.Null(await _sql.GetNextPendingDownload());
var found = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size);
Assert.NotNull(found);
var next = await _sql.GetOrEnqueueArchive(archive);
Assert.Null(await _sql.GetNextPendingDownload());
await ClearDownloaderQueue();
}
private async Task ClearDownloaderQueue()
{
var service = Fixture.GetService<SqlService>();
while (true)
{
var job = await service.GetNextPendingDownload();
if (job == null) break;
await job.Fail(service, "Canceled");
}
}
}
}

53
Wabbajack.Server.Test/ArchiveMaintainerTests.cs
View File

@ -1,53 +0,0 @@
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class ArchiveMaintainerTests : ABuildServerSystemTest
{
public ArchiveMaintainerTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanIngestFiles()
{
var maintainer = Fixture.GetService<ArchiveMaintainer>();
await using var tf = new TempFile();
await using var tf2 = new TempFile();
await tf.Path.WriteAllBytesAsync(RandomData(1024));
await tf.Path.CopyToAsync(tf2.Path);
var hash = await tf.Path.FileHashAsync();
await maintainer.Ingest(tf.Path);
Assert.NotNull(hash);
Assert.True(maintainer.TryGetPath(hash!.Value, out var found));
Assert.Equal(await tf2.Path.ReadAllBytesAsync(), await found.ReadAllBytesAsync());
}
[Fact]
public async Task IngestsExistingFiles()
{
var maintainer = Fixture.GetService<ArchiveMaintainer>();
await using var tf = new TempFile();
await tf.Path.WriteAllBytesAsync(RandomData(1024));
var hash = await tf.Path.FileHashAsync();
Assert.NotNull(hash);
await tf.Path.CopyToAsync(Fixture.ServerArchivesFolder.Combine(hash!.Value.ToHex()));
maintainer.Start();
Assert.True(maintainer.TryGetPath(hash!.Value, out var found));
}
}
}

27
Wabbajack.Server.Test/AuthorControlTests.cs
View File

@ -1,27 +0,0 @@
using System.Net.Http;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Server.Test
{
public class AuthorControlTests : ABuildServerSystemTest
{
public AuthorControlTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task LoginRedirects()
{
var client = new HttpClient();
var result =
await client.GetStringAsync($"{Consts.WabbajackBuildServerUri}author_controls/login/{Fixture.APIKey}");
Assert.Contains("Wabbajack Files", result);
}
}
}

87
Wabbajack.Server.Test/AuthoredFilesTests.cs
View File

@ -1,87 +0,0 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Exceptions;
using Wabbajack.Lib;
using Wabbajack.Lib.AuthorApi;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class AuthoredFilesTests : ABuildServerSystemTest
{
public AuthoredFilesTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanUploadDownloadAndDeleteAuthoredFiles()
{
var cleanup = Fixture.GetService<AuthoredFilesCleanup>();
var sql = Fixture.GetService<SqlService>();
var toDelete = await cleanup.FindFilesToDelete();
await using var file = new TempFile();
await file.Path.WriteAllBytesAsync(RandomData(Consts.UploadedFileBlockSize * 4 + Consts.UploadedFileBlockSize / 3));
var originalHash = await file.Path.FileHashAsync();
var client = await Client.Create(Fixture.APIKey);
using var queue = new WorkQueue(2);
var uri = await client.UploadFile(queue, file.Path, (s, percent) => Utils.Log($"({percent}) {s}"));
var data = (await Fixture.GetService<SqlService>().AllAuthoredFiles()).ToArray();
Assert.Contains((string)file.Path.FileName, data.Select(f => f.OriginalFileName));
var listing = await cleanup.GetCDNMungedNames();
foreach (var d in data)
{
Assert.Contains(d.MungedName, listing);
}
// Just uploaded it, so it shouldn't be marked for deletion
toDelete = await cleanup.FindFilesToDelete();
foreach (var d in data)
{
Assert.DoesNotContain(d.MungedName, toDelete.CDNDelete);
Assert.DoesNotContain(d.ServerAssignedUniqueId, toDelete.SQLDelete);
}
var result = await _client.GetStringAsync(MakeURL("authored_files"));
Assert.Contains((string)file.Path.FileName, result);
var state = await DownloadDispatcher.Infer(uri);
Assert.IsType<WabbajackCDNDownloader.State>(state);
await state.Download(new Archive(state) {Name = (string)file.Path.FileName}, file.Path);
Assert.Equal(originalHash, await file.Path.FileHashAsync());
// Mark it as old
foreach (var d in data)
{
await sql.TouchAuthoredFile(await sql.GetCDNFileDefinition(d.ServerAssignedUniqueId), DateTime.Now - TimeSpan.FromDays(8));
}
// Now it should be marked for deletion
toDelete = await cleanup.FindFilesToDelete();
foreach (var d in data)
{
Assert.Contains(d.MungedName, toDelete.CDNDelete);
Assert.Contains(d.ServerAssignedUniqueId, toDelete.SQLDelete);
}
await cleanup.Execute();
toDelete = await cleanup.FindFilesToDelete();
Assert.Empty(toDelete.CDNDelete);
Assert.Empty(toDelete.SQLDelete);
}
}
}

25
Wabbajack.Server.Test/DiscordFrontentTests.cs
View File

@ -1,25 +0,0 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Test;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Server.Test
{
public class DiscordFrontentTests: ABuildServerSystemTest
{
public DiscordFrontentTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanLogIn()
{
var frontend = Fixture.GetService<DiscordFrontend>();
frontend.Start();
}
}
}

34
Wabbajack.Server.Test/ListIngestionTests.cs
View File

@ -1,34 +0,0 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Wabbajack.Lib;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Server.Test
{
public class ListIngestionTests : ABuildServerSystemTest
{
public ListIngestionTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanIngestModLists()
{
await ClientAPI.SendModListDefinition(new ModList {Name = "sup"});
await Task.Delay(500);
Assert.Contains(AbsolutePath.EntryPoint.Combine("mod_list_definitions")
.EnumerateFiles(false),
f => DateTime.UtcNow - f.LastModifiedUtc < TimeSpan.FromSeconds(15));
var data = AbsolutePath.EntryPoint.Combine("mod_list_definitions").EnumerateFiles(false)
.OrderByDescending(f => f.LastModifiedUtc).First().FromJson<ModList>();
Assert.Equal("sup", data.Name);
}
}
}

39
Wabbajack.Server.Test/LoginTests.cs
View File

@ -1,39 +0,0 @@
using System;
using System.Threading.Tasks;
using Wabbajack.Common;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class LoginTests : ABuildServerSystemTest
{
public LoginTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanCreateLogins()
{
var newUserName = Guid.NewGuid().ToString();
var newKey = await _authedClient.GetStringAsync(MakeURL($"users/add/{newUserName}"));
Assert.NotEmpty(newKey);
Assert.NotNull(newKey);
Assert.NotEqual(newKey, Fixture.APIKey);
var done = await _authedClient.GetStringAsync(MakeURL("users/export"));
Assert.Equal("done", done);
foreach (var (userName, apiKey) in new[] {(newUserName, newKey), (Fixture.User, Fixture.APIKey)})
{
var exported = await Fixture.ServerTempFolder.Combine("exported_users", userName, Consts.AuthorAPIKeyFile)
.ReadAllTextAsync();
Assert.Equal(exported, apiKey);
}
}
}
}

54
Wabbajack.Server.Test/MetricsTests.cs
View File

@ -1,54 +0,0 @@
using System;
using System.Net.Http.Headers;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Lib;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class MetricsTests : ABuildServerSystemTest
{
public MetricsTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanSendAndGetMetrics()
{
var action = "action_" + Guid.NewGuid().ToString();
var subject = "subject_" + Guid.NewGuid().ToString();
await Metrics.Send(action, subject);
var sql = Fixture.GetService<SqlService>();
var conn = await sql.Open();
var result = await conn.QueryFirstOrDefaultAsync<string>("SELECT Subject FROM dbo.Metrics WHERE Action = @Action",
new {Action = action});
Assert.Equal(subject, result);
using var response = await _client.GetAsync(MakeURL($"metrics/report/{action}"));
Assert.Equal(TimeSpan.FromHours(1), response.Headers.CacheControl.MaxAge);
// we'll just make sure this doesn't error, with limited data that's about all we can do atm
using var totalInstalls = await _client.GetAsync(MakeURL($"metrics/total_installs.html"));
Assert.True(totalInstalls.IsSuccessStatusCode);
using var totalUniqueInstalls = await _client.GetAsync(MakeURL($"metrics/total_unique_installs.html"));
Assert.True(totalUniqueInstalls.IsSuccessStatusCode);
using var dumpResponse = await _client.GetAsync(MakeURL("metrics/dump.json"));
Assert.True(dumpResponse.IsSuccessStatusCode);
var data = await dumpResponse.Content.ReadAsStringAsync();
Assert.NotEmpty(data);
var cache = Fixture.GetService<MetricsKeyCache>();
Assert.True(await cache.KeyCount() > 0);
}
}
}

82
Wabbajack.Server.Test/MirroredFilesTests.cs
View File

@ -1,82 +0,0 @@
using System;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Server.Test
{
public class MirroredFilesTests : ABuildServerSystemTest
{
public MirroredFilesTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanUploadAndDownloadMirroredFiles()
{
var file = new TempFile();
await file.Path.WriteAllBytesAsync(RandomData(1024 * 1024 * 6));
var dataHash = await file.Path.FileHashAsync();
Assert.NotNull(dataHash);
await Fixture.GetService<ArchiveMaintainer>().Ingest(file.Path);
Assert.True(Fixture.GetService<ArchiveMaintainer>().HaveArchive(dataHash!.Value));
var sql = Fixture.GetService<SqlService>();
await sql.UpsertMirroredFile(new MirroredFile
{
Created = DateTime.UtcNow,
Rationale = "Test File",
Hash = dataHash!.Value
});
var uploader = Fixture.GetService<MirrorUploader>();
uploader.ActiveFileSyncEnabled = false;
Assert.Equal(1, await uploader.Execute());
var archive = new Archive(new HTTPDownloader.State(MakeURL(dataHash.ToString())))
{
Hash = dataHash!.Value,
Size = file.Path.Size
};
await using var file2 = new TempFile();
await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file2.Path);
Assert.Equal(dataHash!.Value, await file2.Path.FileHashAsync());
var onServer = await uploader.GetHashesOnCDN();
Assert.Contains(dataHash.Value, onServer);
await uploader.DeleteOldMirrorFiles();
// Still in SQL so it will still exist
await using var file3 = new TempFile();
await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file3.Path);
Assert.Equal(dataHash!.Value, await file3.Path.FileHashAsync());
// Enabling the sync should kill off the unattached file
uploader.ActiveFileSyncEnabled = true;
Assert.Equal(0, await uploader.Execute());
var onServer2 = await uploader.GetHashesOnCDN();
Assert.DoesNotContain(dataHash.Value, onServer2);
}
[Fact]
public async Task CanQueueFiles()
{
var service = Fixture.GetService<MirrorQueueService>();
Assert.Equal(1, await service.Execute());
}
}
}

62
Wabbajack.Server.Test/ModFileTests.cs
View File

@ -1,62 +0,0 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Server.Test
{
public class ModFileTests : ABuildServerSystemTest
{
public ModFileTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanGetDownloadStates()
{
var sql = Fixture.GetService<SqlService>();
var hash = Hash.FromBase64("eSIyd+KOG3s=");
var archive =
new Archive(new HTTPDownloader.State(
"https://build.wabbajack.org/WABBAJACK_TEST_FILE.txt"))
{
Size = 20, Hash = hash
};
await sql.EnqueueDownload(archive);
await sql.UpsertMirroredFile(new MirroredFile()
{
Created = DateTime.UtcNow,
Uploaded = DateTime.UtcNow,
Hash = hash,
Rationale = "Test File"
});
var dld = await sql.GetNextPendingDownload();
await dld.Finish(sql);
var state = await ClientAPI.InferDownloadState(archive.Hash);
Assert.NotNull(state);
Assert.Equal(archive.State.GetMetaIniString(), state!.GetMetaIniString());
var archives = await (await ClientAPI.GetClient()).GetJsonAsync<Archive[]>(
$"{Consts.WabbajackBuildServerUri}mod_files/by_hash/{hash.ToHex()}");
Assert.True(archives.Length >= 2);
Assert.NotNull(archives.FirstOrDefault(a => a.State is WabbajackCDNDownloader.State));
await sql.DeleteMirroredFile(hash);
}
}
}

202
Wabbajack.Server.Test/ModListValidationTests.cs
View File

@ -1,202 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class ModListValidationTests : ABuildServerSystemTest
{
public ModListValidationTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanLoadMetadataFromTestServer()
{
var modlist = await MakeModList("CanLoadMetadataFromTestServer.txt");
Consts.ModlistMetadataURL = modlist.ToString();
var data = await ModlistMetadata.LoadFromGithub();
Assert.Equal(3, data.Count);
Assert.Equal("test_list", data.OrderByDescending(x => x.Links.MachineURL).First().Links.MachineURL);
}
[Fact]
public async Task CanIngestModLists()
{
var modlist = await MakeModList("CanIngestModLists.txt");
Consts.ModlistMetadataURL = modlist.ToString();
var sql = Fixture.GetService<SqlService>();
var downloader = Fixture.GetService<ModListDownloader>();
await downloader.Execute();
foreach (var list in ModListMetaData)
{
Assert.True(await sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash));
}
// Nothing has changed so we shouldn't be downloading anything this time
// Test doesn't matter for now
//Assert.Equal(0, await downloader.Execute());
}
[Fact]
public async Task CanValidateModLists()
{
var sql = Fixture.GetService<SqlService>();
await using var conn = await sql.Open();
await conn.ExecuteAsync("DELETE from Patches");
var modlists = await MakeModList("CanValidateModlistsFile.txt");
Consts.ModlistMetadataURL = modlists.ToString();
Utils.Log("Updating modlists");
await RevalidateLists(true);
Utils.Log("Checking validated results");
var data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
await CheckListFeeds(0, 1);
Utils.Log("Break List");
var archive = "CanValidateModlistsFile.txt".RelativeTo(Fixture.ServerPublicFolder);
await archive.MoveToAsync(archive.WithExtension(new Extension(".moved")), true);
// We can revalidate but the non-nexus archives won't be checked yet since the list didn't change
await RevalidateLists(false);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
// Run the non-nexus validator
await RevalidateLists(true);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(1, data.ValidationSummary.Failed);
Assert.Equal(0, data.ValidationSummary.Passed);
await CheckListFeeds(1, 0);
Utils.Log("Fix List");
await archive.WithExtension(new Extension(".moved")).MoveToAsync(archive, false);
await RevalidateLists(true);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
await CheckListFeeds(0, 1);
}
[Fact]
public async Task CanHealLists()
{
var modlists = await MakeModList("CanHealLists.txt");
Consts.ModlistMetadataURL = modlists.ToString();
Utils.Log("Updating modlists");
await RevalidateLists(true);
Utils.Log("Checking validated results");
var data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
await CheckListFeeds(0, 1);
Utils.Log("Break List by changing the file");
var archive = "CanHealLists.txt".RelativeTo(Fixture.ServerPublicFolder);
await archive.WriteAllTextAsync("broken");
// We can revalidate but the non-nexus archives won't be checked yet since the list didn't change
await RevalidateLists(false);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
// Run the non-nexus validator
await RevalidateLists(true);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(1, data.ValidationSummary.Failed);
Assert.Equal(0, data.ValidationSummary.Passed);
Assert.Equal(1, data.ValidationSummary.Updating);
var patcher = Fixture.GetService<PatchBuilder>();
Assert.True(await patcher.Execute() > 1);
await RevalidateLists(false);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
Assert.Equal(0, data.ValidationSummary.Updating);
}
private async Task RevalidateLists(bool runNonNexus)
{
var downloader = Fixture.GetService<ModListDownloader>();
await downloader.Execute();
if (runNonNexus)
{
var nonNexus = Fixture.GetService<NonNexusDownloadValidator>();
await nonNexus.Execute();
}
var validator = Fixture.GetService<ListValidator>();
await validator.Execute();
var archiver = Fixture.GetService<ArchiveDownloader>();
await archiver.Execute();
}
private async Task CheckListFeeds(int failed, int passed)
{
var statusJson = await _client.GetJsonAsync<DetailedStatus>(MakeURL("lists/status/test_list.json"));
Assert.Equal(failed, statusJson.Archives.Count(a => a.IsFailing));
Assert.Equal(passed, statusJson.Archives.Count(a => !a.IsFailing));
var statusHtml = await _client.GetHtmlAsync(MakeURL("lists/status/test_list.html"));
Assert.NotEmpty(statusHtml.DocumentNode.Descendants().Where(n => n.InnerHtml == $"Failed ({failed}):"));
Assert.NotEmpty(statusHtml.DocumentNode.Descendants().Where(n => n.InnerHtml == $"Passed ({passed}):"));
var statusRss = await _client.GetHtmlAsync(MakeURL("lists/status/test_list/broken.rss"));
Assert.Equal(failed, statusRss.DocumentNode.SelectNodes("//item")?.Count ?? 0);
var heartBeat = await _client.GetHtmlAsync(MakeURL("heartbeat/report"));
Assert.Contains(heartBeat.DocumentNode.Descendants(), c => c.InnerText.StartsWith("test_list"));
}
}
}

156
Wabbajack.Server.Test/ModlistUpdater.cs
View File

@ -1,156 +0,0 @@
using System;
using System.Text;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.BuildServer;
using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Wabbajack.Common.Exceptions;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.FileUploader;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Server.Test
{
public class ModlistUpdater : ABuildServerSystemTest
{
public ModlistUpdater(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output,
fixture)
{
}
[Fact]
public async Task CanIndexAndUpdateFiles()
{
var _sql = Fixture.GetService<SqlService>();
await using var conn = await _sql.Open();
await conn.ExecuteAsync("DELETE FROM dbo.NoPatch");
var settings = Fixture.GetService<AppSettings>();
settings.ValidateModUpgrades = false;
var validator = Fixture.GetService<ListValidator>();
var nonNexus = Fixture.GetService<NonNexusDownloadValidator>();
var modLists = await MakeModList("CanIndexAndUpdateFiles.txt");
Consts.ModlistMetadataURL = modLists.ToString();
var listDownloader = Fixture.GetService<ModListDownloader>();
var downloader = Fixture.GetService<ArchiveDownloader>();
var archiver = Fixture.GetService<ArchiveMaintainer>();
var patcher = Fixture.GetService<PatchBuilder>();
patcher.NoCleaning = true;
var sql = Fixture.GetService<SqlService>();
var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!");
var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!");
var oldDataHash = oldFileData.xxHash();
var newDataHash = newFileData.xxHash();
var oldArchive = new Archive(new NexusDownloader.State {Game = Game.Enderal, ModID = 42, FileID = 10})
{
Size = oldFileData.Length,
Hash = oldDataHash
};
var newArchive = new Archive(new NexusDownloader.State {Game = Game.Enderal, ModID = 42, FileID = 11})
{
Size = newFileData.Length,
Hash = newDataHash
};
await IngestData(archiver, oldFileData);
await IngestData(archiver, newFileData);
await sql.EnqueueDownload(oldArchive);
var oldDownload = await sql.GetNextPendingDownload();
await oldDownload.Finish(sql);
await sql.EnqueueDownload(newArchive);
var newDownload = await sql.GetNextPendingDownload();
await newDownload.Finish(sql);
await Assert.ThrowsAsync<HttpException>(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
Assert.True(await patcher.Execute() > 1);
Assert.Equal(new Uri("https://test-files.wabbajack.org/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
Assert.Equal("Purged", await AuthorAPI.NoPatch(oldArchive.Hash, "Testing NoPatch"));
await Assert.ThrowsAsync<HttpException>(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
Assert.True(await sql.IsNoPatch(oldArchive.Hash));
}
[Fact]
public async Task TestEndToEndArchiveUpdating()
{
var _sql = Fixture.GetService<SqlService>();
await using var conn = await _sql.Open();
await conn.ExecuteAsync("DELETE FROM dbo.NoPatch");
var settings = Fixture.GetService<AppSettings>();
settings.ValidateModUpgrades = false;
var modLists = await MakeModList("TestEndToEndArchiveUpdating.txt");
Consts.ModlistMetadataURL = modLists.ToString();
var downloader = Fixture.GetService<ArchiveDownloader>();
var archiver = Fixture.GetService<ArchiveMaintainer>();
var patcher = Fixture.GetService<PatchBuilder>();
patcher.NoCleaning = true;
var sql = Fixture.GetService<SqlService>();
var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!" + Guid.NewGuid());
var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!");
var oldDataHash = oldFileData.xxHash();
var newDataHash = newFileData.xxHash();
await "TestEndToEndArchiveUpdating.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(oldFileData);
var oldArchive = new Archive(new HTTPDownloader.State(MakeURL("TestEndToEndArchiveUpdating.txt")))
{
Size = oldFileData.Length,
Hash = oldDataHash
};
await IngestData(archiver, oldFileData);
await sql.EnqueueDownload(oldArchive);
var oldDownload = await sql.GetNextPendingDownload();
await oldDownload.Finish(sql);
// Now update the file
await"TestEndToEndArchiveUpdating.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(newFileData);
await using var tempFile = new TempFile();
var pendingRequest = DownloadDispatcher.DownloadWithPossibleUpgrade(oldArchive, tempFile.Path);
for (var times = 0; await downloader.Execute() == 0 && times < 40; times ++)
{
await Task.Delay(TimeSpan.FromMilliseconds(200));
}
for (var times = 0; await patcher.Execute() == 0 && times < 40; times ++)
{
await Task.Delay(TimeSpan.FromMilliseconds(200));
}
Assert.Equal(DownloadDispatcher.DownloadResult.Update, await pendingRequest);
Assert.Equal(oldDataHash, await tempFile.Path.FileHashAsync());
}
private async Task IngestData(ArchiveMaintainer am, byte[] data)
{
await using var f = new TempFile();
await f.Path.WriteAllBytesAsync(data);
await am.Ingest(f.Path);
}
}
}

154
Wabbajack.Server.Test/NexusCacheTests.cs
View File

@ -1,154 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.FileUploader;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class NexusCacheTests : ABuildServerSystemTest
{
public NexusCacheTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task TestCanGetModInfo()
{
var sqlService = Fixture.GetService<SqlService>();
var modId = long.MaxValue >> 1;
await sqlService.AddNexusModInfo(Game.SkyrimSpecialEdition, modId, DateTime.Now,
new ModInfo {author = "Buzz", uploaded_by = "bille"});
var api = await NexusApiClient.Get();
var modInfoResponse = await api.GetModInfo(Game.SkyrimSpecialEdition, modId);
Assert.Equal("Buzz", modInfoResponse.author);
Assert.Equal("bille", modInfoResponse.uploaded_by);
}
[Fact]
public async Task TestCanGetModFiles()
{
var sqlService = Fixture.GetService<SqlService>();
var modId = long.MaxValue >> 1;
await sqlService.AddNexusModFiles(Game.SkyrimSpecialEdition, modId, DateTime.Now,
new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>
{
new NexusFileInfo
{
file_name = "blerg"
}
}});
var api = await NexusApiClient.Get();
var modInfoResponse = await api.GetModFiles(Game.SkyrimSpecialEdition, modId);
Assert.Single(modInfoResponse.files);
Assert.Equal("blerg", modInfoResponse.files.First().file_name);
}
[Fact]
public async Task TestCanPurgeModInfo()
{
var sqlService = Fixture.GetService<SqlService>();
var modId = long.MaxValue >> 3;
await sqlService.AddNexusModFiles(Game.SkyrimSpecialEdition, modId, DateTime.Now,
new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>
{
new NexusFileInfo
{
file_name = "blerg"
}
}});
var api = await NexusApiClient.Get();
var modInfoResponse = await api.GetModFiles(Game.SkyrimSpecialEdition, modId);
Assert.Single(modInfoResponse.files);
Assert.Equal("blerg", modInfoResponse.files.First().file_name);
await AuthorAPI.PurgeNexusModInfo(modId);
}
[Fact]
public async Task CanQueryAndFindNexusModfilesFast()
{
var startTime = DateTime.UtcNow;
var sql = Fixture.GetService<SqlService>();
var validator = Fixture.GetService<ListValidator>();
await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow);
await sql.DeleteNexusModInfosUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow);
var result = await validator.FastNexusModStats(new NexusDownloader.State {Game = Game.SkyrimSpecialEdition, ModID = 1137, FileID = 121449});
Assert.Equal(ArchiveStatus.Valid, result);
var gameId = Game.SkyrimSpecialEdition.MetaData().NexusGameId;
var hs = await sql.AllNexusFiles();
var found = hs.FirstOrDefault(h =>
h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449);
Assert.True(found != default);
}
[JsonName("DateBox")]
class Box
{
public DateTime Value
{
get;
set;
}
}
[Fact]
public async Task DatesConvertProperly()
{
var a = DateTime.Now;
var b = DateTime.UtcNow;
Assert.NotEqual(a, new Box{Value = a}.ToJson().FromJsonString<Box>().Value);
Assert.Equal(b, new Box{Value = b}.ToJson().FromJsonString<Box>().Value);
Assert.NotEqual(a.Hour, b.Hour);
Assert.Equal(b.Hour, new Box{Value = a}.ToJson().FromJsonString<Box>().Value.Hour);
var ts = (long)1589528640;
var ds = DateTime.Parse("2020-05-15 07:44:00.000");
Assert.Equal(ds, ts.AsUnixTime());
Assert.Equal(ts, (long)ds.AsUnixTime());
Assert.Equal(ts, (long)ts.AsUnixTime().AsUnixTime());
}
[Fact]
public async Task CanGetAndSetPermissions()
{
var game = Game.Oblivion;
var modId = 4424;
var sql = Fixture.GetService<SqlService>();
foreach (HTMLInterface.PermissionValue result in Enum.GetValues(typeof(HTMLInterface.PermissionValue)))
{
await sql.SetNexusPermission(game, modId, result);
Assert.Equal(result, (await sql.GetNexusPermissions())[(game, modId)]);
}
}
}
}

37
Wabbajack.Server.Test/VirusCheckTests.cs
View File

@ -1,37 +0,0 @@
using System.Threading.Tasks;
using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Wabbajack.Lib;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Server.Test
{
public class VirusCheckTests : ABuildServerSystemTest
{
public VirusCheckTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CheckVirus()
{
var tmpFile = new TempFile();
var meta = Game.SkyrimSpecialEdition.MetaData();
var srcFile = meta.GameLocation().Combine(meta.MainExecutable!);
await srcFile.CopyToAsync(tmpFile.Path);
using (var s = await tmpFile.Path.OpenWrite())
{
s.Position = 1000;
s.WriteByte(42);
}
Assert.True(await VirusScanner.ShouldScan(tmpFile.Path));
Assert.Equal(VirusScanner.Result.NotMalware, await ClientAPI.GetVirusScanResult(tmpFile.Path));
}
}
}

34
Wabbajack.Server.Test/Wabbajack.Server.Test.csproj
View File

@ -1,34 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net5.0-windows</TargetFramework>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
<IsPackable>false</IsPackable>
<!-- temp fix for NETSDK1151 https://github.com/dotnet/sdk/issues/17579-->
<ValidateExecutableReferencesMatchSelfContained>false</ValidateExecutableReferencesMatchSelfContained>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.10.0" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
<PackageReference Include="XunitContext" Version="3.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
<ProjectReference Include="..\Wabbajack.Server\Wabbajack.Server.csproj" />
</ItemGroup>
<ItemGroup>
<None Update="sql\NotifyStates\00e8bbbf591f61a3_6a5eb07c4b3c03fde38c9223a94a38c9076ef8fc8167f77c875c58db8f2aefd2.ini">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="sql\wabbajack_db.sql" />
</ItemGroup>
</Project>

1081
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

File diff suppressed because it is too large Load Diff

166
Wabbajack.Server/ApiKeyAuthorizationHandler.cs
View File

@ -1,166 +0,0 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Security.Claims;
using System.Text.Encodings.Web;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer
{
public class ApiKeyAuthenticationOptions : AuthenticationSchemeOptions
{
public const string DefaultScheme = "API Key";
public string Scheme => DefaultScheme;
public string AuthenticationType = DefaultScheme;
}
public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthenticationOptions>
{
private const string ProblemDetailsContentType = "application/problem+json";
private readonly SqlService _sql;
public const string ApiKeyHeaderName = "X-Api-Key";
private MetricsKeyCache _keyCache;
public ApiKeyAuthenticationHandler(
IOptionsMonitor<ApiKeyAuthenticationOptions> options,
ILoggerFactory logger,
UrlEncoder encoder,
ISystemClock clock,
MetricsKeyCache keyCache,
SqlService db) : base(options, logger, encoder, clock)
{
_sql = db;
_keyCache = keyCache;
}
protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
{
var metricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault();
// Never needed this, disabled for now
//await LogRequest(metricsKey);
if (metricsKey != default)
{
await _keyCache.AddKey(metricsKey);
if (await _sql.IsTarKey(metricsKey))
{
await _sql.IngestMetric(new Metric
{
Action = "TarKey",
Subject = "Auth",
MetricsKey = metricsKey,
Timestamp = DateTime.UtcNow
});
await Task.Delay(TimeSpan.FromSeconds(60));
throw new Exception("Error, lipsum timeout of the cross distant cloud.");
}
}
var authorKey = Request.Headers[ApiKeyHeaderName].FirstOrDefault();
if (authorKey == null)
Request.Cookies.TryGetValue(ApiKeyHeaderName, out authorKey);
if (authorKey == null && metricsKey == null)
{
return AuthenticateResult.NoResult();
}
if (authorKey != null)
{
var owner = await _sql.LoginByApiKey(authorKey);
if (owner == null)
return AuthenticateResult.Fail("Invalid author key");
var claims = new List<Claim> {new Claim(ClaimTypes.Name, owner)};
claims.Add(new Claim(ClaimTypes.Role, "Author"));
claims.Add(new Claim(ClaimTypes.Role, "User"));
var identity = new ClaimsIdentity(claims, Options.AuthenticationType);
var identities = new List<ClaimsIdentity> {identity};
var principal = new ClaimsPrincipal(identities);
var ticket = new AuthenticationTicket(principal, Options.Scheme);
return AuthenticateResult.Success(ticket);
}
if (!await _keyCache.IsValidKey(metricsKey))
{
return AuthenticateResult.Fail("Invalid Metrics Key");
}
else
{
var claims = new List<Claim> {new(ClaimTypes.Role, "User")};
var identity = new ClaimsIdentity(claims, Options.AuthenticationType);
var identities = new List<ClaimsIdentity> {identity};
var principal = new ClaimsPrincipal(identities);
var ticket = new AuthenticationTicket(principal, Options.Scheme);
return AuthenticateResult.Success(ticket);
}
}
[JsonName("RequestLog")]
public class RequestLog
{
public string Path { get; set; }
public string Query { get; set; }
public Dictionary<string, string[]> Headers { get; set; }
}
private async Task LogRequest(string metricsKey)
{
var action = new RequestLog {
Path = Request.Path,
Query = Request.QueryString.Value,
Headers = Request.Headers.GroupBy(s => s.Key)
.ToDictionary(s => s.Key, s => s.SelectMany(v => v.Value).ToArray())
};
var ip = Request.Headers["CF-Connecting-IP"].FirstOrDefault() ??
Request.Headers["X-Forwarded-For"].FirstOrDefault() ??
Request.HttpContext.Connection.RemoteIpAddress.ToString();
await _sql.IngestAccess(ip, action.ToJson());
}
protected override async Task HandleChallengeAsync(AuthenticationProperties properties)
{
Response.StatusCode = 401;
Response.ContentType = ProblemDetailsContentType;
await Response.WriteAsync("Unauthorized");
}
protected override async Task HandleForbiddenAsync(AuthenticationProperties properties)
{
Response.StatusCode = 403;
Response.ContentType = ProblemDetailsContentType;
await Response.WriteAsync("forbidden");
}
}
public static class ApiKeyAuthorizationHandlerExtensions
{
public static AuthenticationBuilder AddApiKeySupport(this AuthenticationBuilder authenticationBuilder, Action<ApiKeyAuthenticationOptions> options)
{
return authenticationBuilder.AddScheme<ApiKeyAuthenticationOptions, ApiKeyAuthenticationHandler>(ApiKeyAuthenticationOptions.DefaultScheme, options);
}
}
}

40
Wabbajack.Server/AppSettings.cs
View File

@ -1,40 +0,0 @@
using Microsoft.Extensions.Configuration;
using Wabbajack.Common;
namespace Wabbajack.BuildServer
{
public class AppSettings
{
public AppSettings(IConfiguration config)
{
config.Bind("WabbajackSettings", this);
}
public string DownloadDir { get; set; }
public AbsolutePath DownloadPath => (AbsolutePath)DownloadDir;
public string ArchiveDir { get; set; }
public AbsolutePath ArchivePath => (AbsolutePath)ArchiveDir;
public string TempFolder { get; set; }
public AbsolutePath TempPath => (AbsolutePath)TempFolder;
public bool JobScheduler { get; set; }
public bool JobRunner { get; set; }
public bool RunFrontEndJobs { get; set; }
public bool RunBackEndJobs { get; set; }
public bool RunNexusPolling { get; set; }
public bool RunDownloader { get; set; }
public string BunnyCDN_StorageZone { get; set; }
public string SqlConnection { get; set; }
public int MaxJobs { get; set; } = 2;
public string SpamWebHook { get; set; } = null;
public string HamWebHook { get; set; } = null;
public bool ValidateModUpgrades { get; set; } = true;
}
}

19
Wabbajack.Server/Badge.cs
View File

@ -1,19 +0,0 @@
using Wabbajack.Common.Serialization.Json;
namespace Wabbajack.Server
{
[JsonName("Badge")]
public class Badge
{
public int schemaVersion { get; set; } = 1;
public string label { get; set; }
public string message { get; set; }
public string color { get; set; }
public Badge(string _label, string _message)
{
label = _label;
message = _message;
}
}
}

123
Wabbajack.Server/Controllers/AuthorControls.cs
View File

@ -1,123 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Security.Claims;
using System.Text;
using System.Threading.Tasks;
using FluentFTP;
using FluentFTP.Helpers;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.Common;
using Wabbajack.Lib.GitHub;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
[Authorize(Roles="Author")]
[Route("/author_controls")]
public class AuthorControls : ControllerBase
{
private ILogger<AuthorControls> _logger;
private SqlService _sql;
private readonly QuickSync _quickSync;
public AuthorControls(ILogger<AuthorControls> logger, SqlService sql, QuickSync quickSync)
{
_logger = logger;
_sql = sql;
_quickSync = quickSync;
}
[Route("login/{authorKey}")]
[AllowAnonymous]
public async Task<IActionResult> Login(string authorKey)
{
Response.Cookies.Append(ApiKeyAuthenticationHandler.ApiKeyHeaderName, authorKey);
return Redirect($"{Consts.WabbajackBuildServerUri}author_controls/home");
}
[Route("lists")]
[HttpGet]
public async Task<IActionResult> AuthorLists()
{
var user = User.FindFirstValue(ClaimTypes.Name);
List<string> lists = new();
var client = await Client.Get();
foreach (var file in Enum.GetValues<Client.List>())
{
lists.AddRange((await client.GetData(file)).Lists.Where(l => l.Maintainers.Contains(user))
.Select(lst => lst.Links.MachineURL));
}
return Ok(lists);
}
[Route("lists/download_metadata")]
[HttpPost]
public async Task<IActionResult> PostDownloadMetadata()
{
var user = User.FindFirstValue(ClaimTypes.Name);
var data = (await Request.Body.ReadAllTextAsync()).FromJsonString<UpdateRequest>();
var client = await Client.Get();
try
{
await client.UpdateList(user, data);
await _quickSync.Notify<ModListDownloader>();
}
catch (Exception ex)
{
_logger.LogError(ex, "During posting of download_metadata");
return BadRequest(ex);
}
return Ok(data);
}
private static async Task<string> HomePageTemplate(object o)
{
var data = await AbsolutePath.EntryPoint.Combine(@"Controllers\Templates\AuthorControls.html")
.ReadAllTextAsync();
var func = NettleEngine.GetCompiler().Compile(data);
return func(o);
}
[Route("home")]
[Authorize("")]
public async Task<IActionResult> HomePage()
{
var user = User.FindFirstValue(ClaimTypes.Name);
var files = (await _sql.AllAuthoredFiles())
.Where(af => af.Author == user)
.Select(af => new
{
Size = af.Size.FileSizeToString(),
OriginalSize = af.Size,
Name = af.OriginalFileName,
MangledName = af.MungedName,
UploadedDate = af.LastTouched
})
.OrderBy(f => f.Name)
.ThenBy(f => f.UploadedDate)
.ToList();
var result = HomePageTemplate(new
{
User = user,
TotalUsage = files.Select(f => f.OriginalSize).Sum().ToFileSizeString(),
WabbajackFiles = files.Where(f => f.Name.EndsWith(Consts.ModListExtensionString)),
OtherFiles = files.Where(f => !f.Name.EndsWith(Consts.ModListExtensionString))
});
return new ContentResult {
ContentType = "text/html",
StatusCode = (int)HttpStatusCode.OK,
Content = await result};
}
}
}

204
Wabbajack.Server/Controllers/AuthoredFiles.cs
View File

@ -1,204 +0,0 @@
using System;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Net;
using System.Security.Claims;
using System.Threading.Tasks;
using FluentFTP;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using SharpCompress.Compressors.LZMA;
using Wabbajack.Common;
using Wabbajack.Lib.AuthorApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
[Authorize(Roles = "Author")]
[Route("/authored_files")]
public class AuthoredFiles : ControllerBase
{
private SqlService _sql;
private ILogger<AuthoredFiles> _logger;
private AppSettings _settings;
private CDNMirrorList _mirrorList;
private DiscordWebHook _discord;
public AuthoredFiles(ILogger<AuthoredFiles> logger, SqlService sql, AppSettings settings, CDNMirrorList mirrorList, DiscordWebHook discord)
{
_sql = sql;
_logger = logger;
_settings = settings;
_mirrorList = mirrorList;
_discord = discord;
}
[HttpPut]
[Route("{serverAssignedUniqueId}/part/{index}")]
public async Task<IActionResult> UploadFilePart(string serverAssignedUniqueId, long index)
{
var user = User.FindFirstValue(ClaimTypes.Name);
var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId);
if (definition.Author != user)
return Forbid("File Id does not match authorized user");
_logger.Log(LogLevel.Information, $"Uploading File part {definition.OriginalFileName} - ({index} / {definition.Parts.Length})");
await _sql.TouchAuthoredFile(definition);
var part = definition.Parts[index];
await using var ms = new MemoryStream();
await Request.Body.CopyToLimitAsync(ms, part.Size);
ms.Position = 0;
if (ms.Length != part.Size)
return BadRequest($"Couldn't read enough data for part {part.Size} vs {ms.Length}");
var hash = await ms.xxHashAsync();
if (hash != part.Hash)
return BadRequest($"Hashes don't match for index {index}. Sizes ({ms.Length} vs {part.Size}). Hashes ({hash} vs {part.Hash}");
ms.Position = 0;
await UploadAsync(ms, $"{definition.MungedName}/parts/{index}");
return Ok(part.Hash.ToBase64());
}
[HttpPut]
[Route("create")]
public async Task<IActionResult> CreateUpload()
{
var user = User.FindFirstValue(ClaimTypes.Name);
var data = await Request.Body.ReadAllTextAsync();
var definition = data.FromJsonString<CDNFileDefinition>();
_logger.Log(LogLevel.Information, $"Creating File upload {definition.OriginalFileName}");
definition = await _sql.CreateAuthoredFile(definition, user);
using (var client = await GetBunnyCdnFtpClient())
{
await client.CreateDirectoryAsync($"{definition.MungedName}");
await client.CreateDirectoryAsync($"{definition.MungedName}/parts");
}
await _discord.Send(Channel.Ham,
new DiscordMessage() {Content = $"{user} has started uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})"});
return Ok(definition.ServerAssignedUniqueId);
}
[HttpPut]
[Route("{serverAssignedUniqueId}/finish")]
public async Task<IActionResult> CreateUpload(string serverAssignedUniqueId)
{
var user = User.FindFirstValue(ClaimTypes.Name);
var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId);
if (definition.Author != user)
return Forbid("File Id does not match authorized user");
_logger.Log(LogLevel.Information, $"Finalizing file upload {definition.OriginalFileName}");
await _sql.Finalize(definition);
await using var ms = new MemoryStream();
await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
{
definition.ToJson(gz);
}
ms.Position = 0;
await UploadAsync(ms, $"{definition.MungedName}/definition.json.gz");
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"{user} has finished uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})"});
var host = Consts.TestMode ? "test-files" : "authored-files";
return Ok($"https://{host}.wabbajack.org/{definition.MungedName}");
}
private async Task<FtpClient> GetBunnyCdnFtpClient()
{
var info = await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles);
var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
await client.ConnectAsync();
return client;
}
private async Task UploadAsync(Stream stream, string path)
{
using var client = await GetBunnyCdnFtpClient();
await client.UploadAsync(stream, path);
}
[HttpDelete]
[Route("{serverAssignedUniqueId}")]
public async Task<IActionResult> DeleteUpload(string serverAssignedUniqueId)
{
var user = User.FindFirstValue(ClaimTypes.Name);
var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId);
if (definition.Author != user)
return Forbid("File Id does not match authorized user");
await _discord.Send(Channel.Ham, new DiscordMessage() {Content = $"{user} is deleting {definition.MungedName}, {definition.Size.ToFileSizeString()} to be freed"});
_logger.Log(LogLevel.Information, $"Deleting upload {definition.OriginalFileName}");
await DeleteFolderOrSilentlyFail($"{definition.MungedName}");
await _sql.DeleteFileDefinition(definition);
return Ok();
}
private async Task DeleteFolderOrSilentlyFail(string path)
{
try
{
using var client = await GetBunnyCdnFtpClient();
await client.DeleteDirectoryAsync(path);
}
catch (Exception)
{
_logger.Log(LogLevel.Information, $"Delete failed for {path}");
}
}
private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
<html><body>
<table>
{{each $.files }}
<tr><td><a href='https://authored-files.wabbajack.org/{{$.MungedName}}'>{{$.OriginalFileName}}</a></td><td>{{$.Size}}</td><td>{{$.LastTouched}}</td><td>{{$.Finalized}}</td><td>{{$.Author}}</td></tr>
{{/each}}
</table>
</body></html>
");
[HttpGet]
[AllowAnonymous]
[Route("")]
public async Task<ContentResult> UploadedFilesGet()
{
var files = await _sql.AllAuthoredFiles();
var response = HandleGetListTemplate(new {files});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
[HttpGet]
[AllowAnonymous]
[Route("mirrors")]
public async Task<IActionResult> GetMirrorList()
{
Response.Headers.Add("x-last-updated", _mirrorList.LastUpdate.ToString(CultureInfo.InvariantCulture));
return Ok(_mirrorList.Mirrors);
}
}
}

83
Wabbajack.Server/Controllers/GameFiles.cs
View File

@ -1,83 +0,0 @@
#nullable enable
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
[Route("/game_files")]
public class EnqueueGameFiles : ControllerBase
{
private readonly ILogger<EnqueueGameFiles> _logger;
private readonly SqlService _sql;
private readonly QuickSync _quickSync;
public EnqueueGameFiles(ILogger<EnqueueGameFiles> logger, SqlService sql, QuickSync quickSync)
{
_logger = logger;
_sql = sql;
_quickSync = quickSync;
}
[Authorize(Roles = "Author")]
[HttpGet("enqueue")]
public async Task<IActionResult> Enqueue()
{
var games = GameRegistry.Games.Where(g => g.Value.IsInstalled).Select(g => g.Value).ToList();
_logger.Log(LogLevel.Information, $"Found {games.Count} installed games");
var files = games.SelectMany(game =>
game.GameLocation().EnumerateFiles(true).Select(file => new {File = file, Game = game})).ToList();
_logger.Log(LogLevel.Information, $"Found {files.Count} game files");
using var queue = new WorkQueue();
var hashed = (await files.PMap(queue, async pair =>
{
var hash = await pair.File.FileHashCachedAsync();
if (hash == null) return null;
return await _sql.GetOrEnqueueArchive(new Archive(new GameFileSourceDownloader.State
{
Game = pair.Game.Game,
GameFile = pair.File.RelativeTo(pair.Game.GameLocation()),
GameVersion = pair.Game.InstalledVersion,
Hash = hash.Value
}) {Name = pair.File.FileName.ToString(), Size = pair.File.Size, Hash = hash.Value});
})).NotNull();
await _quickSync.Notify<ArchiveDownloader>();
return Ok(hashed);
}
[Authorize(Roles = "User")]
[HttpGet("{game}/{version}")]
public async Task<IActionResult> GetFiles(string game, string version)
{
if (!GameRegistry.TryGetByFuzzyName(game, out var meta))
return NotFound($"Game {game} not found");
var files = await _sql.GetGameFiles(meta.Game, version);
return Ok(files.ToJson());
}
[Authorize(Roles = "User")]
[HttpGet]
public async Task<IActionResult> GetAllGames()
{
var registeredGames = await _sql.GetAllRegisteredGames();
return Ok(registeredGames.ToArray().ToJson());
}
}
}

161
Wabbajack.Server/Controllers/Heartbeat.cs
View File

@ -1,161 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Server;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
[Route("/heartbeat")]
public class Heartbeat : ControllerBase
{
static Heartbeat()
{
_startTime = DateTime.Now;
}
private static DateTime _startTime;
private QuickSync _quickSync;
private ListValidator _listValidator;
public Heartbeat(ILogger<Heartbeat> logger, SqlService sql, GlobalInformation globalInformation, QuickSync quickSync, ListValidator listValidator)
{
_globalInformation = globalInformation;
_sql = sql;
_logger = logger;
_quickSync = quickSync;
_listValidator = listValidator;
}
private const int MAX_LOG_SIZE = 128;
private static List<string> Log = new();
private GlobalInformation _globalInformation;
private SqlService _sql;
private ILogger<Heartbeat> _logger;
public static void AddToLog(IStatusMessage msg)
{
lock (Log)
{
Log.Add(msg.ToString());
if (Log.Count > MAX_LOG_SIZE)
Log.RemoveAt(0);
}
}
[HttpGet]
public async Task<IActionResult> GetHeartbeat()
{
return Ok(new HeartbeatResult
{
Uptime = DateTime.Now - _startTime,
LastNexusUpdate = _globalInformation.TimeSinceLastNexusSync,
});
}
private static readonly Func<object, string> HandleGetReport = NettleEngine.GetCompiler().Compile(@"
<html><body>
<h2>Server Status</h2>
<h3>Service Overview ({{services.Length}}):</h3>
<ul>
{{each $.services }}
{{if $.IsLate}}
<li><a href='/heartbeat/report/services/{{$.Name}}.html'><b>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</b></a></li>
{{else}}
<li><a href='/heartbeat/report/services/{{$.Name}}.html'>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</a></li>
{{/if}}
{{/each}}
</ul>
<h3>Lists ({{lists.Length}}):</h3>
<ul>
{{each $.lists }}
<li><a href='/lists/status/{{$.Name}}.html'>{{$.Name}}</a> - {{$.Time}} {{$.FailMessage}}</li>
{{/each}}
</ul>
</body></html>
");
[HttpGet("report")]
public async Task<ContentResult> Report()
{
var response = HandleGetReport(new
{
services = (await _quickSync.Report())
.Select(s => new {Name = s.Key.Name, Time = s.Value.LastRunTime, MaxTime = s.Value.Delay, IsLate = s.Value.LastRunTime > s.Value.Delay})
.OrderBy(s => s.Name)
.ToArray(),
lists = _listValidator.ValidationInfo.Select(s => new
{
Name = s.Key,
Time = s.Value.ValidationTime,
FailMessage = s.Value.Detailed.HasFailures ? "Failed" : ""
})
.OrderBy(l => l.Name)
.ToArray()
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
private static readonly Func<object, string> HandleGetServiceReport = NettleEngine.GetCompiler().Compile(@"
<html><body>
<h2>Service Status: {{Name}} {{TimeSinceLastRun}}</h2>
<h3>Service Overview ({{ActiveWorkQueue.Length}}):</h3>
<ul>
{{each $.ActiveWorkQueue }}
<li>{{$.Name}} {{$.Time}}</li>
{{/each}}
</ul>
</body></html>
");
[HttpGet("report/services/{serviceName}.html")]
public async Task<ContentResult> ReportServiceStatus(string serviceName)
{
var services = await _quickSync.Report();
var info = services.First(kvp => kvp.Key.Name == serviceName);
var response = HandleGetServiceReport(new
{
Name = info.Key.Name,
TimeSinceLastRun = DateTime.UtcNow - info.Value.LastRunTime,
ActiveWorkQueue = info.Value.ActiveWork.Select(p => new
{
Name = p.Item1,
Time = DateTime.UtcNow - p.Item2
}).OrderByDescending(kp => kp.Time)
.ToArray()
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
}
}

80
Wabbajack.Server/Controllers/ListDefinitions.cs
View File

@ -1,80 +0,0 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
[ApiController]
[Route("/list_definitions")]
public class ListDefinitions : ControllerBase
{
private ILogger<ListDefinitions> _logger;
private SqlService _sql;
private DiscordWebHook _discord;
public ListDefinitions(ILogger<ListDefinitions> logger, SqlService sql, DiscordWebHook discord)
{
_logger = logger;
_sql = sql;
_discord = discord;
}
[Route("ingest")]
[Authorize(Roles = "User")]
[HttpPost]
public async Task<IActionResult> PostIngest()
{
var user = Request.Headers[Consts.MetricsKeyHeader].First();
var use_gzip = Request.Headers[Consts.CompressedBodyHeader].Any();
_logger.Log(LogLevel.Information, $"Ingesting Modlist Definition for {user}");
var modlistBytes = await Request.Body.ReadAllAsync();
_logger.LogInformation("Spawning ingestion task");
var tsk = Task.Run(async () =>
{
try
{
if (use_gzip)
{
await using var os = new MemoryStream();
await using var gZipStream =
new GZipStream(new MemoryStream(modlistBytes), CompressionMode.Decompress);
await gZipStream.CopyToAsync(os);
modlistBytes = os.ToArray();
}
var modlist = new MemoryStream(modlistBytes).FromJson<ModList>();
var file = AbsolutePath.EntryPoint.Combine("mod_list_definitions")
.Combine($"{user}_{DateTime.UtcNow.ToFileTimeUtc()}.json");
file.Parent.CreateDirectory();
await using var stream = await file.Create();
modlist.ToJson(stream);
_logger.Log(LogLevel.Information, $"Done Ingesting Modlist Definition for {user}");
}
catch (Exception ex)
{
_logger.LogError(ex, "Error ingesting uploaded modlist");
}
});
return Accepted(0);
}
}
}

235
Wabbajack.Server/Controllers/ListsStatus.cs
View File

@ -1,235 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using ArchiveStatus = Wabbajack.Server.DTOs.ArchiveStatus;
namespace Wabbajack.BuildServer.Controllers
{
[ApiController]
[Route("/lists")]
public class ListsStatus : ControllerBase
{
private ILogger<ListsStatus> _logger;
private ListValidator _validator;
public ListsStatus(ILogger<ListsStatus> logger, ListValidator validator)
{
_logger = logger;
_validator = validator;
}
[HttpGet]
[Route("status.json")]
public async Task<IEnumerable<ModListSummary>> HandleGetLists()
{
return (_validator.Summaries).Select(d => d.Summary);
}
private static readonly Func<object, string> HandleGetRssFeedTemplate = NettleEngine.GetCompiler().Compile(@"
<?xml version=""1.0""?>
<rss version=""2.0"">
<channel>
<title>{{lst.Name}} - Broken Mods</title>
<link>http://build.wabbajack.org/status/{{lst.Name}}.html</link>
<description>These are mods that are broken and need updating</description>
{{ each $.failed }}
<item>
<title>{{$.Archive.Name}} {{$.Archive.Hash}} {{$.Archive.State.PrimaryKeyString}}</title>
<link>{{$.Archive.Name}}</link>
</item>
{{/each}}
</channel>
</rss>
");
[HttpGet]
[Route("status/{Name}/broken.rss")]
public async Task<ContentResult> HandleGetRSSFeed(string Name)
{
var lst = await DetailedStatus(Name);
var response = HandleGetRssFeedTemplate(new
{
lst,
failed = lst.Archives.Where(a => a.IsFailing).ToList(),
passed = lst.Archives.Where(a => !a.IsFailing).ToList()
});
return new ContentResult
{
ContentType = "application/rss+xml",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
<html><body>
<h2>{{lst.Name}} - {{lst.Checked}} - {{ago}}min ago</h2>
<h3>Failed ({{failed.Count}}):</h3>
<ul>
{{each $.failed }}
{{if $.HasUrl}}
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}
{{/each}}
</ul>
<h3>Updated ({{updated.Count}}):</h3>
<ul>
{{each $.updated }}
{{if $.HasUrl}}
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}
{{/each}}
</ul>
<h3>Mirrored ({{mirrored.Count}}):</h3>
<ul>
{{each $.mirrored }}
{{if $.HasUrl}}
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}
{{/each}}
</ul>
<h3>Updating ({{updating.Count}}):</h3>
<ul>
{{each $.updating }}
{{if $.HasUrl}}
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}
{{/each}}
</ul>
<h3>Passed ({{passed.Count}}):</h3>
<ul>
{{each $.passed }}
{{if $.HasUrl}}
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}
{{/each}}
</ul>
</body></html>
");
[HttpGet]
[Route("status/{Name}.html")]
public async Task<ContentResult> HandleGetListHtml(string Name)
{
var lst = await DetailedStatus(Name);
var response = HandleGetListTemplate(new
{
lst,
ago = (DateTime.UtcNow - lst.Checked).TotalMinutes,
failed = lst.Archives.Where(a => a.IsFailing).ToList(),
passed = lst.Archives.Where(a => !a.IsFailing).ToList(),
updated = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Updated).ToList(),
updating = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Updating).ToList(),
mirrored = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Mirrored).ToList()
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
[HttpGet]
[Route("status/{Name}.json")]
[ResponseCache(Duration = 60 * 5)]
public async Task<IActionResult> HandleGetListJson(string Name)
{
var lst = await DetailedStatus(Name);
if (lst == null) return NotFound();
return Ok(lst.ToJson());
}
private async Task<DetailedStatus> DetailedStatus(string Name)
{
var results = _validator.Summaries
.Select(d => d.Detailed)
.FirstOrDefault(d => d.MachineName == Name);
if (results == null)
return null;
results!.Archives.Do(itm =>
{
if (string.IsNullOrWhiteSpace(itm.Archive.Name))
itm.Archive.Name = itm.Archive.State.PrimaryKeyString;
});
results.Archives = results.Archives.OrderBy(a => a.Name).ToList();
return results;
}
[HttpGet]
[Route("status/badge.json")]
public async Task<IActionResult> HandleGitHubBadge()
{
//var failing = _validator.Summaries.Select(x => x.Summary.Failed).Aggregate((x, y) => x + y);
var succeeding = _validator.Summaries.Select(x => x.Summary.Passed).Aggregate((x, y) => x + y);
var total = _validator.Summaries.Count();
double ration = total / (double)succeeding * 100.0;
string color;
if (ration >= 95)
color = "brightgreen";
else if (ration >= 80)
color = "green";
else if (ration >= 50)
color = "yellowgreen";
else if (ration >= 20)
color = "orange";
else
color = "red";
Response.ContentType = "application/json";
return Ok(new Badge("Modlist Availability", $"{ration}%"){color = color});
}
[HttpGet]
[Route("status/{Name}/badge.json")]
public async Task<IActionResult> HandleNamedGitHubBadge(string Name)
{
var info = _validator.Summaries.Select(x => x.Summary)
.FirstOrDefault(x => x.MachineURL == Name);
if (info == null)
return new NotFoundObjectResult("Not Found!");
var failing = info.HasFailures;
Response.ContentType = "application/json";
return Ok(new Badge(info.Name, failing ? "Failing" : "Succeeding"){color = failing ? "red" : "green"});
}
}
}

245
Wabbajack.Server/Controllers/Metrics.cs
View File

@ -1,245 +0,0 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.Common;
using Wabbajack.Server;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using WebSocketSharp;
using LogLevel = Microsoft.Extensions.Logging.LogLevel;
namespace Wabbajack.BuildServer.Controllers
{
[ApiController]
[Route("/metrics")]
public class MetricsController : ControllerBase
{
private SqlService _sql;
private ILogger<MetricsController> _logger;
private MetricsKeyCache _keyCache;
public MetricsController(ILogger<MetricsController> logger, SqlService sql, MetricsKeyCache keyCache)
{
_sql = sql;
_logger = logger;
_keyCache = keyCache;
}
[HttpGet]
[Route("{subject}/{value}")]
public async Task<Result> LogMetricAsync(string subject, string value)
{
var date = DateTime.UtcNow;
var metricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault();
if (metricsKey != null)
await _keyCache.AddKey(metricsKey);
// Used in tests
if (value == "Default" || value == "untitled" || subject == "failed_download" || Guid.TryParse(value, out _))
return new Result { Timestamp = date};
await Log(date, subject, value, metricsKey);
return new Result { Timestamp = date};
}
[HttpGet]
[Route("report/{subject}")]
[ResponseCache(Duration = 60 * 60)]
public async Task<IActionResult> MetricsReport(string subject)
{
var metrics = (await _sql.MetricsReport(subject)).ToList();
var labels = metrics.GroupBy(m => m.Date)
.OrderBy(m => m.Key)
.Select(m => m.Key)
.ToArray();
var labelStrings = labels.Select(l => l.ToString("MM-dd-yyy")).ToList();
var results = metrics
.GroupBy(m => m.Subject)
.Select(g =>
{
var indexed = g.ToDictionary(m => m.Date, m => m.Count);
return new MetricResult
{
SeriesName = g.Key,
Labels = labelStrings,
Values = labels.Select(l => indexed.TryGetValue(l, out var found) ? found : 0).ToList()
};
});
return Ok(results.ToList());
}
[HttpGet]
[Route("badge/{name}/total_installs_badge.json")]
public async Task<IActionResult> TotalInstallsBadge(string name)
{
var results = await _sql.TotalInstalls(name);
Response.ContentType = "application/json";
return Ok(results == 0
? new Badge($"Modlist {name} not found!", "Error") {color = "red"}
: new Badge("Installations: ", "____") {color = "green"});
}
[HttpGet]
[Route("badge/{name}/unique_installs_badge.json")]
public async Task<IActionResult> UniqueInstallsBadge(string name)
{
var results = await _sql.UniqueInstalls(name);
Response.ContentType = "application/json";
return Ok(results == 0
? new Badge($"Modlist {name} not found!", "Error") {color = "red"}
: new Badge("Installations: ", "____"){color = "green"}) ;
}
private static readonly Func<object, string> ReportTemplate = NettleEngine.GetCompiler().Compile(@"
<html><body>
<h2>Tar Report for {{$.key}}</h2>
<h3>Ban Status: {{$.status}}</h3>
<table>
{{each $.log }}
<tr>
<td>{{$.Timestamp}}</td>
<td>{{$.Path}}</td>
<td>{{$.Key}}</td>
</tr>
{{/each}}
</table>
</body></html>
");
[HttpGet]
[Route("tarlog/{key}")]
public async Task<IActionResult> TarLog(string key)
{
var isTarKey = await _sql.IsTarKey(key);
List<(DateTime, string, string)> report = new List<(DateTime, string, string)>();
if (isTarKey) report = await _sql.FullTarReport(key);
var response = ReportTemplate(new
{
key = key,
status = isTarKey ? "BANNED" : "NOT BANNED",
log = report.Select(entry => new
{
Timestamp = entry.Item1,
Path = entry.Item2,
Key = entry.Item3
}).ToList()
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
{
//_logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}");
await _sql.IngestMetric(new Metric
{
Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey
});
}
public class Result
{
public DateTime Timestamp { get; set; }
}
class TotalListTemplateData
{
public string Title { get; set; }
public long Total { get; set; }
public Item[] Items { get; set; }
public class Item
{
public long Count { get; set; }
public string Title { get; set; }
}
}
private static Func<object, string> _totalListTemplate;
private static Func<object, string> TotalListTemplate
{
get
{
if (_totalListTemplate == null)
{
var resource = Assembly.GetExecutingAssembly()
.GetManifestResourceStream("Wabbajack.Server.Controllers.Templates.TotalListTemplate.html")!
.ReadAll();
_totalListTemplate = NettleEngine.GetCompiler().Compile(Encoding.UTF8.GetString(resource));
}
return _totalListTemplate;
}
}
[HttpGet("total_installs.html")]
[ResponseCache(Duration = 60 * 60)]
public async Task<ContentResult> TotalInstalls()
{
var data = await _sql.GetTotalInstalls();
var result = TotalListTemplate(new TotalListTemplateData
{
Title = "Total Installs",
Total = data.Sum(d => d.Item2),
Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2})
.ToArray()
});
return new ContentResult {
ContentType = "text/html",
StatusCode = (int)HttpStatusCode.OK,
Content = result};
}
[HttpGet("total_unique_installs.html")]
[ResponseCache(Duration = 60 * 60)]
public async Task<ContentResult> TotalUniqueInstalls()
{
var data = await _sql.GetTotalUniqueInstalls();
var result = TotalListTemplate(new TotalListTemplateData
{
Title = "Total Unique Installs",
Total = data.Sum(d => d.Item2),
Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2})
.ToArray()
});
return new ContentResult {
ContentType = "text/html",
StatusCode = (int)HttpStatusCode.OK,
Content = result};
}
[HttpGet("dump.json")]
public async Task<IActionResult> DataDump()
{
return Ok(await _sql.MetricsDump().ToArrayAsync());
}
}
}

31
Wabbajack.Server/Controllers/ModFiles.cs
View File

@ -1,31 +0,0 @@
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.BuildServer.Controllers
{
[Authorize(Roles = "User")]
[ApiController]
[Route("/mod_files")]
public class ModFiles : ControllerBase
{
private SqlService _sql;
private ILogger<ModFiles> _logger;
public ModFiles(ILogger<ModFiles> logger, SqlService sql)
{
_logger = logger;
_sql = sql;
}
[HttpGet("by_hash/{hashAsHex}")]
public async Task<IActionResult> GetByHash(string hashAsHex)
{
var files = await _sql.ResolveDownloadStatesByHash(Hash.FromHex(hashAsHex));
return Ok(files.ToJson());
}
}
}

164
Wabbajack.Server/Controllers/ModUpgrade.cs
View File

@ -1,164 +0,0 @@
using System;
using System.Linq;
using System.Security.Claims;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
[ApiController]
public class ModUpgrade : ControllerBase
{
private ILogger<ModUpgrade> _logger;
private SqlService _sql;
private DiscordWebHook _discord;
private AppSettings _settings;
private QuickSync _quickSync;
private Task<BunnyCdnFtpInfo> _creds;
private Task<BunnyCdnFtpInfo> _mirrorCreds;
public ModUpgrade(ILogger<ModUpgrade> logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync, AppSettings settings)
{
_logger = logger;
_sql = sql;
_discord = discord;
_settings = settings;
_quickSync = quickSync;
_creds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches);
_mirrorCreds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
}
[HttpPost]
[Authorize(Roles = "User")]
[Route("/mod_upgrade")]
public async Task<IActionResult> PostModUpgrade()
{
var isAuthor = User.Claims.Any(c => c.Type == ClaimTypes.Role && c.Value == "Author");
var request = (await Request.Body.ReadAllTextAsync()).FromJsonString<ModUpgradeRequest>();
if (!isAuthor)
{
var srcDownload = await _sql.GetArchiveDownload(request.OldArchive.State.PrimaryKeyString,
request.OldArchive.Hash, request.OldArchive.Size);
var destDownload = await _sql.GetArchiveDownload(request.NewArchive.State.PrimaryKeyString,
request.NewArchive.Hash, request.NewArchive.Size);
if (srcDownload == default || destDownload == default ||
await _sql.FindPatch(srcDownload.Id, destDownload.Id) == default)
{
if (!await request.IsValid())
{
_logger.Log(LogLevel.Information,
$"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as upgrade is invalid");
return BadRequest("Invalid mod upgrade");
}
if (_settings.ValidateModUpgrades && !await _sql.HashIsInAModlist(request.OldArchive.Hash))
{
_logger.Log(LogLevel.Information,
$"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as src hash is not in a curated modlist");
return BadRequest("Hash is not in a recent modlist");
}
}
}
try
{
if (await request.OldArchive.State.Verify(request.OldArchive))
{
//_logger.LogInformation(
// $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), old archive is valid");
return NotFound("File is Valid");
}
}
catch (Exception)
{
//_logger.LogInformation(
// $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), due to upgrade failure");
return NotFound("File is Valid");
}
var oldDownload = await _sql.GetOrEnqueueArchive(request.OldArchive);
if (await _sql.IsNoPatch(oldDownload.Archive.Hash))
{
return BadRequest("File has NoPatch attached");
}
var newDownload = await _sql.GetOrEnqueueArchive(request.NewArchive);
var patch = await _sql.FindOrEnqueuePatch(oldDownload.Id, newDownload.Id);
if (patch.Finished.HasValue)
{
if (patch.PatchSize != 0)
{
//_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found");
var host = (await _creds).Username == "wabbajacktest" ? "test-files" : "patches";
await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id);
return
Ok(
$"https://{host}.wabbajack.org/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
}
//_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed");
return NotFound("Patch creation failed");
}
if (!newDownload.DownloadFinished.HasValue)
{
await _quickSync.Notify<ArchiveDownloader>();
}
else
{
await _quickSync.Notify<PatchBuilder>();
}
//_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found is processing");
// Still processing
return Accepted();
}
[HttpGet]
[Authorize(Roles = "User")]
[Route("/mod_upgrade/find/{hashAsHex}")]
public async Task<IActionResult> FindUpgrade(string hashAsHex)
{
var hash = Hash.FromHex(hashAsHex);
var patches = await _sql.PatchesForSource(hash);
return Ok(patches.Select(p => p.Dest.Archive).ToList().ToJson());
}
[HttpGet]
[Authorize(Roles = "Author")]
[Route("/mod_upgrade/no_patch/{hashAsHex}/{rationaleAsHex}")]
public async Task<IActionResult> PurgePatch(string hashAsHex, string rationaleAsHex)
{
var hash = Hash.FromHex(hashAsHex);
var rationale = Encoding.UTF8.GetString(rationaleAsHex.FromHex());
await _sql.PurgePatch(hash, rationale);
return Ok("Purged");
}
[HttpGet]
[Authorize(Roles = "User")]
[Route("/mirror/{hashAsHex}")]
public async Task<IActionResult> HaveHash(string hashAsHex)
{
var result = await _sql.HaveMirror(Hash.FromHex(hashAsHex));
if (result) return Ok($"https://{(await _mirrorCreds).Username}.b-cdn.net/{hashAsHex}");
return NotFound("Not Mirrored");
}
}
}

175
Wabbajack.Server/Controllers/NexusCache.cs
View File

@ -1,175 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Common.Exceptions;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
//[Authorize]
[ApiController]
[Authorize(Roles = "User")]
[Route("/v1/games/")]
public class NexusCache : ControllerBase
{
private AppSettings _settings;
private static long CachedCount = 0;
private static long ForwardCount = 0;
private SqlService _sql;
private ILogger<NexusCache> _logger;
private NexusKeyMaintainance _keys;
public NexusCache(ILogger<NexusCache> logger, SqlService sql, AppSettings settings, NexusKeyMaintainance keys)
{
_settings = settings;
_sql = sql;
_logger = logger;
_keys = keys;
}
/// <summary>
/// Looks up the mod details for a given Gamename/ModId pair. If the entry is not found in the cache it will
/// be requested from the server (using the caller's Nexus API key if provided).
/// </summary>
/// <param name="db"></param>
/// <param name="GameName">The Nexus game name</param>
/// <param name="ModId">The Nexus mod id</param>
/// <returns>A Mod Info result</returns>
[HttpGet]
[Route("{GameName}/mods/{ModId}.json")]
public async Task<ModInfo> GetModInfo(string GameName, long ModId)
{
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var result = await _sql.GetNexusModInfoString(game, ModId);
string method = "CACHED";
if (result == null)
{
var api = await GetClient();
result = await api.GetModInfo(game, ModId, false);
await _sql.AddNexusModInfo(game, ModId, result.updated_time, result);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);
}
else
{
Interlocked.Increment(ref CachedCount);
}
Response.Headers.Add("x-cache-result", method);
return result;
}
private async Task<NexusApiClient> GetClient()
{
var key = Request.Headers["apikey"].FirstOrDefault();
if (key == null)
return await _keys.GetClient();
if (await _sql.HaveKey(key))
return await NexusApiClient.Get(key);
var client = await NexusApiClient.Get(key);
var (daily, hourly) = await client.GetRemainingApiCalls();
await _sql.SetNexusAPIKey(key, daily, hourly);
return client;
}
[HttpGet]
[Route("{GameName}/mods/{ModId}/files.json")]
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, long ModId)
{
//_logger.Log(LogLevel.Information, $"{GameName} {ModId}");
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var result = await _sql.GetModFiles(game, ModId);
string method = "CACHED";
if (result == null)
{
var api = await GetClient();
try
{
result = await api.GetModFiles(game, ModId, false);
}
catch (HttpException ex)
{
if (ex.Code == 403)
result = new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>()};
else
throw;
}
var date = result.files.Select(f => f.uploaded_time).OrderByDescending(o => o).FirstOrDefault();
date = date == default ? DateTime.UtcNow : date;
await _sql.AddNexusModFiles(game, ModId, date, result);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);
}
else
{
Interlocked.Increment(ref CachedCount);
}
Response.Headers.Add("x-cache-result", method);
return result;
}
[HttpGet]
[Route("{GameName}/mods/{ModId}/files/{FileId}.json")]
public async Task<ActionResult<NexusFileInfo>> GetModFile(string GameName, long ModId, long FileId)
{
try
{
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var result = await _sql.GetModFile(game, ModId, FileId);
string method = "CACHED";
if (result == null)
{
var api = await GetClient();
result = await api.GetModFile(game, ModId, FileId, false);
var date = result.uploaded_time;
date = date == default ? DateTime.UtcNow : date;
await _sql.AddNexusModFile(game, ModId, FileId, date, result);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);
}
else
{
Interlocked.Increment(ref CachedCount);
}
Response.Headers.Add("x-cache-result", method);
return result;
}
catch (Exception ex)
{
_logger.LogInformation("Unable to find mod file {GameName} {ModId}, {FileId}", GameName, ModId, FileId);
return NotFound();
}
}
[HttpGet]
[Authorize(Roles ="Author")]
[Route("/purge_nexus_cache/{ModId}")]
public async Task<IActionResult> PurgeNexusCache(long ModId)
{
_logger.LogInformation($"Purging nexus cache for {ModId}");
await _sql.PurgeNexusCache(ModId);
return Ok("Purged");
}
}
}

38
Wabbajack.Server/Controllers/SiteIntegration.cs
View File

@ -1,38 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.BuildServer.Controllers
{
[ApiController]
[Authorize(Roles = "User")]
[Route("/site-integration")]
public class SiteIntegration : ControllerBase
{
private ILogger<SiteIntegration> _logger;
public SiteIntegration(ILogger<SiteIntegration> logger)
{
_logger = logger;
}
private HashSet<string> Allowed = new HashSet<string>
{ "loverslabcookies", "deadlystream", "tesall", "tesalliance", "vectorplexus"};
[Route("auth-info/{site}")]
public async Task<IActionResult> GetAuthInfo(string site)
{
if (!Allowed.Contains(site))
{
return BadRequest("No key found");
}
return Ok(Encoding.UTF8.GetString(await Utils.FromEncryptedData(site)));
}
}
}

70
Wabbajack.Server/Controllers/Templates/AuthorControls.html
View File

@ -1,70 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Author Controls - {{$.User}} - {{$.TotalUsage}}</title>
</head>
<body>
<h2>Author Controls - {{$.User}} - {{$.TotalUsage}}</h2>
<br/>
<h3>Wabbajack Files</h3>
<table>
<tr>
<td><b>Commands</b></td>
<td><b>Name</b></td>
<td><b>Size</b></td>
<td><b>Finished Uploading</b></td>
<td><b>Unique Name</b></td>
</tr>
{{each $.WabbajackFiles }}
<tr>
<td><button onclick="deleteFile('{{$.MangledName}}');">Delete</button></td>
<td>{{$.Name}}</td>
<td>{{$.Size}}</td>
<td>{{$.UploadedDate}}</td>
<td>{{$.MangledName}}</td>
</tr>
{{/each}}
</table>
<h3>Other Files</h3>
<table>
<tr>
<td><b>Commands</b></td>
<td><b>Name</b></td>
<td><b>Size</b></td>
<td><b>Finished Uploading</b></td>
<td><b>Unique Name</b></td>
</tr>
{{each $.OtherFiles }}
<tr>
<td><button onclick="deleteFile('{{$.MangledName}}');">Delete</button></td>
<td>{{$.Name}}</td>
<td>{{$.Size}}</td>
<td>{{$.UploadedDate}}</td>
<td>{{$.MangledName}}</td>
</tr>
{{/each}}
</table>
<script lang="javascript">
if (!Array.prototype.last){
Array.prototype.last = function(){
return this[this.length - 1];
};
};
function deleteFile(mangled)
{
if(window.confirm("Are you sure you want to delete: " + mangled)) {
fetch("/authored_files/"+mangled.split("_").last(), {method: "DELETE"})
.then(r => location.reload());
}
}
</script>
</body>
</html>

21
Wabbajack.Server/Controllers/Templates/TotalListTemplate.html
View File

@ -1,21 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Total Installs</title>
</head>
<body>
<h2>{{$.Title}} - Total: {{$.Total}}</h2>
<table>
{{each $.Items }}
<tr>
<td>{{$.Count}}</td>
<td>{{$.Title}}</td>
</tr>
{{/each}}
</table>
</body>
</html>

51
Wabbajack.Server/Controllers/Users.cs
View File

@ -1,51 +0,0 @@
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.BuildServer.Controllers
{
[Authorize]
[Route("/users")]
public class Users : ControllerBase
{
private AppSettings _settings;
private ILogger<Users> _logger;
private SqlService _sql;
public Users(ILogger<Users> logger, SqlService sql, AppSettings settings)
{
_settings = settings;
_logger = logger;
_sql = sql;
}
[HttpGet]
[Route("add/{Name}")]
public async Task<string> AddUser(string Name)
{
return await _sql.AddLogin(Name);
}
[HttpGet]
[Route("export")]
public async Task<string> Export()
{
var mainFolder = _settings.TempPath.Combine("exported_users");
mainFolder.CreateDirectory();
foreach (var (owner, key) in await _sql.GetAllUserKeys())
{
var folder = mainFolder.Combine(owner);
folder.CreateDirectory();
await folder.Combine(Consts.AuthorAPIKeyFile).WriteAllTextAsync(key);
}
return "done";
}
}
}

49
Wabbajack.Server/Controllers/VirusScan.cs
View File

@ -1,49 +0,0 @@
using System.IO;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
[ApiController]
[Authorize(Roles = "User")]
public class VirusScan : ControllerBase
{
private ILogger<VirusScan> _logger;
private SqlService _sql;
public VirusScan(ILogger<VirusScan> logger, SqlService sql)
{
_logger = logger;
_sql = sql;
}
[HttpPost]
[Route("/virus_scan")]
public async Task<IActionResult> CheckFile()
{
var result = await VirusScanner.ScanStream(Request.Body);
await _sql.AddVirusResult(result.Item1, result.Item2);
return Ok(result.Item2.ToString());
}
[HttpGet]
[Route("/virus_scan/{hashAsHex}")]
public async Task<IActionResult> CheckHash(string hashAsHex)
{
var result = await _sql.FindVirusResult(Hash.FromHex(hashAsHex));
if (result == null)
{
return NotFound();
}
return Ok(result.ToString());
}
}
}

11
Wabbajack.Server/DTOs/AggregateMetric.cs
View File

@ -1,11 +0,0 @@
using System;
namespace Wabbajack.Server.DTOs
{
public class AggregateMetric
{
public DateTime Date { get; set; }
public string Subject { get; set; }
public int Count { get; set; }
}
}

31
Wabbajack.Server/DTOs/ArchiveDownload.cs
View File

@ -1,31 +0,0 @@
using System;
using System.Threading.Tasks;
using Wabbajack.Lib;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.DTOs
{
public class ArchiveDownload
{
public Guid Id { get; set; }
public Archive Archive { get; set; }
public bool? IsFailed { get; set; }
public DateTime? DownloadFinished { get; set; }
public string FailMessage { get; set; }
public async Task Fail(SqlService service, string message)
{
IsFailed = true;
DownloadFinished = DateTime.UtcNow;
FailMessage = message;
await service.UpdatePendingDownload(this);
}
public async Task Finish(SqlService service)
{
IsFailed = false;
DownloadFinished = DateTime.UtcNow;
await service.UpdatePendingDownload(this);
}
}
}

11
Wabbajack.Server/DTOs/ArchiveStatus.cs
View File

@ -1,11 +0,0 @@
namespace Wabbajack.Server.DTOs
{
public enum ArchiveStatus
{
Valid,
InValid,
Updating,
Updated,
Mirrored
}
}

15
Wabbajack.Server/DTOs/AuthoredFilesSummary.cs
View File

@ -1,15 +0,0 @@
using System;
namespace Wabbajack.Server.DTOs
{
public class AuthoredFilesSummary
{
public long Size { get; set; }
public string OriginalFileName { get; set; }
public string Author { get; set; }
public DateTime LastTouched { get; set; }
public DateTime? Finalized { get; set; }
public string MungedName { get; set; }
public string ServerAssignedUniqueId { get; set; }
}
}

37
Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs
View File

@ -1,37 +0,0 @@
using System.Collections.Generic;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
using Wabbajack.Common;
namespace Wabbajack.Server.DTOs
{
public enum StorageSpace
{
AuthoredFiles,
Patches,
Mirrors
}
public class BunnyCdnFtpInfo
{
public string Username { get; set; }
public string Password { get; set; }
public string Hostname { get; set; }
public static async Task<BunnyCdnFtpInfo> GetCreds(StorageSpace space)
{
return (await Utils.FromEncryptedJson<Dictionary<string, BunnyCdnFtpInfo>>("bunnycdn"))[space.ToString()];
}
public async Task<FtpClient> GetClient()
{
return await CircuitBreaker.WithAutoRetryAllAsync(async () =>
{
var ftpClient = new FtpClient(Hostname, new NetworkCredential(Username, Password));
await ftpClient.ConnectAsync();
return ftpClient;
});
}
}
}

34
Wabbajack.Server/DTOs/DetailedStatus.cs
View File

@ -1,34 +0,0 @@
using System;
using System.Collections.Generic;
using Newtonsoft.Json;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib;
using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.Server.DTOs
{
[JsonName("DetailedStatus")]
public class DetailedStatus
{
public string Name { get; set; }
public DateTime Checked { get; set; } = DateTime.UtcNow;
public List<DetailedStatusItem> Archives { get; set; }
public DownloadMetadata DownloadMetaData { get; set; }
public bool HasFailures { get; set; }
public string MachineName { get; set; }
}
[JsonName("DetailedStatusItem")]
public class DetailedStatusItem
{
public bool IsFailing { get; set; }
public Archive Archive { get; set; }
public string Name => string.IsNullOrWhiteSpace(Archive.Name) ? Archive.State.PrimaryKeyString : Archive.Name;
public string Url => Archive.State.GetManifestURL(Archive);
[JsonIgnore]
public bool HasUrl => Url != null;
public ArchiveStatus ArchiveStatus { get; set; }
}
}

106
Wabbajack.Server/DTOs/DiscordMessage.cs
View File

@ -1,106 +0,0 @@
using System;
using Newtonsoft.Json;
using Wabbajack.Common.Serialization.Json;
namespace Wabbajack.Server.DTOs
{
[JsonName("DiscordMessage")]
public class DiscordMessage
{
[JsonProperty("username")]
public string UserName { get; set; }
[JsonProperty("avatar_url")]
public Uri AvatarUrl { get; set; }
[JsonProperty("content")]
public string Content { get; set; }
[JsonProperty("embeds")]
public DiscordEmbed[] Embeds { get; set; }
}
[JsonName("DiscordEmbed")]
public class DiscordEmbed
{
[JsonProperty("title")]
public string Title { get; set; }
[JsonProperty("color")]
public int Color { get; set; }
[JsonProperty("author")]
public DiscordAuthor Author { get; set; }
[JsonProperty("url")]
public Uri Url { get; set; }
[JsonProperty("description")]
public string Description { get; set; }
[JsonProperty("fields")]
public DiscordField Field { get; set; }
[JsonProperty("thumbnail")]
public DiscordNumbnail Thumbnail { get; set; }
[JsonProperty("image")]
public DiscordImage Image { get; set; }
[JsonProperty("footer")]
public DiscordFooter Footer { get; set; }
[JsonProperty("timestamp")]
public DateTime Timestamp { get; set; } = DateTime.UtcNow;
}
[JsonName("DiscordAuthor")]
public class DiscordAuthor
{
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("url")]
public Uri Url { get; set; }
[JsonProperty("icon_url")]
public Uri IconUrl { get; set; }
}
[JsonName("DiscordField")]
public class DiscordField
{
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("value")]
public string Value { get; set; }
[JsonProperty("inline")]
public bool Inline { get; set; }
}
[JsonName("DiscordThumbnail")]
public class DiscordNumbnail
{
[JsonProperty("Url")]
public Uri Url { get; set; }
}
[JsonName("DiscordImage")]
public class DiscordImage
{
[JsonProperty("Url")]
public Uri Url { get; set; }
}
[JsonName("DiscordFooter")]
public class DiscordFooter
{
[JsonProperty("text")]
public string Text { get; set; }
[JsonProperty("icon_url")]
public Uri icon_url { get; set; }
}
}

14
Wabbajack.Server/DTOs/HeartbeatResult.cs
View File

@ -1,14 +0,0 @@
using System;
using Wabbajack.Common.Serialization.Json;
namespace Wabbajack.Server.DTOs
{
[JsonName("HeartbeatResult")]
public class HeartbeatResult
{
public TimeSpan Uptime { get; set; }
public TimeSpan LastNexusUpdate { get; set; }
public TimeSpan LastListValidation { get; set; }
}
}

12
Wabbajack.Server/DTOs/Metric.cs
View File

@ -1,12 +0,0 @@
using System;
namespace Wabbajack.Server.DTOs
{
public class Metric
{
public DateTime Timestamp { get; set; }
public string Action { get; set; }
public string Subject { get; set; }
public string MetricsKey { get; set; }
}
}

11
Wabbajack.Server/DTOs/MetricResult.cs
View File

@ -1,11 +0,0 @@
using System.Collections.Generic;
namespace Wabbajack.Server.DTOs
{
public class MetricResult
{
public string SeriesName { get; set; }
public List<string> Labels { get; set; }
public List<int> Values { get; set; }
}
}

30
Wabbajack.Server/DTOs/MirroredFile.cs
View File

@ -1,30 +0,0 @@
using System;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.DTOs
{
public class MirroredFile
{
public Hash Hash { get; set; }
public DateTime Created { get; set; }
public DateTime? Uploaded { get; set; }
public string Rationale { get; set; }
public string FailMessage { get; set; }
public async Task Finish(SqlService sql)
{
Uploaded = DateTime.UtcNow;
await sql.UpsertMirroredFile(this);
}
public async Task Fail(SqlService sql, string message)
{
Uploaded = DateTime.UtcNow;
FailMessage = message;
await sql.UpsertMirroredFile(this);
}
}
}

14
Wabbajack.Server/DTOs/NexusUpdateEntry.cs
View File

@ -1,14 +0,0 @@
using Newtonsoft.Json;
namespace Wabbajack.Server.DTOs
{
public class NexusUpdateEntry
{
[JsonProperty("mod_id")]
public long ModId { get; set; }
[JsonProperty("latest_file_update")]
public long LatestFileUpdate { get; set; }
[JsonProperty("latest_mod_activity")]
public long LastestModActivity { get; set; }
}
}

38
Wabbajack.Server/DTOs/Patch.cs
View File

@ -1,38 +0,0 @@
using System;
using System.Threading.Tasks;
using Microsoft.VisualBasic;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.DTOs
{
public class Patch
{
public ArchiveDownload Src { get; set; }
public ArchiveDownload Dest { get; set; }
public long PatchSize { get; set; }
public DateTime? Finished { get; set; }
public bool? IsFailed { get; set; }
public string FailMessage { get; set; }
public async Task Finish(SqlService sql, long size)
{
IsFailed = false;
Finished = DateTime.UtcNow;
PatchSize = size;
await sql.FinializePatch(this);
}
public async Task Fail(SqlService sql, string msg)
{
IsFailed = true;
Finished = DateTime.UtcNow;
FailMessage = msg;
await sql.FinializePatch(this);
}
}
}

21
Wabbajack.Server/DTOs/ValidationData.cs
View File

@ -1,21 +0,0 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.Server.DTOs
{
public class ValidationData
{
public Dictionary<(long Game, long ModId, long FileId), string> NexusFiles { get; set; } = new ();
public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
public List<ModlistMetadata> ModLists { get; set; }
public ConcurrentHashSet<(Game Game, long ModId)> SlowQueriedFor { get; set; } = new ConcurrentHashSet<(Game Game, long ModId)>();
public Dictionary<Hash, bool> Mirrors { get; set; }
public Lazy<Task<Dictionary<Hash, string>>> AllowedMirrors { get; set; }
public IEnumerable<AuthoredFilesSummary> AllAuthoredFiles { get; set; }
}
}

55
Wabbajack.Server/DataLayer/ApiKeys.cs
View File

@ -1,55 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Common;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task<string> LoginByApiKey(string key)
{
await using var conn = await Open();
var result = await conn.QueryAsync<string>(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @ApiKey",
new {ApiKey = key});
return result.FirstOrDefault();
}
public async Task<string> AddLogin(string name)
{
var key = NewAPIKey();
await using var conn = await Open();
await conn.ExecuteAsync("INSERT INTO dbo.ApiKeys (Owner, ApiKey) VALUES (@Owner, @ApiKey)",
new {Owner = name, ApiKey = key});
return key;
}
public static string NewAPIKey()
{
var arr = new byte[128];
new Random().NextBytes(arr);
return arr.ToHex();
}
public async Task<IEnumerable<(string Owner, string Key)>> GetAllUserKeys()
{
await using var conn = await Open();
var result = await conn.QueryAsync<(string Owner, string Key)>("SELECT Owner, ApiKey FROM dbo.ApiKeys");
return result;
}
public async Task<bool> IsTarKey(string metricsKey)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT MetricsKey FROM TarKey WHERE MetricsKey = @MetricsKey", new {MetricsKey = metricsKey});
return result == metricsKey;
}
}
}

271
Wabbajack.Server/DataLayer/ArchiveDownloads.cs
View File

@ -1,271 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.AuthorApi;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task<Guid> AddKnownDownload(Archive a, DateTime downloadFinished)
{
await using var conn = await Open();
var Id = Guid.NewGuid();
await conn.ExecuteAsync(
"INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader, DownloadFinished, IsFailed) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader, @DownloadFinished, @IsFailed)",
new
{
Id = Id,
PrimaryKeyString = a.State.PrimaryKeyString,
Size = a.Size == 0 ? null : (long?)a.Size,
Hash = a.Hash == default ? null : (Hash?)a.Hash,
DownloadState = a.State,
Downloader = AbstractDownloadState.TypeToName[a.State.GetType()],
DownloadFinished = downloadFinished,
IsFailed = false
});
return Id;
}
public async Task<Guid> EnqueueDownload(Archive a)
{
await using var conn = await Open();
var Id = Guid.NewGuid();
await conn.ExecuteAsync(
"INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)",
new
{
Id = Id,
PrimaryKeyString = a.State.PrimaryKeyString,
Size = a.Size == 0 ? null : (long?)a.Size,
Hash = a.Hash == default ? null : (Hash?)a.Hash,
DownloadState = a.State,
Downloader = AbstractDownloadState.TypeToName[a.State.GetType()],
});
return Id;
}
public async Task<HashSet<(Hash Hash, string PrimaryKeyString)>> GetAllArchiveDownloads()
{
await using var conn = await Open();
return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, PrimaryKeyString FROM ArchiveDownloads")).ToHashSet();
}
public async Task<HashSet<(Hash Hash, AbstractDownloadState State)>> GetAllArchiveDownloadStates()
{
await using var conn = await Open();
return (await conn.QueryAsync<(Hash, AbstractDownloadState)>("SELECT Hash, DownloadState FROM ArchiveDownloads")).ToHashSet();
}
public async Task<ArchiveDownload> GetArchiveDownload(Guid id)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE Id = @id",
new {Id = id});
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
public async Task<ArchiveDownload> GetArchiveDownload(string primaryKeyString)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND IsFailed = 0",
new {PrimaryKeyString = primaryKeyString});
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
public async Task<ArchiveDownload> GetArchiveDownload(string primaryKeyString, Hash hash, long size)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size",
new
{
PrimaryKeyString = primaryKeyString,
Hash = hash,
Size = size
});
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
public async Task<ArchiveDownload> GetOrEnqueueArchive(Archive a)
{
await using var conn = await Open();
await using var trans = await conn.BeginTransactionAsync();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size",
new
{
PrimaryKeyString = a.State.PrimaryKeyString,
Hash = a.Hash,
Size = a.Size
}, trans);
if (result.Item1 != default)
{
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
var id = Guid.NewGuid();
await conn.ExecuteAsync(
"INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)",
new
{
Id = id,
PrimaryKeyString = a.State.PrimaryKeyString,
Size = a.Size == 0 ? null : (long?)a.Size,
Hash = a.Hash == default ? null : (Hash?)a.Hash,
DownloadState = a.State,
Downloader = AbstractDownloadState.TypeToName[a.State.GetType()]
}, trans);
await trans.CommitAsync();
return new ArchiveDownload {Id = id, Archive = a,};
}
public async Task<ArchiveDownload> GetNextPendingDownload(bool ignoreNexus = false)
{
await using var conn = await Open();
(Guid, long?, Hash?, AbstractDownloadState) result;
if (ignoreNexus)
{
result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, AbstractDownloadState)>(
"SELECT TOP(1) Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL AND Downloader != 'NexusDownloader+State'");
}
else
{
result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, AbstractDownloadState)>(
"SELECT TOP(1) Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL");
}
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
Archive = new Archive(result.Item4) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default,},
};
}
public async Task UpdatePendingDownload(ArchiveDownload ad)
{
await using var conn = await Open();
await conn.ExecuteAsync(
"UPDATE dbo.ArchiveDownloads SET IsFailed = @IsFailed, DownloadFinished = @DownloadFinished, Hash = @Hash, Size = @Size, FailMessage = @FailMessage WHERE Id = @Id",
new
{
Id = ad.Id,
IsFailed = ad.IsFailed,
DownloadFinished = ad.DownloadFinished,
Size = ad.Archive.Size,
Hash = ad.Archive.Hash,
FailMessage = ad.FailMessage
});
}
public async Task<int> EnqueueModListFilesForIndexing()
{
await using var conn = await Open();
return await conn.ExecuteAsync(@"
INSERT INTO dbo.ArchiveDownloads (Id, PrimaryKeyString, Hash, DownloadState, Size, Downloader)
SELECT DISTINCT NEWID(), mla.PrimaryKeyString, mla.Hash, mla.State, mla.Size, SUBSTRING(mla.PrimaryKeyString, 0, CHARINDEX('|', mla.PrimaryKeyString))
FROM [dbo].[ModListArchives] mla
LEFT JOIN dbo.ArchiveDownloads ad on mla.PrimaryKeyString = ad.PrimaryKeyString AND mla.Hash = ad.Hash
WHERE ad.PrimaryKeyString is null");
}
public async Task<List<Archive>> GetGameFiles(Game game, string version)
{
await using var conn = await Open();
var files = (await conn.QueryAsync<(Hash, long, AbstractDownloadState)>(
$"SELECT Hash, Size, DownloadState FROM dbo.ArchiveDownloads WHERE PrimaryKeyString like 'GameFileSourceDownloader+State|{game}|{version}|%'"))
.Select(f => new Archive(f.Item3)
{
Hash = f.Item1,
Size = f.Item2
}).ToList();
return files;
}
public async Task<Archive[]> ResolveDownloadStatesByHash(Hash hash)
{
await using var conn = await Open();
var files = (await conn.QueryAsync<(long, Hash, AbstractDownloadState)>(
@"SELECT Size, Hash, DownloadState from dbo.ArchiveDownloads WHERE Hash = @Hash AND IsFailed = 0 AND DownloadFinished IS NOT NULL ORDER BY DownloadFinished DESC",
new {Hash = hash})
).Select(e =>
new Archive(e.Item3) {Size = e.Item1, Hash = e.Item2}
).ToList();
if (await HaveMirror(hash) && files.Count > 0)
{
var ffile = files.First();
var host = Consts.TestMode ? "test-files" : "mirror";
var url = new Uri($"https://{host}.wabbajack.org/{hash.ToHex()}");
files.Add(new Archive(
new WabbajackCDNDownloader.State(url)) {Hash = hash, Size = ffile.Size, Name = ffile.Name});
}
return files.ToArray();
}
public async Task<IEnumerable<(Game, string)>> GetAllRegisteredGames()
{
await using var conn = await Open();
var pks = (await conn.QueryAsync<string>(
@"SELECT PrimaryKeyString FROM dbo.ArchiveDownloads WHERE PrimaryKeyString like 'GameFileSourceDownloader+State|%'")
);
return pks.Select(p => p.Split("|"))
.Select(t => (GameRegistry.GetByFuzzyName(t[1]).Game, t[2]))
.Distinct();
}
}
}

83
Wabbajack.Server/DataLayer/AuthoredFiles.cs
View File

@ -1,83 +0,0 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Microsoft.AspNetCore.Mvc.Infrastructure;
using Wabbajack.BuildServer.Controllers;
using Wabbajack.Common;
using Wabbajack.Lib.AuthorApi;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task TouchAuthoredFile(CDNFileDefinition definition, DateTime? date = null)
{
await using var conn = await Open();
if (date == null)
{
await conn.ExecuteAsync(
"UPDATE AuthoredFiles SET LastTouched = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid",
new {Uid = definition.ServerAssignedUniqueId});
}
else
{
await conn.ExecuteAsync(
"UPDATE AuthoredFiles SET LastTouched = @Date WHERE ServerAssignedUniqueId = @Uid",
new {Uid = definition.ServerAssignedUniqueId, Date = date});
}
}
public async Task<CDNFileDefinition> CreateAuthoredFile(CDNFileDefinition definition, string login)
{
definition.Author = login;
var uid = Guid.NewGuid().ToString();
await using var conn = await Open();
definition.ServerAssignedUniqueId = uid;
await conn.ExecuteAsync("INSERT INTO dbo.AuthoredFiles (ServerAssignedUniqueId, LastTouched, CDNFileDefinition) VALUES (@Uid, GETUTCDATE(), @CdnFile)",
new {
Uid = uid,
CdnFile = definition
});
return definition;
}
public async Task Finalize(CDNFileDefinition definition)
{
await using var conn = await Open();
await conn.ExecuteAsync("UPDATE AuthoredFiles SET LastTouched = GETUTCDATE(), Finalized = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid",
new {
Uid = definition.ServerAssignedUniqueId
});
}
public async Task<CDNFileDefinition> GetCDNFileDefinition(string serverAssignedUniqueId)
{
await using var conn = await Open();
return (await conn.QueryAsync<CDNFileDefinition>(
"SELECT CDNFileDefinition FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid",
new {Uid = serverAssignedUniqueId})).First();
}
public async Task DeleteFileDefinition(CDNFileDefinition definition)
{
await using var conn = await Open();
await conn.ExecuteAsync(
"DELETE FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid",
new {Uid = definition.ServerAssignedUniqueId});
return;
}
public async Task<IEnumerable<AuthoredFilesSummary>> AllAuthoredFiles()
{
await using var conn = await Open();
var results = await conn.QueryAsync<AuthoredFilesSummary>("SELECT CONVERT(NVARCHAR(50), ServerAssignedUniqueId) as ServerAssignedUniqueId, Size, OriginalFileName, Author, LastTouched, Finalized, MungedName from dbo.AuthoredFilesSummaries ORDER BY LastTouched DESC");
return results;
}
}
}

107
Wabbajack.Server/DataLayer/Mappers.cs
View File

@ -1,107 +0,0 @@
using System;
using System.Data;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.AuthorApi;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
static SqlService()
{
SqlMapper.AddTypeHandler(new HashMapper());
SqlMapper.AddTypeHandler(new RelativePathMapper());
SqlMapper.AddTypeHandler(new JsonMapper<AbstractDownloadState>());
SqlMapper.AddTypeHandler(new JsonMapper<CDNFileDefinition>());
SqlMapper.AddTypeHandler(new JsonMapper<ModlistMetadata>());
SqlMapper.AddTypeHandler(new VersionMapper());
SqlMapper.AddTypeHandler(new GameMapper());
SqlMapper.AddTypeHandler(new DateTimeHandler());
}
/// <summary>
/// Needed to make sure dates are all in UTC format
/// </summary>
private class DateTimeHandler : SqlMapper.TypeHandler<DateTime>
{
public override void SetValue(IDbDataParameter parameter, DateTime value)
{
parameter.Value = value;
}
public override DateTime Parse(object value)
{
return DateTime.SpecifyKind((DateTime)value, DateTimeKind.Utc);
}
}
class JsonMapper<T> : SqlMapper.TypeHandler<T>
{
public override void SetValue(IDbDataParameter parameter, T value)
{
parameter.Value = value.ToJson();
}
public override T Parse(object value)
{
return ((string)value).FromJsonString<T>();
}
}
class RelativePathMapper : SqlMapper.TypeHandler<RelativePath>
{
public override void SetValue(IDbDataParameter parameter, RelativePath value)
{
parameter.Value = value.ToJson();
}
public override RelativePath Parse(object value)
{
return (RelativePath)(string)value;
}
}
class HashMapper : SqlMapper.TypeHandler<Hash>
{
public override void SetValue(IDbDataParameter parameter, Hash value)
{
parameter.Value = (long)value;
}
public override Hash Parse(object value)
{
return Hash.FromLong((long)value);
}
}
class VersionMapper : SqlMapper.TypeHandler<Version>
{
public override void SetValue(IDbDataParameter parameter, Version value)
{
parameter.Value = value.ToString();
}
public override Version Parse(object value)
{
return Version.Parse((string)value);
}
}
class GameMapper : SqlMapper.TypeHandler<Game>
{
public override void SetValue(IDbDataParameter parameter, Game value)
{
parameter.Value = value.ToString();
}
public override Game Parse(object value)
{
return GameRegistry.GetByFuzzyName((string)value).Game;
}
}
}
}

186
Wabbajack.Server/DataLayer/Metrics.cs
View File

@ -1,186 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task IngestMetric(Metric metric)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", metric);
}
public async Task IngestAccess(string ip, string log)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"INSERT INTO dbo.AccessLog (Timestamp, Action, Ip) VALUES (@Timestamp, @Action, @Ip)", new
{
Timestamp = DateTime.UtcNow,
Ip = ip,
Action = log
});
}
public async Task<IEnumerable<AggregateMetric>> MetricsReport(string action)
{
await using var conn = await Open();
return (await conn.QueryAsync<AggregateMetric>(@"
select
datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)) as Date,
GroupingSubject as Subject,
count(*) as Count
from dbo.metrics where
Action = @Action
AND GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics
WHERE action = @Action
AND MetricsKey is not null
AND Subject != 'Default'
AND Subject != 'untitled'
AND TRY_CONVERT(uniqueidentifier, Subject) is null
AND Timestamp >= DATEADD(DAY, -1, GETUTCDATE()))
group by
datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)),
GroupingSubject
Order by datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)) asc",
new {Action = action}))
.ToList();
}
public async Task<List<(DateTime, string, string)>> FullTarReport(string key)
{
await using var conn = await Open();
return (await conn.QueryAsync<(DateTime, string, string)>(@"
SELECT u.Timestamp, u.Path, u.MetricsKey FROM
(SELECT al.Timestamp, JSON_VALUE(al.Action, '$.Path') as Path, al.MetricsKey FROM dbo.AccessLog al
WHERE al.MetricsKey = @MetricsKey
UNION ALL
SELECT m.Timestamp, m.Action + ' ' + m.Subject as Path, m.MetricsKey FROM dbo.Metrics m
WHERE m.MetricsKey = @MetricsKey
AND m.Action != 'TarKey') u
ORDER BY u.Timestamp Desc",
new {MetricsKey = key})).ToList();
}
public async Task<bool> ValidMetricsKey(string metricsKey)
{
await using var conn = await Open();
return (await conn.QuerySingleOrDefaultAsync<string>("SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey",
new {MetricsKey = metricsKey})) != default;
}
public async Task AddMetricsKey(string metricsKey)
{
await using var conn = await Open();
await using var trans = conn.BeginTransaction();
if ((await conn.QuerySingleOrDefaultAsync<string>(
"SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey",
new {MetricsKey = metricsKey}, trans)) != default)
return;
await conn.ExecuteAsync("INSERT INTO dbo.MetricsKeys (MetricsKey) VALUES (@MetricsKey)",
new {MetricsKey = metricsKey}, trans);
}
public async Task<string[]> AllKeys()
{
await using var conn = await Open();
return (await conn.QueryAsync<string>("SELECT MetricsKey from dbo.MetricsKeys")).ToArray();
}
public async Task<long> UniqueInstalls(string machineUrl)
{
await using var conn = await Open();
return await conn.QueryFirstAsync<long>(
@"SELECT COUNT(*) FROM (
SELECT DISTINCT MetricsKey from dbo.Metrics where Action = 'finish_install' and GroupingSubject in (
SELECT JSON_VALUE(Metadata, '$.title') FROM dbo.ModLists
WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)) s",
new {MachineURL = machineUrl});
}
public async Task<long> TotalInstalls(string machineUrl)
{
await using var conn = await Open();
return await conn.QueryFirstAsync<long>(
@"SELECT COUNT(*) from dbo.Metrics where Action = 'finish_install' and GroupingSubject in (
SELECT JSON_VALUE(Metadata, '$.title') FROM dbo.ModLists
WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)",
new {MachineURL = machineUrl});
}
public async Task<IEnumerable<(string, long)>> GetTotalInstalls()
{
await using var conn = await Open();
return await conn.QueryAsync<(string, long)>(
@"SELECT GroupingSubject, Count(*) as Count
From dbo.Metrics
WHERE
GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics
WHERE action = 'finish_install'
AND MetricsKey is not null)
group by GroupingSubject
order by Count(*) desc");
}
public async Task<IEnumerable<(string, long)>> GetTotalUniqueInstalls()
{
await using var conn = await Open();
return await conn.QueryAsync<(string, long)>(
@"Select GroupingSubject, Count(*) as Count
FROM
(select DISTINCT MetricsKey, GroupingSubject
From dbo.Metrics
WHERE
GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics
WHERE action = 'finish_install'
AND MetricsKey is not null)) m
GROUP BY GroupingSubject
Order by Count(*) desc
");
}
public async IAsyncEnumerable<MetricRow> MetricsDump()
{
var keys = new Dictionary<string, long>();
await using var conn = await Open();
foreach (var row in await conn.QueryAsync<(long, DateTime, string, string, string, string)>(@"select Id, Timestamp, Action, Subject, MetricsKey, GroupingSubject from dbo.metrics WHERE MetricsKey is not null"))
{
if (!keys.TryGetValue(row.Item5, out var keyid))
{
keyid = keys.Count;
keys[row.Item5] = keyid;
}
yield return new MetricRow
{
Id = row.Item1,
Timestamp = row.Item2,
Action = row.Item3,
Subject = row.Item4,
MetricsKey = keyid,
GroupingSubject = row.Item6
};
}
}
public class MetricRow
{
public long Id;
public DateTime Timestamp;
public string Action;
public string Subject;
public string GroupingSubject;
public long MetricsKey;
}
}
}

178
Wabbajack.Server/DataLayer/MirroredFiles.cs
View File

@ -1,178 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using CefSharp.DevTools.Network;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task<MirroredFile> GetNextMirroredFile()
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Hash, DateTime, DateTime, string, string)>(
"SELECT Hash, Created, Uploaded, Rationale, FailMessage from dbo.MirroredArchives WHERE Uploaded IS NULL");
if (result == default) return null;
return new MirroredFile
{
Hash = result.Item1, Created = result.Item2, Uploaded = result.Item3, Rationale = result.Item4, FailMessage = result.Item5
};
}
public async Task<Dictionary<Hash, bool>> GetAllMirroredHashes()
{
await using var conn = await Open();
return (await conn.QueryAsync<(Hash, DateTime?)>("SELECT Hash, Uploaded FROM dbo.MirroredArchives"))
.GroupBy(d => d.Item1)
.ToDictionary(d => d.Key, d => d.First().Item2.HasValue);
}
public async Task StartMirror((Hash Hash, string Reason) mirror)
{
await using var conn = await Open();
await using var trans = await conn.BeginTransactionAsync();
if (await conn.QueryFirstOrDefaultAsync<Hash>(@"SELECT Hash FROM dbo.MirroredArchives WHERE Hash = @Hash",
new {Hash = mirror.Hash}, trans) != default)
{
return;
}
await conn.ExecuteAsync(
@"INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) VALUES (@Hash, GETUTCDATE(), @Reason)",
new {Hash = mirror.Hash, Reason = mirror.Reason}, trans);
await trans.CommitAsync();
}
public async Task<Dictionary<Hash, string>> GetAllowedMirrors()
{
await using var conn = await Open();
return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, Reason FROM dbo.AllowedMirrorsCache"))
.GroupBy(d => d.Item1)
.ToDictionary(d => d.Key, d => d.First().Item2);
}
public async Task UpsertMirroredFile(MirroredFile file)
{
await using var conn = await Open();
await using var trans = await conn.BeginTransactionAsync();
await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash", new {file.Hash}, trans);
await conn.ExecuteAsync(
"INSERT INTO dbo.MirroredArchives (Hash, Created, Uploaded, Rationale, FailMessage) VALUES (@Hash, @Created, @Uploaded, @Rationale, @FailMessage)",
new
{
Hash = file.Hash,
Created = file.Created,
Uploaded = file.Uploaded,
Rationale = file.Rationale,
FailMessage = file.FailMessage
}, trans);
await trans.CommitAsync();
}
public async Task DeleteMirroredFile(Hash hash)
{
await using var conn = await Open();
await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash",
new {Hash = hash});
}
public async Task InsertAllNexusMirrors()
{
var permissions = (await GetNexusPermissions()).Where(p => p.Value == HTMLInterface.PermissionValue.Yes);
var downloads = (await GetAllArchiveDownloadStates()).Where(a => a.State is NexusDownloader.State).ToDictionary(a =>
{
var nd = (NexusDownloader.State)a.State;
return (nd.Game, nd.ModID);
}, a => a.Hash);
var existing = await GetAllMirroredHashes();
foreach (var (key, _) in permissions)
{
if (!downloads.TryGetValue(key, out var hash)) continue;
if (existing.ContainsKey(hash)) continue;
await UpsertMirroredFile(new MirroredFile
{
Hash = hash,
Created = DateTime.UtcNow,
Rationale =
$"Mod ({key.Item1} {key.Item2}) has allowed re-upload permissions on the Nexus"
});
}
}
public async Task<bool> HaveMirror(Hash hash)
{
await using var conn = await Open();
return await conn.QueryFirstOrDefaultAsync<Hash>("SELECT Hash FROM dbo.MirroredArchives WHERE Hash = @Hash",
new {Hash = hash}) != default;
}
public async Task QueueMirroredFiles()
{
await using var conn = await Open();
await conn.ExecuteAsync(@"
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT hs.Hash, GETUTCDATE(), 'File has re-upload permissions on the Nexus' FROM
(SELECT DISTINCT ad.Hash FROM dbo.NexusModPermissions p
INNER JOIN GameMetadata md on md.NexusGameId = p.NexusGameID
INNER JOIN dbo.ArchiveDownloads ad on ad.PrimaryKeyString like 'NexusDownloader+State|'+md.WabbajackName+'|'+CAST(p.ModID as nvarchar)+'|%'
WHERE p.Permissions = 1
AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
) hs
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT DISTINCT Hash, GETUTCDATE(), 'File is hosted on GitHub'
FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%github.com/%'
AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT DISTINCT Hash, GETUTCDATE(), 'File license allows uploading to any Non-nexus site'
FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%enbdev.com/%'
AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT DISTINCT Hash, GETUTCDATE(), 'DynDOLOD file' /*, Name*/
from dbo.ModListArchives mla WHERE Name like '%DynDoLOD%standalone%'
and Hash not in (select Hash from dbo.MirroredArchives)
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT DISTINCT Hash, GETUTCDATE(), 'Distribution allowed by author' /*, Name*/
from dbo.ModListArchives mla WHERE Name like '%particle%patch%'
and Hash not in (select Hash from dbo.MirroredArchives)
");
}
public async Task AddNexusModWithOpenPerms(Game gameGame, long modId)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"INSERT INTO dbo.NexusModsWithOpenPerms(NexusGameID, NexusModID) VALUES(@game, @mod)",
new {game = gameGame.MetaData().NexusGameId, modId});
}
public async Task SyncActiveMirroredFiles()
{
await using var conn = await Open();
await conn.ExecuteAsync(@"EXEC dbo.QueueMirroredFiles");
}
}
}

114
Wabbajack.Server/DataLayer/ModLists.cs
View File

@ -1,114 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Lib;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist, bool brokenDownload)
{
await using var conn = await Open();
await using var tran = await conn.BeginTransactionAsync();
await conn.ExecuteAsync(@"DELETE FROM dbo.ModLists Where MachineUrl = @MachineUrl",
new {MachineUrl = metadata.Links.MachineURL}, tran);
var archives = modlist.Archives;
var directives = modlist.Directives;
modlist.Archives = new List<Archive>();
modlist.Directives = new List<Directive>();
await conn.ExecuteAsync(
@"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList, BrokenDownload) VALUES (@MachineUrl, @Hash, @Metadata, @ModList, @BrokenDownload)",
new
{
MachineUrl = metadata.Links.MachineURL,
Hash = hash,
MetaData = metadata.ToJson(),
ModList = modlist.ToJson(),
BrokenDownload = brokenDownload
}, tran);
var entries = archives.Select(a =>
new
{
MachineUrl = metadata.Links.MachineURL,
Name = a.Name,
Hash = a.Hash,
Size = a.Size,
State = a.State.ToJson(),
PrimaryKeyString = a.State.PrimaryKeyString
}).ToArray();
await conn.ExecuteAsync(@"DELETE FROM dbo.ModListArchives WHERE MachineURL = @machineURL",
new {MachineUrl = metadata.Links.MachineURL}, tran);
foreach (var entry in entries)
{
await conn.ExecuteAsync(
"INSERT INTO dbo.ModListArchives (MachineURL, Name, Hash, Size, PrimaryKeyString, State) VALUES (@MachineURL, @Name, @Hash, @Size, @PrimaryKeyString, @State)",
entry, tran);
}
await tran.CommitAsync();
}
public async Task<bool> HaveIndexedModlist(string machineUrl, Hash hash)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT MachineURL from dbo.Modlists WHERE MachineURL = @MachineUrl AND Hash = @Hash",
new {MachineUrl = machineUrl, Hash = hash});
return result != null;
}
public async Task<bool> HashIsInAModlist(Hash hash)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<bool>("SELECT Hash FROM dbo.ModListArchives Where Hash = @Hash",
new {Hash = hash});
return result;
}
public async Task<List<Archive>> ModListArchives(string machineURL)
{
await using var conn = await Open();
var archives = await conn.QueryAsync<(string, Hash, long, AbstractDownloadState)>("SELECT Name, Hash, Size, State FROM dbo.ModListArchives WHERE MachineUrl = @MachineUrl",
new {MachineUrl = machineURL});
return archives.Select(t => new Archive(t.Item4)
{
Name = string.IsNullOrWhiteSpace(t.Item1) ? t.Item4.PrimaryKeyString : t.Item1,
Size = t.Item3,
Hash = t.Item2
}).ToList();
}
public async Task<List<Archive>> ModListArchives()
{
await using var conn = await Open();
var archives = await conn.QueryAsync<(string, Hash, long, AbstractDownloadState)>("SELECT Name, Hash, Size, State FROM dbo.ModListArchives");
return archives.Select(t => new Archive(t.Item4)
{
Name = string.IsNullOrWhiteSpace(t.Item1) ? t.Item4.PrimaryKeyString : t.Item1,
Size = t.Item3,
Hash = t.Item2
}).ToList();
}
public async Task<int> PurgeList(string machineURL)
{
await using var conn = await Open();
var ret1 = await conn.ExecuteAsync(@" delete from dbo.ModListArchives where MachineURL = @machineURL",
new {machineURL});
var ret2 = await conn.ExecuteAsync(@" delete from dbo.ModLists where MachineURL = @machineURL",
new {machineURL});
return ret1 + ret2;
}
}
}

227
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -1,227 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Dapper;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
namespace Wabbajack.Server.DataLayer
{
/// <summary>
/// SQL routines that read/write cached information from the Nexus
/// </summary>
public partial class SqlService
{
public async Task<long> DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date)
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
return deleted;
}
public async Task<long> DeleteNexusModFilesUpdatedBeforeDate(Game game, long modId, DateTime date)
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
return deleted;
}
public async Task<ModInfo> GetNexusModInfoString(Game game, long modId)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT Data FROM dbo.NexusModInfos WHERE Game = @Game AND @ModId = ModId",
new {Game = game.MetaData().NexusGameId, ModId = modId});
return result == null ? null : JsonConvert.DeserializeObject<ModInfo>(result);
}
public async Task AddNexusModInfo(Game game, long modId, DateTime lastCheckedUtc, ModInfo data)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"MERGE dbo.NexusModInfos AS Target
USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
ON Target.Game = Source.Game AND Target.ModId = Source.ModId
WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
new
{
Game = game.MetaData().NexusGameId,
ModId = modId,
LastChecked = lastCheckedUtc,
Data = JsonConvert.SerializeObject(data)
});
}
public async Task AddNexusModFiles(Game game, long modId, DateTime lastCheckedUtc, NexusApiClient.GetModFilesResponse data)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"MERGE dbo.NexusModFiles AS Target
USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
ON Target.Game = Source.Game AND Target.ModId = Source.ModId
WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
new
{
Game = game.MetaData().NexusGameId,
ModId = modId,
LastChecked = lastCheckedUtc,
Data = JsonConvert.SerializeObject(data)
});
}
public async Task AddNexusModFileSlow(Game game, long modId, long fileId, DateTime lastCheckedUtc)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"MERGE dbo.NexusModFilesSlow AS Target
USING (SELECT @GameId GameId, @ModId ModId, @LastChecked LastChecked, @FileId FileId) AS Source
ON Target.GameId = Source.GameId AND Target.ModId = Source.ModId AND Target.FileId = Source.FileId
WHEN MATCHED THEN UPDATE SET Target.LastChecked = @LastChecked
WHEN NOT MATCHED THEN INSERT (GameId, ModId, LastChecked, FileId) VALUES (@GameId, @ModId, @LastChecked, @FileId);",
new
{
GameId = game.MetaData().NexusGameId,
ModId = modId,
FileId = fileId,
LastChecked = lastCheckedUtc,
});
}
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(Game game, long modId)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT Data FROM dbo.NexusModFiles WHERE Game = @Game AND @ModId = ModId",
new {Game = game.MetaData().NexusGameId, ModId = modId});
return result == null ? null : JsonConvert.DeserializeObject<NexusApiClient.GetModFilesResponse>(result);
}
public async Task PurgeNexusCache(long modId)
{
await using var conn = await Open();
await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModPermissions WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModFile WHERE ModId = @ModID", new {ModId = modId});
}
public async Task<Dictionary<(Game, long), HTMLInterface.PermissionValue>> GetNexusPermissions()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<(int, long, int)>("SELECT NexusGameID, ModID, Permissions FROM NexusModPermissions");
return results.ToDictionary(f => (GameRegistry.ByNexusID[f.Item1], f.Item2),
f => (HTMLInterface.PermissionValue)f.Item3);
}
public async Task<Dictionary<(Game, long), HTMLInterface.PermissionValue>> GetHiddenNexusMods()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<(int, long, int)>(@"SELECT NexusGameID, ModID, Permissions FROM NexusModPermissions WHERE Permissions = @Permissions
UNION
SELECT Game, mf.ModID, 3 from dbo.NexusModFiles mf
LEFT JOIN NexusModPermissions mp on mf.Game = mp.NexusGameID AND mf.ModId = mp.ModID
WHERE JSON_QUERY(Data, '$.files') = '[]' AND mp.Permissions != 4",
new {Permissions = (int)HTMLInterface.PermissionValue.Hidden});
return results.ToDictionary(f => (GameRegistry.ByNexusID[f.Item1], f.Item2),
f => (HTMLInterface.PermissionValue)f.Item3);
}
public async Task SetNexusPermissions(IEnumerable<(Game, long, HTMLInterface.PermissionValue)> permissions)
{
await using var conn = await Open();
var tx = await conn.BeginTransactionAsync();
await conn.ExecuteAsync("DELETE FROM NexusModPermissions", transaction:tx);
foreach (var (game, modId, perm) in permissions)
{
await conn.ExecuteAsync(
"INSERT INTO NexusModPermissions (NexusGameID, ModID, Permissions) VALUES (@NexusGameID, @ModID, @Permissions)",
new {NexusGameID = game.MetaData().NexusGameId, ModID = modId, Permissions = (int)perm}, tx);
}
await tx.CommitAsync();
}
public async Task UpdateGameMetadata()
{
await using var conn = await Open();
var existing = (await conn.QueryAsync<string>("SELECT WabbajackName FROM dbo.GameMetadata")).ToHashSet();
var missing = GameRegistry.Games.Values.Where(g => !existing.Contains(g.Game.ToString())).ToList();
foreach (var add in missing.Where(g => g.NexusGameId != 0))
{
await conn.ExecuteAsync(
"INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, @WabbajackName)",
new {NexusGameId = add.NexusGameId, WabbajackName = add.Game.ToString()});
}
}
public async Task SetNexusPermission(Game game, long modId, HTMLInterface.PermissionValue perm)
{
await using var conn = await Open();
var tx = await conn.BeginTransactionAsync();
await conn.ExecuteAsync("DELETE FROM NexusModPermissions WHERE NexusGameID = @GameID AND ModID = @ModID", new
{
GameID = game.MetaData().NexusGameId,
ModID = modId
},
transaction:tx);
await conn.ExecuteAsync(
"INSERT INTO NexusModPermissions (NexusGameID, ModID, Permissions) VALUES (@NexusGameID, @ModID, @Permissions)",
new {NexusGameID = game.MetaData().NexusGameId, ModID = modId, Permissions = (int)perm}, tx);
await tx.CommitAsync();
}
public async Task<NexusFileInfo> GetModFile(Game game, long modId, long fileId)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT Data FROM dbo.NexusModFile WHERE Game = @Game AND @ModId = ModId AND @FileId = FileId",
new {Game = game.MetaData().NexusGameId, ModId = modId, FileId = fileId});
return result == null ? null : JsonConvert.DeserializeObject<NexusFileInfo>(result);
}
public async Task AddNexusModFile(Game game, long modId, long fileId, DateTime lastCheckedUtc, NexusFileInfo data)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"INSERT INTO dbo.NexusModFile (Game, ModId, FileId, LastChecked, Data)
VALUES (@Game, @ModId, @FileId, @LastChecked, @Data)",
new
{
Game = game.MetaData().NexusGameId,
ModId = modId,
FileId = fileId,
LastChecked = lastCheckedUtc,
Data = JsonConvert.SerializeObject(data)
});
}
}
}

51
Wabbajack.Server/DataLayer/NexusKeys.cs
View File

@ -1,51 +0,0 @@
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task SetNexusAPIKey(string key, long daily, long hourly)
{
await using var conn = await Open();
await using var trans = await conn.BeginTransactionAsync();
await conn.ExecuteAsync(@"DELETE FROM NexusKeys WHERE ApiKey = @ApiKey", new {ApiKey = key}, trans);
await conn.ExecuteAsync(@"INSERT INTO NexusKeys (ApiKey, DailyRemain, HourlyRemain) VALUES (@ApiKey, @DailyRemain, @HourlyRemain)",
new {ApiKey = key, DailyRemain = daily, HourlyRemain = hourly}, trans);
await trans.CommitAsync();
}
public async Task DeleteNexusAPIKey(string key)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"DELETE FROM NexusKeys WHERE ApiKey = @ApiKey", new {ApiKey = key});
}
public async Task<List<string>> GetNexusApiKeys(int threshold = 1500)
{
await using var conn = await Open();
return (await conn.QueryAsync<string>(@"SELECT ApiKey FROM NexusKeys WHERE DailyRemain >= @Threshold ORDER BY DailyRemain DESC",
new {Threshold = threshold})).ToList();
}
public async Task<List<(string Key, int Daily, int Hourly)>> GetNexusApiKeysWithCounts(int threshold = 1500)
{
await using var conn = await Open();
return (await conn.QueryAsync<(string, int, int)>(@"SELECT ApiKey, DailyRemain, HourlyRemain FROM NexusKeys WHERE DailyRemain >= @Threshold ORDER BY DailyRemain DESC",
new {Threshold = threshold})).ToList();
}
public async Task<bool> HaveKey(string key)
{
await using var conn = await Open();
return (await conn.QueryAsync<string>(@"SELECT ApiKey FROM NexusKeys WHERE ApiKey = @ApiKey",
new {ApiKey = key})).Any();
}
}
}

47
Wabbajack.Server/DataLayer/NonNexusModlistArchives.cs
View File

@ -1,47 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task<List<Archive>> GetNonNexusModlistArchives()
{
await using var conn = await Open();
var results = await conn.QueryAsync<(Hash Hash, long Size, string State)>(
@"SELECT Hash, Size, State FROM dbo.ModListArchives WHERE PrimaryKeyString NOT LIKE 'NexusDownloader+State|%'");
return results.Select(r => new Archive (r.State.FromJsonString<AbstractDownloadState>())
{
Size = r.Size,
Hash = r.Hash,
}).ToList();}
public async Task UpdateNonNexusModlistArchivesStatus(IEnumerable<(Archive Archive, bool IsValid)> results)
{
await using var conn = await Open();
var trans = await conn.BeginTransactionAsync();
await conn.ExecuteAsync("DELETE FROM dbo.ModlistArchiveStatus;", transaction:trans);
foreach (var itm in results.DistinctBy(itm => (itm.Archive.Hash, itm.Archive.State.PrimaryKeyString)))
{
await conn.ExecuteAsync(
@"INSERT INTO dbo.ModlistArchiveStatus (PrimaryKeyStringHash, PrimaryKeyString, Hash, IsValid)
VALUES (HASHBYTES('SHA2_256', @PrimaryKeyString), @PrimaryKeyString, @Hash, @IsValid)", new
{
PrimaryKeyString = itm.Archive.State.PrimaryKeyString,
Hash = itm.Archive.Hash,
IsValid = itm.IsValid
}, trans);
}
await trans.CommitAsync();
}
}
}

265
Wabbajack.Server/DataLayer/Patches.cs
View File

@ -1,265 +0,0 @@
using System;
using System.Collections.Generic;
using System.Data.SqlClient;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
/// <summary>
/// Adds a patch record
/// </summary>
/// <param name="patch"></param>
/// <returns></returns>
public async Task<bool> AddPatch(Patch patch)
{
await using var conn = await Open();
await using var trans = conn.BeginTransaction();
if (await conn.QuerySingleOrDefaultAsync<(Guid, Guid)>(
"Select SrcID, DestID FROM dbo.Patches where SrcID = @SrcId and DestID = @DestId",
new {SrcId = patch.Src.Id, DestId = patch.Dest.Id}, trans) != default)
return false;
await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)",
new {SrcId = patch.Src.Id, DestId = patch.Dest.Id}, trans);
await trans.CommitAsync();
return true;
}
/// <summary>
/// Adds a patch record
/// </summary>
/// <param name="patch"></param>
/// <returns></returns>
public async Task FinializePatch(Patch patch)
{
await using var conn = await Open();
await conn.ExecuteAsync("UPDATE dbo.Patches SET PatchSize = @PatchSize, Finished = @Finished, IsFailed = @IsFailed, FailMessage = @FailMessage WHERE SrcId = @SrcId AND DestID = @DestId",
new
{
SrcId = patch.Src.Id,
DestId = patch.Dest.Id,
PatchSize = patch.PatchSize,
Finished = patch.Finished,
IsFailed = patch.IsFailed,
FailMessage = patch.FailMessage
});
}
public async Task<Patch> FindPatch(Guid src, Guid dest)
{
await using var conn = await Open();
var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool?, string)>(
@"SELECT p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads src ON p.SrcId = src.Id
LEFT JOIN dbo.ArchiveDownloads dest ON p.SrcId = dest.Id
WHERE SrcId = @SrcId
AND DestId = @DestId
AND src.DownloadFinished IS NOT NULL
AND dest.DownloadFinished IS NOT NULL",
new
{
SrcId = src,
DestId = dest
});
if (patch == default)
return default;
return new Patch {
Src = await GetArchiveDownload(src),
Dest = await GetArchiveDownload(dest),
PatchSize = patch.Item1,
Finished = patch.Item2,
IsFailed = patch.Item3,
FailMessage = patch.Item4
};
}
public async Task<Patch> FindOrEnqueuePatch(Guid src, Guid dest)
{
await using var conn = await Open();
var trans = await conn.BeginTransactionAsync();
var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
"SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId",
new
{
SrcId = src,
DestId = dest
}, trans);
if (patch == default)
{
await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)",
new {SrcId = src, DestId = dest}, trans);
await trans.CommitAsync();
return new Patch {Src = await GetArchiveDownload(src), Dest = await GetArchiveDownload(dest),};
}
else
{
return new Patch {
Src = await GetArchiveDownload(src),
Dest = await GetArchiveDownload(dest),
PatchSize = patch.Item3,
Finished = patch.Item4,
IsFailed = patch.Item5,
FailMessage = patch.Item6
};
}
}
public async Task<Patch> GetPendingPatch()
{
await using var conn = await Open();
var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
@"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads src ON src.Id = p.SrcId
LEFT JOIN dbo.ArchiveDownloads dest ON dest.Id = p.DestId
WHERE p.Finished is NULL AND src.IsFailed = 0 AND dest.IsFailed = 0 ");
if (patch == default)
return default(Patch);
return new Patch {
Src = await GetArchiveDownload(patch.Item1),
Dest = await GetArchiveDownload(patch.Item2),
PatchSize = patch.Item3,
Finished = patch.Item4,
IsFailed = patch.Item5,
FailMessage = patch.Item6
};
}
public async Task<List<Patch>> PatchesForSource(Guid sourceDownload)
{
await using var conn = await Open();
var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
"SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId", new {SrcId = sourceDownload});
return await AsPatches(patches);
}
public async Task<List<Patch>> PatchesForSource(Hash sourceHash)
{
await using var conn = await Open();
var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
@"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id
WHERE a.Hash = @Hash AND p.Finished IS NOT NULL AND p.IsFailed = 0", new {Hash = sourceHash});
return await AsPatches(patches);
}
public async Task MarkPatchUsage(Guid srcId, Guid destId)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"UPDATE dbo.Patches SET Downloads = Downloads + 1, LastUsed = GETUTCDATE() WHERE SrcId = @srcId AND DestID = @destId",
new {SrcId = srcId, DestId = destId});
}
public async Task<List<Patch>> GetOldPatches()
{
await using var conn = await Open();
var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
@"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id
WHERE a.Hash not in (SELECT Hash FROM dbo.ModListArchives)");
return await AsPatches(patches);
}
private async Task<List<Patch>> AsPatches(IEnumerable<(Guid, Guid, long, DateTime?, bool?, string)> patches)
{
List<Patch> results = new List<Patch>();
foreach (var (srcId, destId, patchSize, finished, isFailed, failMessage) in patches)
{
results.Add(new Patch
{
Src = await GetArchiveDownload(srcId),
Dest = await GetArchiveDownload(destId),
PatchSize = patchSize,
Finished = finished,
IsFailed = isFailed,
FailMessage = failMessage
});
}
return results;
}
public async Task DeletePatch(Patch patch)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"DELETE FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestID",
new
{
SrcId = patch.Src.Id,
DestId = patch.Dest.Id
});
}
public async Task<HashSet<(Hash, Hash)>> AllPatchHashes()
{
await using var conn = await Open();
return (await conn.QueryAsync<(Hash, Hash)>(@"SELECT a1.Hash, a2.Hash
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a1 ON a1.Id = p.SrcId
LEFT JOIN dbo.ArchiveDownloads a2 on a2.Id = p.DestId
WHERE p.Finished IS NOT NULL")).ToHashSet();
}
public async Task DeletePatchesForHashPair((Hash, Hash) sqlFile)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"DELETE p
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a1 ON a1.Id = p.SrcId
LEFT JOIN dbo.ArchiveDownloads a2 on a2.Id = p.DestId
WHERE a1.Hash = @SrcHash
AND a2.Hash = @DestHash", new
{
SrcHash = sqlFile.Item1,
DestHash = sqlFile.Item2
});
}
public async Task PurgePatch(Hash hash, string rationale)
{
await using var conn = await Open();
await using var tx = await conn.BeginTransactionAsync();
await conn.ExecuteAsync(
"DELETE p FROM dbo.Patches p LEFT JOIN dbo.ArchiveDownloads ad ON ad.Id = p.SrcId WHERE ad.Hash = @Hash ",
new {Hash = hash}, tx);
await conn.ExecuteAsync(
"INSERT INTO dbo.NoPatch (Hash, Created, Rationale) VALUES (@Hash, GETUTCDATE(), @Rationale)",
new
{
Hash = hash,
Rationale = rationale
}, tx);
await tx.CommitAsync();
}
public async Task<bool> IsNoPatch(Hash hash)
{
await using var conn = await Open();
return await conn.QueryFirstOrDefaultAsync<Hash>("SELECT Hash FROM NoPatch WHERE Hash = @Hash", new {Hash = hash}) != default;
}
}
}

27
Wabbajack.Server/DataLayer/SqlService.cs
View File

@ -1,27 +0,0 @@
using System.Data.SqlClient;
using System.Threading.Tasks;
using Wabbajack.BuildServer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
private AppSettings _settings;
private Task<BunnyCdnFtpInfo> _mirrorCreds;
public SqlService(AppSettings settings)
{
_settings = settings;
_mirrorCreds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
}
public async Task<SqlConnection> Open()
{
var conn = new SqlConnection(_settings.SqlConnection);
await conn.OpenAsync();
return conn;
}
}
}

58
Wabbajack.Server/DataLayer/ValidationData.cs
View File

@ -1,58 +0,0 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task<ValidationData> GetValidationData()
{
var archiveStatus = AllModListArchivesStatus();
var modLists = AllModLists();
var mirrors = GetAllMirroredHashes();
var authoredFiles = AllAuthoredFiles();
var nexusFiles = await AllNexusFiles();
return new ValidationData
{
NexusFiles = nexusFiles.ToDictionary(nf => (nf.NexusGameId, nf.ModId, nf.FileId), nf => nf.category),
ArchiveStatus = await archiveStatus,
ModLists = await modLists,
Mirrors = await mirrors,
AllowedMirrors = new Lazy<Task<Dictionary<Hash, string>>>(async () => await GetAllowedMirrors()),
AllAuthoredFiles = await authoredFiles,
};
}
public async Task<Dictionary<(string PrimaryKeyString, Hash Hash), bool>> AllModListArchivesStatus()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<(string, Hash, bool)>(
@"SELECT PrimaryKeyString, Hash, IsValid FROM dbo.ModListArchiveStatus");
return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3);
}
public async Task<HashSet<(long NexusGameId, long ModId, long FileId, string category)>> AllNexusFiles()
{
await using var conn = await Open();
var results = await conn.QueryAsync<(long, long, long, string)>(@"SELECT Game, ModId, FileId, JSON_VALUE(Data, '$.category_name') FROM dbo.NexusModFile");
return results.ToHashSet();
}
public async Task<List<ModlistMetadata>> AllModLists()
{
await using var conn = await Open();
var results = await conn.QueryAsync<ModlistMetadata>(@"SELECT Metadata FROM dbo.ModLists");
return results.ToList();
}
}
}

37
Wabbajack.Server/DataLayer/VirusScanResults.cs
View File

@ -1,37 +0,0 @@
using System;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task<VirusScanner.Result?> FindVirusResult(Hash hash)
{
await using var conn = await Open();
var results = await conn.QueryFirstOrDefaultAsync<(Hash, bool)>(
@"SELECT Hash, IsMalware FROM dbo.VirusScanResults WHERE Hash = @Hash",
new {Hash = hash});
if (results == default)
return null;
return results.Item2 ? VirusScanner.Result.Malware : VirusScanner.Result.NotMalware;
}
public async Task AddVirusResult(Hash hash, VirusScanner.Result result)
{
await using var conn = await Open();
try
{
var results = await conn.QueryFirstOrDefaultAsync<(Hash, bool)>(
@"INSERT INTO dbo.VirusScanResults (Hash, IsMalware) VALUES (@Hash, @IsMalware)",
new {Hash = hash, IsMalware = result == VirusScanner.Result.Malware});
}
catch (Exception)
{
// ignored
}
}
}
}

12
Wabbajack.Server/GlobalInformation.cs
View File

@ -1,12 +0,0 @@
using System;
namespace Wabbajack.Server
{
public class GlobalInformation
{
public TimeSpan NexusRSSPollRate = TimeSpan.FromMinutes(1);
public TimeSpan NexusAPIPollRate = TimeSpan.FromMinutes(15);
public DateTime LastNexusSyncUTC { get; set; }
public TimeSpan TimeSinceLastNexusSync => DateTime.UtcNow - LastNexusSyncUTC;
}
}

47
Wabbajack.Server/Program.cs
View File

@ -1,47 +0,0 @@
using System;
using System.Linq;
using System.Net;
using System.Security.Cryptography.X509Certificates;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Hosting;
using Wabbajack.Common;
namespace Wabbajack.Server
{
public class Program
{
public static void Main(string[] args)
{
LoggingSettings.LogToFile = true;
Consts.IsServer = true;
bool testMode = args.Contains("TESTMODE");
CreateHostBuilder(args, testMode).Build().Run();
}
public static IHostBuilder CreateHostBuilder(string[] args, bool testMode) =>
Host.CreateDefaultBuilder(args)
.ConfigureWebHostDefaults(webBuilder =>
{
webBuilder.UseStartup<Startup>()
.UseKestrel(options =>
{
options.Listen(IPAddress.Any, testMode ? 8080 : 80);
if (!testMode)
{
options.Listen(IPAddress.Any, 443, listenOptions =>
{
using (var store = new X509Store(StoreName.My))
{
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Find(X509FindType.FindBySubjectName,
"build.wabbajack.org", true)[0];
listenOptions.UseHttps(cert);
}
});
}
options.Limits.MaxRequestBodySize = null;
});
});
}
}

124
Wabbajack.Server/Services/AbstractService.cs
View File

@ -1,124 +0,0 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
namespace Wabbajack.Server.Services
{
public interface IStartable
{
public void Start();
}
public interface IReportingService
{
public TimeSpan Delay { get; }
public DateTime LastStart { get; }
public DateTime LastEnd { get; }
public (String, DateTime)[] ActiveWorkStatus { get; }
}
public abstract class AbstractService<TP, TR> : IStartable, IReportingService
{
protected AppSettings _settings;
private TimeSpan _delay;
protected ILogger<TP> _logger;
protected QuickSync _quickSync;
public TimeSpan Delay => _delay;
public DateTime LastStart { get; private set; }
public DateTime LastEnd { get; private set; }
public (String, DateTime)[] ActiveWorkStatus { get; private set; }= { };
public AbstractService(ILogger<TP> logger, AppSettings settings, QuickSync quickSync, TimeSpan delay)
{
_settings = settings;
_delay = delay;
_logger = logger;
_quickSync = quickSync;
}
public virtual async Task Setup()
{
}
public void Start()
{
if (_settings.RunBackEndJobs)
{
Task.Run(async () =>
{
await Setup();
await _quickSync.Register(this);
while (true)
{
await _quickSync.ResetToken<TP>();
try
{
_logger.LogInformation($"Running: {GetType().Name}");
ActiveWorkStatus = Array.Empty<(String, DateTime)>();
LastStart = DateTime.UtcNow;
await Execute();
LastEnd = DateTime.UtcNow;
}
catch (Exception ex)
{
_logger.LogError(ex, "Running Service Loop");
Utils.Log($"Error in service {this.GetType()} : {ex}");
}
var token = await _quickSync.GetToken<TP>();
try
{
await Task.Delay(_delay, token);
}
catch (TaskCanceledException)
{
}
}
});
}
}
public abstract Task<TR> Execute();
protected void ReportStarting(string value)
{
lock (this)
{
ActiveWorkStatus = ActiveWorkStatus.Cons((value, DateTime.UtcNow)).ToArray();
}
}
protected void ReportEnding(string value)
{
lock (this)
{
ActiveWorkStatus = ActiveWorkStatus.Where(x => x.Item1 != value).ToArray();
}
}
}
public static class AbstractServiceExtensions
{
public static void UseService<T>(this IApplicationBuilder b)
{
var poll = (IStartable)b.ApplicationServices.GetService(typeof(T));
poll.Start();
}
}
}

149
Wabbajack.Server/Services/ArchiveDownloader.cs
View File

@ -1,149 +0,0 @@
using System;
using System.Reflection.Metadata.Ecma335;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class ArchiveDownloader : AbstractService<ArchiveDownloader, int>
{
private SqlService _sql;
private ArchiveMaintainer _archiveMaintainer;
private NexusApiClient _nexusClient;
private DiscordWebHook _discord;
private NexusKeyMaintainance _nexus;
public ArchiveDownloader(ILogger<ArchiveDownloader> logger, AppSettings settings, SqlService sql, ArchiveMaintainer archiveMaintainer, DiscordWebHook discord, QuickSync quickSync, NexusKeyMaintainance nexus)
: base(logger, settings, quickSync, TimeSpan.FromMinutes(10))
{
_sql = sql;
_archiveMaintainer = archiveMaintainer;
_discord = discord;
_nexus = nexus;
}
public override async Task<int> Execute()
{
_nexusClient ??= await _nexus.GetClient();
int count = 0;
while (true)
{
var (daily, hourly) = await _nexusClient.GetRemainingApiCalls();
bool ignoreNexus = (daily < 100 && hourly < 10);
//var ignoreNexus = true;
if (ignoreNexus)
_logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {daily}, Hourly:{hourly})");
else
_logger.LogInformation($"Looking for any download (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})");
var nextDownload = await _sql.GetNextPendingDownload(ignoreNexus);
if (nextDownload == null)
break;
_logger.LogInformation($"Checking for previously archived {nextDownload.Archive.Hash}");
if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash))
{
await nextDownload.Finish(_sql);
continue;
}
if (nextDownload.Archive.State is ManualDownloader.State)
{
await nextDownload.Finish(_sql);
continue;
}
try
{
_logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
ReportStarting(nextDownload.Archive.State.PrimaryKeyString);
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"
});
await DownloadDispatcher.PrepareAll(new[] {nextDownload.Archive.State});
await using var tempPath = new TempFile();
if (!await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path))
{
_logger.LogError(
$"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, "Downloader returned false");
continue;
}
var hash = await tempPath.Path.FileHashAsync();
if (hash == null || (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash))
{
_logger.Log(LogLevel.Warning,
$"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}");
await nextDownload.Fail(_sql, "Invalid Hash");
continue;
}
if (nextDownload.Archive.Size != default &&
tempPath.Path.Size != nextDownload.Archive.Size)
{
await nextDownload.Fail(_sql, "Invalid Size");
continue;
}
nextDownload.Archive.Hash = hash.Value;
nextDownload.Archive.Size = tempPath.Path.Size;
_logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await _archiveMaintainer.Ingest(tempPath.Path);
_logger.Log(LogLevel.Information,
$"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Finish(_sql);
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}"
});
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, ex.ToString());
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"
});
}
finally
{
ReportEnding(nextDownload.Archive.State.PrimaryKeyString);
}
count++;
}
if (count > 0)
{
// Wake the Patch builder up in case it needs to build a patch now
await _quickSync.Notify<PatchBuilder>();
}
return count;
}
}
}

76
Wabbajack.Server/Services/ArchiveMaintainer.cs
View File

@ -1,76 +0,0 @@
using System;
using System.Collections.Concurrent;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using File = System.IO.File;
namespace Wabbajack.Server.Services
{
/// <summary>
/// Maintains a concurrent cache of all the files we've downloaded, indexed by Hash.
/// </summary>
public class ArchiveMaintainer
{
private AppSettings _settings;
private ILogger<ArchiveMaintainer> _logger;
public ArchiveMaintainer(ILogger<ArchiveMaintainer> logger, AppSettings settings)
{
_settings = settings;
_logger = logger;
_logger.Log(LogLevel.Information, "Creating Archive Maintainer");
}
public void Start()
{
_logger.Log(LogLevel.Information, $"Found {_settings.ArchivePath.EnumerateFiles(false).Count()} archives");
}
private AbsolutePath ArchivePath(Hash hash)
{
return _settings.ArchivePath.Combine(hash.ToHex());
}
public async Task Ingest(AbsolutePath file)
{
var hash = await file.FileHashAsync();
if (hash == null) return;
var path = ArchivePath(hash.Value);
if (HaveArchive(hash.Value))
{
await file.DeleteAsync();
return;
}
var newPath = _settings.ArchivePath.Combine(hash.Value.ToHex());
await file.MoveToAsync(newPath);
}
public bool HaveArchive(Hash hash)
{
return ArchivePath(hash).Exists;
}
public bool TryGetPath(Hash hash, out AbsolutePath path)
{
path = ArchivePath(hash);
return path.Exists;
}
}
public static class ArchiveMaintainerExtensions
{
public static void UseArchiveMaintainer(this IApplicationBuilder b)
{
var poll = (ArchiveMaintainer)b.ApplicationServices.GetService(typeof(ArchiveMaintainer));
poll.Start();
}
}
}

139
Wabbajack.Server/Services/AuthoredFilesCleanup.cs
View File

@ -1,139 +0,0 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using FluentFTP;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.AuthorApi;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using WebSocketSharp;
namespace Wabbajack.Server.Services
{
public class AuthoredFilesCleanup : AbstractService<AuthoredFilesCleanup, int>
{
private SqlService _sql;
private DiscordWebHook _discord;
public AuthoredFilesCleanup(ILogger<AuthoredFilesCleanup> logger, AppSettings settings, QuickSync quickSync, SqlService sql, DiscordWebHook discord) : base(logger, settings, quickSync, TimeSpan.FromHours(6))
{
_sql = sql;
_discord = discord;
}
public override async Task<int> Execute()
{
var toDelete = await FindFilesToDelete();
var log = new[] {$"CDNDelete ({toDelete.CDNDelete.Length}):\n\n"}
.Concat(toDelete.CDNDelete)
.Concat(new[] {$"SQLDelete ({toDelete.SQLDelete.Length}"})
.Concat(toDelete.SQLDelete)
.Concat(new[] {$"CDNRemain ({toDelete.CDNNotDeleted.Length}"})
.Concat(toDelete.CDNNotDeleted)
.Concat(new[] {$"SQLRemain ({toDelete.SQLNotDeleted.Length}"})
.Concat(toDelete.SQLNotDeleted)
.ToArray();
//await AbsolutePath.EntryPoint.Combine("cdn_delete_log.txt").WriteAllLinesAsync(log);
foreach (var sqlFile in toDelete.SQLDelete)
{
Utils.Log($"Deleting {sqlFile} from SQL");
await _sql.DeleteFileDefinition(await _sql.GetCDNFileDefinition(sqlFile));
}
using var queue = new WorkQueue(6);
await toDelete.CDNDelete.Select((d, idx) => (d, idx)).PMap(queue, async cdnFile =>
{
using var conn = await (await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles)).GetClient();
Utils.Log($"Deleting {cdnFile} from CDN");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"({cdnFile.idx}/{toDelete.CDNDelete.Length}) {cdnFile.d} is no longer referenced by any modlist and will be removed from the CDN"
});
if (await conn.DirectoryExistsAsync(cdnFile.d))
await conn.DeleteDirectoryAsync(cdnFile.d);
if (await conn.FileExistsAsync(cdnFile.d))
await conn.DeleteFileAsync(cdnFile.d);
});
return toDelete.CDNDelete.Length + toDelete.SQLDelete.Length;
}
public async Task<(string[] CDNDelete, string[] SQLDelete, string[] CDNNotDeleted, string[] SQLNotDeleted)> FindFilesToDelete()
{
var cdnNames = (await GetCDNMungedNames()).ToHashSet();
var usedNames = (await GetUsedCDNFiles()).ToHashSet();
var sqlFiles = (await _sql.AllAuthoredFiles()).ToDictionary(f => f.MungedName);
var keep = GetKeepList(cdnNames, usedNames, sqlFiles).ToHashSet();
var cdnDelete = cdnNames.Where(h => !keep.Contains(h)).ToArray();
var sqlDelete = sqlFiles.Where(s => !keep.Contains(s.Value.MungedName))
.Select(s => s.Value.ServerAssignedUniqueId)
.ToArray();
var cdnhs = cdnDelete.ToHashSet();
var notDeletedCDN = cdnNames.Where(f => !cdnhs.Contains(f)).ToArray();
var sqlhs = sqlDelete.ToHashSet();
var sqlNotDeleted = sqlFiles.Where(f => !sqlDelete.Contains(f.Value.ServerAssignedUniqueId))
.Select(f => f.Value.MungedName)
.ToArray();
return (cdnDelete, sqlDelete, notDeletedCDN, sqlNotDeleted);
}
private IEnumerable<string> GetKeepList(HashSet<string> cdnNames, HashSet<string> usedNames, Dictionary<string, AuthoredFilesSummary> sqlFiles)
{
var cutOff = DateTime.UtcNow - TimeSpan.FromDays(7);
foreach (var file in sqlFiles.Where(f => f.Value.LastTouched > cutOff))
yield return file.Value.MungedName;
foreach (var file in usedNames)
yield return file;
}
public async Task<string[]> GetCDNMungedNames()
{
using var client = await (await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles)).GetClient();
var lst = await client.GetListingAsync(@"\");
return lst.Select(l => l.Name).ToArray();
}
public async Task<string[]> GetUsedCDNFiles()
{
var modlists = (await ModlistMetadata.LoadFromGithub())
.Concat((await ModlistMetadata.LoadUnlistedFromGithub()))
.Select(f => f.Links.Download)
.Where(f => f.StartsWith(Consts.WabbajackAuthoredFilesPrefix))
.Select(f => f.Substring(Consts.WabbajackAuthoredFilesPrefix.Length));
var files = (await _sql.ModListArchives())
.Select(a => a.State)
.OfType<WabbajackCDNDownloader.State>()
.Select(s => s.Url.ToString().Substring(Consts.WabbajackAuthoredFilesPrefix.Length));
var names = modlists.Concat(files).Distinct().ToArray();
var namesBoth = names.Concat(names.Select(HttpUtility.UrlDecode))
.Concat(names.Select(HttpUtility.UrlEncode))
.Distinct()
.ToArray();
return namesBoth;
}
}
}

58
Wabbajack.Server/Services/CDNMirrorList.cs
View File

@ -1,58 +0,0 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using System.Xml.Linq;
using HtmlAgilityPack;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
namespace Wabbajack.Server.Services
{
public class CDNMirrorList : AbstractService<CDNMirrorList, int>
{
public CDNMirrorList(ILogger<CDNMirrorList> logger, AppSettings settings, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(6))
{
}
public string[] Mirrors { get; private set; }
public DateTime LastUpdate { get; private set; }
public override async Task<int> Execute()
{
var client = new Lib.Http.Client();
var json = await client.GetStringAsync("https://bunnycdn.com/api/system/edgeserverlist");
client.Headers.Add(("Host", "wabbajack.b-cdn.net"));
using var queue = new WorkQueue();
var mirrors = json.FromJsonString<string[]>();
_logger.LogInformation($"Found {mirrors.Length} edge severs");
var servers = (await mirrors
.PMap(queue, async ip =>
{
try
{
// We use a volume server, so this file will only exist on some (lower cost) servers
using var result = await client.GetAsync(
$"https://{ip}/WABBAJACK_TEST_FILE.zip_48f799f6-39b2-4229-a329-7459c9965c2d/definition.json.gz",
errorsAsExceptions: false, retry: false);
var data = await result.Content.ReadAsByteArrayAsync();
return (ip, use: result.IsSuccessStatusCode, size : data.Length);
}
catch (Exception)
{
return (ip, use : false, size: 0);
}
}))
.Where(r => r.use && r.size == 267)
.Select(r => r.ip)
.ToArray();
_logger.LogInformation($"Found {servers.Length} valid mirrors");
Mirrors = servers;
LastUpdate = DateTime.UtcNow;
return Mirrors.Length;
}
}
}

199
Wabbajack.Server/Services/DiscordFrontend.cs
View File

@ -1,199 +0,0 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Discord;
using Discord.WebSocket;
using Microsoft.Extensions.Logging;
using OMODFramework;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
using Utils = Wabbajack.Common.Utils;
namespace Wabbajack.Server.Services
{
public class DiscordFrontend : IStartable
{
private ILogger<DiscordFrontend> _logger;
private AppSettings _settings;
private QuickSync _quickSync;
private DiscordSocketClient _client;
private SqlService _sql;
private MetricsKeyCache _keyCache;
private ListValidator _listValidator;
public DiscordFrontend(ILogger<DiscordFrontend> logger, AppSettings settings, QuickSync quickSync, ListValidator listValidator, SqlService sql, MetricsKeyCache keyCache)
{
_logger = logger;
_settings = settings;
_quickSync = quickSync;
_client = new DiscordSocketClient();
_client.Log += LogAsync;
_client.Ready += ReadyAsync;
_client.MessageReceived += MessageReceivedAsync;
_sql = sql;
_keyCache = keyCache;
_listValidator = listValidator;
}
private async Task MessageReceivedAsync(SocketMessage arg)
{
_logger.LogInformation(arg.Content);
if (arg.Content.StartsWith("!dervenin"))
{
var parts = arg.Content.Split(" ", StringSplitOptions.RemoveEmptyEntries);
if (parts[0] != "!dervenin")
return;
if (parts.Length == 1)
{
await ReplyTo(arg, "Wat?");
}
if (parts[1] == "purge-nexus-cache")
{
if (parts.Length != 3)
{
await ReplyTo(arg, "Welp you did that wrong, gotta give me a mod-id or url");
return;
}
await PurgeNexusCache(arg, parts[2]);
}
else if (parts[1] == "quick-sync")
{
var options = await _quickSync.Report();
if (parts.Length != 3)
{
var optionsStr = string.Join(", ", options.Select(o => o.Key.Name));
await ReplyTo(arg, $"Can't expect me to quicksync the whole damn world! Try: {optionsStr}");
}
else
{
foreach (var pair in options.Where(o => o.Key.Name == parts[2]))
{
await _quickSync.Notify(pair.Key);
await ReplyTo(arg, $"Notified {pair.Key}");
}
}
}
else if (parts[1] == "purge-list")
{
if (parts.Length != 3)
{
await ReplyTo(arg, $"Yeah, I'm not gonna purge the whole server...");
}
else
{
var deleted = await _sql.PurgeList(parts[2]);
_listValidator.ValidationInfo.TryRemove(parts[2], out var _);
await _quickSync.Notify<ModListDownloader>();
await ReplyTo(arg, $"Purged all traces of #{parts[2]} from the server, triggered list downloading. {deleted} records removed");
}
}
else if (parts[1] == "mirror-mod")
{
await MirrorModCommand(arg, parts);
}
else if (parts[1] == "users")
{
await ReplyTo(arg, $"Wabbajack has {await _keyCache.KeyCount()} known unique users");
}
}
}
private async Task MirrorModCommand(SocketMessage msg, string[] parts)
{
if (parts.Length != 2)
{
await ReplyTo(msg, "Command is: mirror-mod <game-name> <mod-id>");
return;
}
if (long.TryParse(parts[2], out var modId))
{
await ReplyTo(msg, $"Got {modId} for a mod-id, expected a integer");
return;
}
if (GameRegistry.TryGetByFuzzyName(parts[1], out var game))
{
var gameNames = GameRegistry.Games.Select(g => g.Value.NexusName)
.Where(g => !string.IsNullOrWhiteSpace(g))
.Select(g => (string)g)
.ToHashSet();
var joined = string.Join(", ", gameNames.OrderBy(g => g));
await ReplyTo(msg, $"Got {parts[1]} for a game name, expected something like: {joined}");
}
if (game!.NexusGameId == default)
{
await ReplyTo(msg, $"No NexusGameID found for {game}");
}
await _sql.AddNexusModWithOpenPerms(game.Game, modId);
await _quickSync.Notify<MirrorUploader>();
await ReplyTo(msg, "Done, and I notified the uploader");
}
private async Task PurgeNexusCache(SocketMessage arg, string mod)
{
if (Uri.TryCreate(mod, UriKind.Absolute, out var url))
{
mod = Enumerable.Last(url.AbsolutePath.Split("/", StringSplitOptions.RemoveEmptyEntries));
}
if (int.TryParse(mod, out var mod_id))
{
await _sql.PurgeNexusCache(mod_id);
await _quickSync.Notify<ListValidator>();
await ReplyTo(arg, $"It is done, {mod_id} has been purged, list validation has been triggered");
}
}
private async Task ReplyTo(SocketMessage socketMessage, string message)
{
await socketMessage.Channel.SendMessageAsync(message);
}
private async Task ReadyAsync()
{
}
private async Task LogAsync(LogMessage arg)
{
switch (arg.Severity)
{
case LogSeverity.Info:
_logger.LogInformation(arg.Message);
break;
case LogSeverity.Warning:
_logger.LogWarning(arg.Message);
break;
case LogSeverity.Critical:
_logger.LogCritical(arg.Message);
break;
case LogSeverity.Error:
_logger.LogError(arg.Exception, arg.Message);
break;
case LogSeverity.Verbose:
_logger.LogTrace(arg.Message);
break;
case LogSeverity.Debug:
_logger.LogDebug(arg.Message);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
public void Start()
{
_client.LoginAsync(TokenType.Bot, Utils.FromEncryptedJson<string>("discord-key").Result).Wait();
_client.StartAsync().Wait();
}
}
}

71
Wabbajack.Server/Services/DiscordWebHook.cs
View File

@ -1,71 +0,0 @@
using System;
using System.Net.Http;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public enum Channel
{
// High volume messaging, really only useful for internal devs
Spam,
// Low volume messages designed for admins
Ham
}
public class DiscordWebHook : AbstractService<DiscordWebHook, int>
{
private Random _random = new Random();
public DiscordWebHook(ILogger<DiscordWebHook> logger, AppSettings settings, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(1))
{
_settings = settings;
_logger = logger;
var message = new DiscordMessage
{
Content = $"\"{GetQuote()}\" - Sheogorath (as he brings the server online)",
};
var a = Send(Channel.Ham, message);
var b = Send(Channel.Spam, message);
}
public async Task Send(Channel channel, DiscordMessage message)
{
try
{
string url = channel switch
{
Channel.Spam => _settings.SpamWebHook,
Channel.Ham => _settings.HamWebHook,
_ => null
};
if (url == null) return;
var client = new Wabbajack.Lib.Http.Client();
await client.PostAsync(url, new StringContent(message.ToJson(true), Encoding.UTF8, "application/json"));
}
catch (Exception ex)
{
_logger.LogError(ex, ex.ToString());
}
}
private string GetQuote()
{
var data = Assembly.GetExecutingAssembly()!.GetManifestResourceStream("Wabbajack.Server.sheo_quotes.txt");
var lines = Encoding.UTF8.GetString(data.ReadAll()).Split('\n');
return lines[_random.Next(lines.Length)].Trim();
}
public override async Task<int> Execute()
{
return 0;
}
}
}

441
Wabbajack.Server/Services/ListValidator.cs
View File

@ -1,441 +0,0 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using ArchiveStatus = Wabbajack.Server.DTOs.ArchiveStatus;
namespace Wabbajack.Server.Services
{
public class ListValidator : AbstractService<ListValidator, int>
{
private SqlService _sql;
private DiscordWebHook _discord;
private NexusKeyMaintainance _nexus;
private ArchiveMaintainer _archives;
public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries => ValidationInfo.Values.Select(e => (e.Summary, e.Detailed));
public ConcurrentDictionary<string, (ModListSummary Summary, DetailedStatus Detailed, TimeSpan ValidationTime)> ValidationInfo = new();
public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives, QuickSync quickSync)
: base(logger, settings, quickSync, TimeSpan.FromMinutes(5))
{
_sql = sql;
_discord = discord;
_nexus = nexus;
_archives = archives;
}
public override async Task<int> Execute()
{
var data = await _sql.GetValidationData();
_logger.LogInformation("Found {count} nexus files", data.NexusFiles.Count);
using var queue = new WorkQueue();
var oldSummaries = Summaries;
var stopwatch = new Stopwatch();
stopwatch.Start();
var results = await data.ModLists.Where(m => !m.ForceDown).PMap(queue, async metadata =>
{
var timer = new Stopwatch();
timer.Start();
var oldSummary =
oldSummaries.FirstOrDefault(s => s.Summary.MachineURL == metadata.Links.MachineURL);
var mainFile = await DownloadDispatcher.Infer(new Uri(metadata.Links.Download));
var mainArchive = new Archive(mainFile!)
{
Size = metadata.DownloadMetadata!.Size,
Hash = metadata.DownloadMetadata!.Hash
};
bool mainFailed = false;
try
{
if (mainArchive.State is WabbajackCDNDownloader.State)
{
if (!await mainArchive.State.Verify(mainArchive))
{
mainFailed = true;
}
}
}
catch (Exception ex)
{
mainFailed = true;
}
var listArchives = await _sql.ModListArchives(metadata.Links.MachineURL);
var archives = await listArchives.PMap(queue, async archive =>
{
if (mainFailed)
return (archive, ArchiveStatus.InValid);
try
{
ReportStarting(archive.State.PrimaryKeyString);
if (timer.Elapsed > Delay)
{
return (archive, ArchiveStatus.InValid);
}
var (_, result) = await ValidateArchive(data, archive);
if (result == ArchiveStatus.InValid)
{
if (data.Mirrors.TryGetValue(archive.Hash, out var done))
return (archive, done ? ArchiveStatus.Mirrored : ArchiveStatus.Updating);
if ((await data.AllowedMirrors.Value).TryGetValue(archive.Hash, out var reason))
{
await _sql.StartMirror((archive.Hash, reason));
return (archive, ArchiveStatus.Updating);
}
if (archive.State is NexusDownloader.State)
return (archive, result);
return await TryToHeal(data, archive, metadata);
}
return (archive, result);
}
catch (Exception ex)
{
_logger.LogError(ex, $"During Validation of {archive.Hash} {archive.State.PrimaryKeyString}");
Utils.Log($"Exception in validation of {archive.Hash} {archive.State.PrimaryKeyString} " + ex);
return (archive, ArchiveStatus.InValid);
}
finally
{
ReportEnding(archive.State.PrimaryKeyString);
}
});
var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid || f.Item2 == ArchiveStatus.Updating);
var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated);
var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating);
var mirroredCount = archives.Count(f => f.Item2 == ArchiveStatus.Mirrored);
var summary = new ModListSummary
{
Checked = DateTime.UtcNow,
Failed = failedCount,
Passed = passCount,
Updating = updatingCount,
Mirrored = mirroredCount,
MachineURL = metadata.Links.MachineURL,
Name = metadata.Title,
ModListIsMissing = mainFailed
};
var detailed = new DetailedStatus
{
Name = metadata.Title,
Checked = DateTime.UtcNow,
DownloadMetaData = metadata.DownloadMetadata,
HasFailures = failedCount > 0,
MachineName = metadata.Links.MachineURL,
Archives = archives.Select(a => new DetailedStatusItem
{
Archive = a.Item1,
IsFailing = a.Item2 == ArchiveStatus.InValid,
ArchiveStatus = a.Item2
}).ToList()
};
if (timer.Elapsed > Delay)
{
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Embeds = new[]
{
new DiscordEmbed
{
Title =
$"Failing {summary.Name} (`{summary.MachineURL}`) because the max validation time expired",
Url = new Uri(
$"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html")
}
}
});
}
if (oldSummary != default && oldSummary.Summary.Failed != summary.Failed)
{
_logger.Log(LogLevel.Information, $"Number of failures {oldSummary.Summary.Failed} -> {summary.Failed}");
if (summary.HasFailures)
{
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Embeds = new[]
{
new DiscordEmbed
{
Title =
$"Number of failures in {summary.Name} (`{summary.MachineURL}`) was {oldSummary.Summary.Failed} is now {summary.Failed}",
Url = new Uri(
$"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html")
}
}
});
}
if (!summary.HasFailures && oldSummary.Summary.HasFailures)
{
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Embeds = new[]
{
new DiscordEmbed
{
Title = $"{summary.Name} (`{summary.MachineURL}`) is now passing.",
Url = new Uri(
$"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html")
}
}
});
}
}
timer.Stop();
ValidationInfo[summary.MachineURL] = (summary, detailed, timer.Elapsed);
return (summary, detailed);
});
stopwatch.Stop();
_logger.LogInformation($"Finished Validation in {stopwatch.Elapsed}");
return Summaries.Count(s => s.Summary.HasFailures);
}
private AsyncLock _healLock = new AsyncLock();
private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive, ModlistMetadata modList)
{
try
{
using var _ = await _healLock.WaitAsync();
var srcDownload =
await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size);
if (srcDownload == null || srcDownload.IsFailed == true)
{
_logger.Log(LogLevel.Information,
$"Cannot heal {archive.State.PrimaryKeyString} Size: {archive.Size} Hash: {(long)archive.Hash} because it hasn't been previously successfully downloaded");
return (archive, ArchiveStatus.InValid);
}
var patches = await _sql.PatchesForSource(archive.Hash);
foreach (var patch in patches)
{
if (patch.Finished is null)
return (archive, ArchiveStatus.Updating);
if (patch.IsFailed == true)
return (archive, ArchiveStatus.InValid);
var (_, status) = await ValidateArchive(data, patch.Dest.Archive);
if (status == ArchiveStatus.Valid)
return (archive, ArchiveStatus.Updated);
}
var upgradeTime = DateTime.UtcNow;
_logger.LogInformation(
$"Validator Finding Upgrade for {archive.Hash} {archive.State.PrimaryKeyString}");
Func<Archive, Task<AbsolutePath>> resolver = async findIt =>
{
_logger.LogInformation($"Quick find for {findIt.State.PrimaryKeyString}");
var foundArchive = await _sql.GetArchiveDownload(findIt.State.PrimaryKeyString);
if (foundArchive == null)
{
_logger.LogInformation($"No Quick find for {findIt.State.PrimaryKeyString}");
return default;
}
return _archives.TryGetPath(foundArchive.Archive.Hash, out var path) ? path : default;
};
if (archive.State is NexusDownloader.State)
{
DownloadDispatcher.GetInstance<NexusDownloader>().Client = await _nexus.GetClient();
}
var upgrade = await DownloadDispatcher.FindUpgrade(archive, resolver);
if (upgrade == default)
{
_logger.Log(LogLevel.Information,
$"Cannot heal {archive.State.PrimaryKeyString} because an alternative wasn't found");
return (archive, ArchiveStatus.InValid);
}
_logger.LogInformation(
$"Upgrade {upgrade.Archive.State.PrimaryKeyString} found for {archive.State.PrimaryKeyString}");
{
}
var found = await _sql.GetArchiveDownload(upgrade.Archive.State.PrimaryKeyString, upgrade.Archive.Hash,
upgrade.Archive.Size);
Guid id;
if (found == null)
{
if (upgrade.NewFile.Path.Exists)
await _archives.Ingest(upgrade.NewFile.Path);
id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime);
}
else
{
id = found.Id;
}
var destDownload = await _sql.GetArchiveDownload(id);
if (destDownload.Archive.Hash == srcDownload.Archive.Hash &&
destDownload.Archive.State.PrimaryKeyString == srcDownload.Archive.State.PrimaryKeyString)
{
_logger.Log(LogLevel.Information, $"Can't heal because src and dest match");
return (archive, ArchiveStatus.InValid);
}
if (destDownload.Archive.Hash == default)
{
_logger.Log(LogLevel.Information,
"Can't heal because we got back a default hash for the downloaded file");
return (archive, ArchiveStatus.InValid);
}
var existing = await _sql.FindPatch(srcDownload.Id, destDownload.Id);
if (existing == null)
{
if (await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload}))
{
_logger.Log(LogLevel.Information,
$"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`"
});
}
}
await upgrade.NewFile.DisposeAsync();
_logger.LogInformation($"Patch in progress {archive.Hash} {archive.State.PrimaryKeyString}");
return (archive, ArchiveStatus.Updating);
}
catch (Exception ex)
{
_logger.LogError(ex, "During healing");
return (archive, ArchiveStatus.InValid);
}
}
private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(ValidationData data, Archive archive)
{
switch (archive.State)
{
case GoogleDriveDownloader.State _:
// Disabled for now due to GDrive rate-limiting the build server
return (archive, ArchiveStatus.Valid);
case NexusDownloader.State nexusState when data.NexusFiles.TryGetValue(
(nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID), out var category):
return (archive, category != null ? ArchiveStatus.Valid : ArchiveStatus.InValid);
case NexusDownloader.State ns:
return (archive, await FastNexusModStats(ns));
case ManualDownloader.State _:
return (archive, ArchiveStatus.Valid);
case ModDBDownloader.State _:
return (archive, ArchiveStatus.Valid);
case GameFileSourceDownloader.State _:
return (archive, ArchiveStatus.Valid);
case MediaFireDownloader.State _:
return (archive, ArchiveStatus.Valid);
case DeprecatedLoversLabDownloader.State _:
return (archive, ArchiveStatus.InValid);
case DeprecatedVectorPlexusDownloader.State _:
return (archive, ArchiveStatus.InValid);
default:
{
if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash),
out bool isValid))
{
return isValid ? (archive, ArchiveStatus.Valid) : (archive, ArchiveStatus.InValid);
}
return (archive, ArchiveStatus.Valid);
}
}
}
public async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
{
// Check if some other thread has added them
var file = await _sql.GetModFile(ns.Game, ns.ModID, ns.FileID);
if (file == null)
{
try
{
NexusApiClient nexusClient = await _nexus.GetClient();
var queryTime = DateTime.UtcNow;
_logger.Log(LogLevel.Information, "Found missing Nexus file info {Game} {ModID} {FileID}", ns.Game, ns.ModID, ns.FileID);
try
{
file = await nexusClient.GetModFile(ns.Game, ns.ModID, ns.FileID, false);
}
catch
{
file = new NexusFileInfo() {category_name = null};
}
try
{
await _sql.AddNexusModFile(ns.Game, ns.ModID, ns.FileID, queryTime, file);
}
catch (Exception)
{
// Could be a PK constraint failure
}
}
catch (Exception)
{
return ArchiveStatus.InValid;
}
}
return file?.category_name != null ? ArchiveStatus.Valid : ArchiveStatus.InValid;
}
}
}

62
Wabbajack.Server/Services/MetricsKeyCache.cs
View File

@ -1,62 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.Services
{
public class MetricsKeyCache : IStartable
{
private ILogger<MetricsKeyCache> _logger;
private SqlService _sql;
private HashSet<string> _knownKeys = new();
private AsyncLock _lock = new();
public MetricsKeyCache(ILogger<MetricsKeyCache> logger, SqlService sql)
{
_logger = logger;
_sql = sql;
}
public async Task<bool> IsValidKey(string key)
{
using (var _ = await _lock.WaitAsync())
{
if (_knownKeys.Contains(key)) return true;
}
if (await _sql.ValidMetricsKey(key))
{
using var _ = await _lock.WaitAsync();
_knownKeys.Add(key);
return true;
}
return false;
}
public async Task AddKey(string key)
{
using (var _ = await _lock.WaitAsync())
{
if (_knownKeys.Contains(key)) return;
_knownKeys.Add(key);
}
await _sql.AddMetricsKey(key);
}
public void Start()
{
_knownKeys = (_sql.AllKeys().Result).ToHashSet();
}
public async Task<long> KeyCount()
{
using var _ = await _lock.WaitAsync();
return _knownKeys.Count;
}
}
}

28
Wabbajack.Server/Services/MirrorQueueService.cs
View File

@ -1,28 +0,0 @@
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.Services
{
public class MirrorQueueService : AbstractService<MirrorQueueService, int>
{
private DiscordWebHook _discord;
private SqlService _sql;
public MirrorQueueService(ILogger<MirrorQueueService> logger, AppSettings settings, QuickSync quickSync,
DiscordWebHook discordWebHook, SqlService sqlService) :
base(logger, settings, quickSync, TimeSpan.FromMinutes(5))
{
_discord = discordWebHook;
_sql = sqlService;
}
public override async Task<int> Execute()
{
await _sql.QueueMirroredFiles();
return 1;
}
}
}

208
Wabbajack.Server/Services/MirrorUploader.cs
View File

@ -1,208 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
using FluentFTP.Helpers;
using Microsoft.Extensions.Logging;
using Org.BouncyCastle.Utilities.Collections;
using Wabbajack.BuildServer;
using Wabbajack.BuildServer.Controllers;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.AuthorApi;
using Wabbajack.Lib.FileUploader;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class MirrorUploader : AbstractService<MirrorUploader, int>
{
private SqlService _sql;
private ArchiveMaintainer _archives;
private DiscordWebHook _discord;
public bool ActiveFileSyncEnabled { get; set; } = true;
public MirrorUploader(ILogger<MirrorUploader> logger, AppSettings settings, SqlService sql, QuickSync quickSync, ArchiveMaintainer archives, DiscordWebHook discord)
: base(logger, settings, quickSync, TimeSpan.FromHours(1))
{
_sql = sql;
_archives = archives;
_discord = discord;
}
public override async Task<int> Execute()
{
int uploaded = 0;
if (ActiveFileSyncEnabled)
await _sql.SyncActiveMirroredFiles();
TOP:
var toUpload = await _sql.GetNextMirroredFile();
if (toUpload == default)
{
await DeleteOldMirrorFiles();
return uploaded;
}
uploaded += 1;
try
{
var creds = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
using var queue = new WorkQueue();
if (_archives.TryGetPath(toUpload.Hash, out var path))
{
_logger.LogInformation($"Uploading mirror file {toUpload.Hash} {path.Size.FileSizeToString()}");
bool exists = false;
using (var client = await GetClient(creds))
{
exists = await client.FileExistsAsync($"{toUpload.Hash.ToHex()}/definition.json.gz");
}
if (exists)
{
_logger.LogInformation($"Skipping {toUpload.Hash} it's already on the server");
await toUpload.Finish(_sql);
goto TOP;
}
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Uploading {toUpload.Hash} - {toUpload.Created} because {toUpload.Rationale}"
});
var definition = await Client.GenerateFileDefinition(queue, path, (s, percent) => { });
using (var client = await GetClient(creds))
{
await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}");
await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}/parts");
}
string MakePath(long idx)
{
return $"{definition.Hash.ToHex()}/parts/{idx}";
}
await definition.Parts.PMap(queue, async part =>
{
_logger.LogInformation($"Uploading mirror part ({part.Index}/{definition.Parts.Length})");
var buffer = new byte[part.Size];
await using (var fs = await path.OpenShared())
{
fs.Position = part.Offset;
await fs.ReadAsync(buffer);
}
await CircuitBreaker.WithAutoRetryAllAsync(async () =>{
using var client = await GetClient(creds);
var name = MakePath(part.Index);
await client.UploadAsync(new MemoryStream(buffer), name);
});
});
await CircuitBreaker.WithAutoRetryAllAsync(async () =>
{
using var client = await GetClient(creds);
_logger.LogInformation($"Finishing mirror upload");
await using var ms = new MemoryStream();
await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
{
definition.ToJson(gz);
}
ms.Position = 0;
var remoteName = $"{definition.Hash.ToHex()}/definition.json.gz";
await client.UploadAsync(ms, remoteName);
});
await toUpload.Finish(_sql);
}
else
{
await toUpload.Fail(_sql, "Archive not found");
}
}
catch (Exception ex)
{
_logger.LogInformation($"{toUpload.Created} {toUpload.Uploaded}");
_logger.LogError(ex, "Error uploading");
await toUpload.Fail(_sql, ex.ToString());
}
goto TOP;
}
private static async Task<FtpClient> GetClient(BunnyCdnFtpInfo creds = null)
{
return await CircuitBreaker.WithAutoRetryAllAsync<FtpClient>(async () =>
{
creds ??= await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password));
ftpClient.DataConnectionType = FtpDataConnectionType.EPSV;
await ftpClient.ConnectAsync();
return ftpClient;
});
}
/// <summary>
/// Gets a list of all the Mirrored file hashes that physically exist on the CDN (via FTP lookup)
/// </summary>
/// <returns></returns>
public async Task<HashSet<Hash>> GetHashesOnCDN()
{
using var ftpClient = await GetClient();
var serverFiles = (await ftpClient.GetNameListingAsync("\\"));
return serverFiles
.Select(f => ((RelativePath)f).FileName)
.Select(l =>
{
try
{
return Hash.FromHex((string)l);
}
catch (Exception) { return default; }
})
.Where(h => h != default)
.ToHashSet();
}
public async Task DeleteOldMirrorFiles()
{
var existingHashes = await GetHashesOnCDN();
var fromSql = await _sql.GetAllMirroredHashes();
foreach (var (hash, _) in fromSql.Where(s => s.Value))
{
Utils.Log($"Removing {hash} from SQL it's no longer in the CDN");
if (!existingHashes.Contains(hash))
await _sql.DeleteMirroredFile(hash);
}
var toDelete = existingHashes.Where(h => !fromSql.ContainsKey(h)).ToArray();
using var client = await GetClient();
foreach (var hash in toDelete)
{
await _discord.Send(Channel.Spam,
new DiscordMessage {Content = $"Removing mirrored file {hash}, as it's no longer in sql"});
Utils.Log($"Removing {hash} from the CDN it's no longer in SQL");
await client.DeleteDirectoryAsync(hash.ToHex());
}
}
}
}

161
Wabbajack.Server/Services/ModListDownloader.cs
View File

@ -1,161 +0,0 @@
using System;
using System.IO.Compression;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class ModListDownloader : AbstractService<ModListDownloader, int>
{
private ArchiveMaintainer _maintainer;
private SqlService _sql;
private DiscordWebHook _discord;
public ModListDownloader(ILogger<ModListDownloader> logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql, DiscordWebHook discord, QuickSync quickSync)
: base(logger, settings, quickSync, TimeSpan.FromMinutes(1))
{
_logger = logger;
_settings = settings;
_maintainer = maintainer;
_sql = sql;
_discord = discord;
}
public override async Task<int> Execute()
{
int downloaded = 0;
var lists = (await ModlistMetadata.LoadFromGithub())
.Concat(await ModlistMetadata.LoadUnlistedFromGithub()).ToList();
foreach (var list in lists)
{
try
{
ReportStarting(list.Links.MachineURL);
if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
continue;
if (!_maintainer.HaveArchive(list.DownloadMetadata!.Hash))
{
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
var tf = new TempFile();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
if (state == null)
{
_logger.Log(LogLevel.Error,
$"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}");
continue;
}
downloaded += 1;
await state.Download(new Archive(state) {Name = $"{list.Links.MachineURL}.wabbajack"}, tf.Path);
var hash = await tf.Path.FileHashAsync();
if (hash != list.DownloadMetadata.Hash)
{
_logger.Log(LogLevel.Error,
$"Downloaded modlist {list.Links.MachineURL} {list.DownloadMetadata.Hash} didn't match metadata hash of {hash}");
await _sql.IngestModList(list.DownloadMetadata.Hash, list, new ModList(), true);
continue;
}
await _maintainer.Ingest(tf.Path);
}
_maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath);
ModList modlist;
await using (var fs = await modlistPath.OpenRead())
using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
await using (var entry = zip.GetEntry("modlist")?.Open())
{
if (entry == null)
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
try
{
modlist = entry.FromJson<ModList>();
}
catch (JsonReaderException)
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
}
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Ingesting {list.Links.MachineURL} version {modlist.Version}"
});
await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist, false);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash} - {ex.Message}"
});
}
finally
{
ReportEnding(list.Links.MachineURL);
}
}
_logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
if (downloaded > 0)
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Downloaded {downloaded} new lists"});
var fc = await _sql.EnqueueModListFilesForIndexing();
_logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading");
if (fc > 0)
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Enqueing {fc} files for downloading"});
return downloaded;
}
}
public static class ModListDownloaderExtensions
{
public static void UseModListDownloader(this IApplicationBuilder b)
{
var poll = (ModListDownloader)b.ApplicationServices.GetService(typeof(ModListDownloader));
poll.Start();
}
}
}

110
Wabbajack.Server/Services/NexusKeyMaintainance.cs
View File

@ -1,110 +0,0 @@
using System;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Common.Exceptions;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.Services
{
public class NexusKeyMaintainance : AbstractService<NexusKeyMaintainance, int>
{
private SqlService _sql;
private string _selfKey;
public NexusKeyMaintainance(ILogger<NexusKeyMaintainance> logger, AppSettings settings, SqlService sql, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(4))
{
_sql = sql;
}
public async Task<NexusApiClient> GetClient()
{
var keys = await _sql.GetNexusApiKeysWithCounts(1500);
foreach (var key in keys.Where(k => k.Key != _selfKey))
{
try
{
var client = new TrackingClient(_sql, key);
if (await client.IsPremium())
return client;
_logger.LogWarning($"Purging non premium key");
await _sql.DeleteNexusAPIKey(key.Key);
continue;
}
catch (Exception ex)
{
Utils.Log($"Error getting tracking client: {ex}");
}
}
var bclient = await NexusApiClient.Get();
await bclient.GetUserStatus();
return bclient;
}
public override async Task<int> Execute()
{
_selfKey ??= await Utils.FromEncryptedJson<string>("nexusapikey");
var keys = await _sql.GetNexusApiKeysWithCounts(0);
_logger.Log(LogLevel.Information, $"Verifying {keys.Count} API Keys");
foreach (var key in keys)
{
try
{
var client = new TrackingClient(_sql, key);
var status = await client.GetUserStatus();
if (!status.is_premium)
{
await _sql.DeleteNexusAPIKey(key.Key);
continue;
}
var (daily, hourly) = await client.GetRemainingApiCalls();
await _sql.SetNexusAPIKey(key.Key, daily, hourly);
}
catch (HttpException ex)
{
_logger.Log(LogLevel.Warning, $"Nexus error, not purging API key : {ex.Message}");
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Update error, purging API key : {ex.Message}");
await _sql.DeleteNexusAPIKey(key.Key);
}
}
return keys.Count;
}
}
public class TrackingClient : NexusApiClient
{
private SqlService _sql;
public TrackingClient(SqlService sql, (string Key, int Daily, int Hourly) key) : base(key.Key)
{
_sql = sql;
DailyRemaining = key.Daily;
HourlyRemaining = key.Hourly;
}
protected override async Task UpdateRemaining(HttpResponseMessage response)
{
await base.UpdateRemaining(response);
try
{
var dailyRemaining = int.Parse(response.Headers.GetValues("x-rl-daily-remaining").First());
var hourlyRemaining = int.Parse(response.Headers.GetValues("x-rl-hourly-remaining").First());
await _sql.SetNexusAPIKey(ApiKey, dailyRemaining, hourlyRemaining);
}
catch (Exception)
{
}
}
}
}

66
Wabbajack.Server/Services/NexusPermissionsUpdater.cs
View File

@ -1,66 +0,0 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class NexusPermissionsUpdater : AbstractService<NexusPermissionsUpdater, int>
{
private DiscordWebHook _discord;
private SqlService _sql;
public NexusPermissionsUpdater(ILogger<NexusPermissionsUpdater> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromMinutes(5))
{
_discord = discord;
_sql = sql;
}
public override async Task<int> Execute()
{
await _sql.UpdateGameMetadata();
var data = await _sql.ModListArchives();
var nexusArchives = data.Select(a => a.State).OfType<NexusDownloader.State>().Select(d => (d.Game, d.ModID))
.Where(g => g.Game.MetaData().NexusGameId != 0)
.Distinct()
.ToList();
_logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods");
using var queue = new WorkQueue();
var prev = await _sql.GetHiddenNexusMods();
_logger.LogInformation($"Found {prev.Count} hidden nexus mods to check");
await prev.PMap(queue, async archive =>
{
var (game, modID) = archive.Key;
_logger.LogInformation($"Checking permissions for {game} {modID}");
var result = await HTMLInterface.GetUploadPermissions(game, modID);
await _sql.SetNexusPermission(game, modID, result);
if (archive.Value != result)
{
await _discord.Send(Channel.Ham,
new DiscordMessage {
Content = $"Permissions status of {game} {modID} was {archive.Value} is now {result}"
});
await _sql.PurgeNexusCache(modID);
await _quickSync.Notify<ListValidator>();
}
});
return 1;
}
}
}

165
Wabbajack.Server/Services/NexusPoll.cs
View File

@ -1,165 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class NexusPoll
{
private SqlService _sql;
private AppSettings _settings;
private GlobalInformation _globalInformation;
private ILogger<NexusPoll> _logger;
private NexusKeyMaintainance _keys;
public NexusPoll(ILogger<NexusPoll> logger, AppSettings settings, SqlService service, GlobalInformation globalInformation, NexusKeyMaintainance keys)
{
_sql = service;
_settings = settings;
_globalInformation = globalInformation;
_logger = logger;
_keys = keys;
}
public async Task UpdateNexusCacheRSS()
{
using var _ = _logger.BeginScope("Nexus Update via RSS");
_logger.Log(LogLevel.Information, "Starting");
var results = await NexusUpdatesFeeds.GetUpdates();
long updated = 0;
foreach (var result in results)
{
try
{
var purgedMods =
await _sql.DeleteNexusModFilesUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
var purgedFiles =
await _sql.DeleteNexusModInfosUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
var totalPurged = purgedFiles + purgedMods;
if (totalPurged > 0)
_logger.Log(LogLevel.Information, $"Purged {totalPurged} cache items {result.Game} {result.ModId} {result.TimeStamp}");
updated += totalPurged;
}
catch (Exception ex)
{
_logger.LogError(ex, $"Failed Nexus update for {result.Game} - {result.ModId} - {result.TimeStamp}");
}
}
if (updated > 0)
_logger.Log(LogLevel.Information, $"RSS Purged {updated} nexus cache entries");
_globalInformation.LastNexusSyncUTC = DateTime.UtcNow;
}
public async Task UpdateNexusCacheAPI()
{
using var _ = _logger.BeginScope("Nexus Update via API");
_logger.Log(LogLevel.Information, "Starting Nexus Update via API");
var api = await _keys.GetClient();
var gameTasks = GameRegistry.Games.Values
.Where(game => game.NexusName != null)
.Select(async game =>
{
var mods = await api.Get<List<NexusUpdateEntry>>(
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m");
return (game, mods);
})
.Select(async rTask =>
{
var (game, mods) = await rTask;
return mods.Select(mod => new { game = game, mod = mod });
}).ToList();
_logger.Log(LogLevel.Information, $"Getting update list for {gameTasks.Count} games");
var purge = (await Task.WhenAll(gameTasks))
.SelectMany(i => i)
.ToList();
_logger.Log(LogLevel.Information, $"Found {purge.Count} updated mods in the last month");
using var queue = new WorkQueue();
var collected = purge.Select(d =>
{
var a = d.mod.LatestFileUpdate.AsUnixTime();
// Mod activity could hide files
var b = d.mod.LastestModActivity.AsUnixTime();
return new {Game = d.game.Game, Date = (a > b) ? a : b, ModId = d.mod.ModId};
});
var purged = await collected.PMap(queue, async t =>
{
long purgeCount = 0;
purgeCount += await _sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date);
purgeCount += await _sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date);
return purgeCount;
});
_logger.Log(LogLevel.Information, $"Purged {purged.Sum()} cache entries");
_globalInformation.LastNexusSyncUTC = DateTime.UtcNow;
}
public void Start()
{
if (!_settings.RunBackEndJobs) return;
/*
Task.Run(async () =>
{
while (true)
{
try
{
await UpdateNexusCacheRSS();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error polling from Nexus");
}
await Task.Delay(_globalInformation.NexusRSSPollRate);
}
});
*/
Task.Run(async () =>
{
while (true)
{
try
{
await UpdateNexusCacheAPI();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting API feed from Nexus");
}
await Task.Delay(_globalInformation.NexusAPIPollRate);
}
});
}
}
public static class NexusPollExtensions
{
public static void UseNexusPoll(this IApplicationBuilder b)
{
var poll = (NexusPoll)b.ApplicationServices.GetService(typeof(NexusPoll));
poll.Start();
}
}
}

85
Wabbajack.Server/Services/NonNexusDownloadValidator.cs
View File

@ -1,85 +0,0 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Dapper;
using Microsoft.Extensions.Logging;
using Splat;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DataLayer;
using LogLevel = Microsoft.Extensions.Logging.LogLevel;
namespace Wabbajack.Server.Services
{
public class NonNexusDownloadValidator : AbstractService<NonNexusDownloadValidator, int>
{
private SqlService _sql;
public NonNexusDownloadValidator(ILogger<NonNexusDownloadValidator> logger, AppSettings settings, SqlService sql, QuickSync quickSync)
: base(logger, settings, quickSync, TimeSpan.FromHours(2))
{
_sql = sql;
}
public override async Task<int> Execute()
{
var archives = await _sql.GetNonNexusModlistArchives();
_logger.Log(LogLevel.Information, $"Validating {archives.Count} non-Nexus archives");
using var queue = new WorkQueue(10);
await DownloadDispatcher.PrepareAll(archives.Select(a => a.State));
var random = new Random();
var results = await archives.PMap(queue, async archive =>
{
try
{
await Task.Delay(random.Next(1000, 5000));
var token = new CancellationTokenSource();
token.CancelAfter(TimeSpan.FromMinutes(10));
ReportStarting(archive.State.PrimaryKeyString);
bool isValid = false;
switch (archive.State)
{
//case WabbajackCDNDownloader.State _:
//case GoogleDriveDownloader.State _: // Let's try validating Google again 2/10/2021
case GameFileSourceDownloader.State _:
isValid = true;
break;
case ManualDownloader.State _:
case ModDBDownloader.State _:
case HTTPDownloader.State h when h.Url.StartsWith("https://wabbajack"):
isValid = true;
break;
default:
isValid = await archive.State.Verify(archive, token.Token);
break;
}
return (Archive: archive, IsValid: isValid);
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error for {archive.Name} {archive.State.PrimaryKeyString} {ex}");
return (Archive: archive, IsValid: false);
}
finally
{
ReportEnding(archive.State.PrimaryKeyString);
}
});
await _sql.UpdateNonNexusModlistArchivesStatus(results);
var failed = results.Count(r => !r.IsValid);
var passed = results.Count() - failed;
foreach(var (archive, _) in results.Where(f => !f.IsValid))
_logger.Log(LogLevel.Warning, $"Validation failed for {archive.Name} from {archive.State.PrimaryKeyString}");
_logger.Log(LogLevel.Information, $"Non-nexus validation completed {failed} out of {passed} failed");
return failed;
}
}
}

242
Wabbajack.Server/Services/PatchBuilder.cs
View File

@ -1,242 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
using FluentFTP.Helpers;
using Microsoft.Extensions.Logging;
using Splat;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using LogLevel = Microsoft.Extensions.Logging.LogLevel;
namespace Wabbajack.Server.Services
{
public class PatchBuilder : AbstractService<PatchBuilder, int>
{
private DiscordWebHook _discordWebHook;
private SqlService _sql;
private ArchiveMaintainer _maintainer;
public PatchBuilder(ILogger<PatchBuilder> logger, SqlService sql, AppSettings settings, ArchiveMaintainer maintainer,
DiscordWebHook discordWebHook, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromMinutes(1))
{
_discordWebHook = discordWebHook;
_sql = sql;
_maintainer = maintainer;
}
public bool NoCleaning { get; set; }
public override async Task<int> Execute()
{
int count = 0;
while (true)
{
count++;
var patch = await _sql.GetPendingPatch();
if (patch == default) break;
try
{
_logger.LogInformation(
$"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}");
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
if (patch.Src.Archive.Hash == patch.Dest.Archive.Hash && patch.Src.Archive.State.PrimaryKeyString == patch.Dest.Archive.State.PrimaryKeyString)
{
await patch.Fail(_sql, "Hashes match");
continue;
}
if (patch.Src.Archive.Size > 2_500_000_000 || patch.Dest.Archive.Size > 2_500_000_000)
{
await patch.Fail(_sql, "Too large to patch");
continue;
}
_maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath);
_maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath);
await using var sigFile = new TempFile();
await using var patchFile = new TempFile();
await using var srcStream = await srcPath.OpenShared();
await using var destStream = await destPath.OpenShared();
await using var sigStream = await sigFile.Path.Create();
await using var patchOutput = await patchFile.Path.Create();
OctoDiff.Create(destStream, srcStream, sigStream, patchOutput, new OctoDiff.ProgressReporter(TimeSpan.FromSeconds(1), (s, p) => _logger.LogInformation($"Patch Builder: {p} {s}")));
await patchOutput.DisposeAsync();
var size = patchFile.Path.Size;
await UploadToCDN(patchFile.Path, PatchName(patch));
await patch.Finish(_sql, size);
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Built {size.ToFileSizeString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
}
catch (Exception ex)
{
_logger.LogError(ex, "Error while building patch");
await patch.Fail(_sql, ex.ToString());
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Failure building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
}
}
if (count > 0)
{
// Notify the List Validator that we may have more patches
await _quickSync.Notify<ListValidator>();
}
if (!NoCleaning)
await CleanupOldPatches();
return count;
}
private static string PatchName(Patch patch)
{
return PatchName(patch.Src.Archive.Hash, patch.Dest.Archive.Hash);
}
private static string PatchName(Hash oldHash, Hash newHash)
{
return $"{oldHash.ToHex()}_{newHash.ToHex()}";
}
private async Task CleanupOldPatches()
{
var patches = await _sql.GetOldPatches();
using var client = await GetBunnyCdnFtpClient();
foreach (var patch in patches)
{
_logger.LogInformation($"Cleaning patch {patch.Src.Archive.Hash} -> {patch.Dest.Archive.Hash}");
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Removing {patch.PatchSize.FileSizeToString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString} due it no longer being required by curated lists"
});
if (!await DeleteFromCDN(client, PatchName(patch)))
{
_logger.LogWarning($"Patch file didn't exist {PatchName(patch)}");
}
await _sql.DeletePatch(patch);
var pendingPatch = await _sql.GetPendingPatch();
if (pendingPatch != default) break;
}
var files = await client.GetListingAsync($"\\");
_logger.LogInformation($"Found {files.Length} on the CDN");
var sqlFiles = await _sql.AllPatchHashes();
_logger.LogInformation($"Found {sqlFiles.Count} in SQL");
HashSet<(Hash, Hash)> NamesToPairs(IEnumerable<FtpListItem> ftpFiles)
{
return ftpFiles.Select(f => f.Name).Where(f => f.Contains("_")).Select(p =>
{
try
{
var lst = p.Split("_", StringSplitOptions.RemoveEmptyEntries).Select(Hash.FromHex).ToArray();
return (lst[0], lst[1]);
}
catch (ArgumentException)
{
return default;
}
catch (FormatException)
{
return default;
}
}).Where(f => f != default).ToHashSet();
}
var oldHashPairs = NamesToPairs(files.Where(f => DateTime.UtcNow - f.Modified > TimeSpan.FromDays(2)));
foreach (var (oldHash, newHash) in oldHashPairs.Where(o => !sqlFiles.Contains(o)))
{
_logger.LogInformation($"Removing CDN File entry for {oldHash} -> {newHash} it's not SQL");
await client.DeleteFileAsync(PatchName(oldHash, newHash));
}
var hashPairs = NamesToPairs(files);
foreach (var sqlFile in sqlFiles.Where(s => !hashPairs.Contains(s)))
{
_logger.LogInformation($"Removing SQL File entry for {sqlFile.Item1} -> {sqlFile.Item2} it's not on the CDN");
await _sql.DeletePatchesForHashPair(sqlFile);
}
}
private async Task UploadToCDN(AbsolutePath patchFile, string patchName)
{
for (var times = 0; times < 5; times ++)
{
try
{
_logger.Log(LogLevel.Information,
$"Uploading {patchFile.Size.ToFileSizeString()} patch file to CDN {patchName}");
using var client = await GetBunnyCdnFtpClient();
await client.UploadFileAsync((string)patchFile, patchName, FtpRemoteExists.Overwrite);
return;
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error uploading {patchFile} to CDN");
}
}
_logger.Log(LogLevel.Error, $"Couldn't upload {patchFile} to {patchName}");
}
private async Task<bool> DeleteFromCDN(FtpClient client, string patchName)
{
if (!await client.FileExistsAsync(patchName))
return false;
await client.DeleteFileAsync(patchName);
return true;
}
private async Task<FtpClient> GetBunnyCdnFtpClient()
{
var info = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches);
var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
await client.ConnectAsync();
return client;
}
}
}

82
Wabbajack.Server/Services/QuickSync.cs
View File

@ -1,82 +0,0 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
namespace Wabbajack.Server.Services
{
public class QuickSync
{
private Dictionary<Type, CancellationTokenSource> _syncs = new Dictionary<Type, CancellationTokenSource>();
private Dictionary<Type, IReportingService> _services = new Dictionary<Type, IReportingService>();
private AsyncLock _lock = new AsyncLock();
private ILogger<QuickSync> _logger;
public QuickSync(ILogger<QuickSync> logger)
{
_logger = logger;
}
public async Task<Dictionary<Type, (TimeSpan Delay, TimeSpan LastRunTime, (String, DateTime)[] ActiveWork)>> Report()
{
using var _ = await _lock.WaitAsync();
return _services.ToDictionary(s => s.Key,
s => (s.Value.Delay, DateTime.UtcNow - s.Value.LastEnd, s.Value.ActiveWorkStatus));
}
public async Task Register<T>(T service)
where T : IReportingService
{
using var _ = await _lock.WaitAsync();
_services[service.GetType()] = service;
}
public async Task<CancellationToken> GetToken<T>()
{
using var _ = await _lock.WaitAsync();
if (_syncs.TryGetValue(typeof(T), out var result))
{
return result.Token;
}
var token = new CancellationTokenSource();
_syncs[typeof(T)] = token;
return token.Token;
}
public async Task ResetToken<T>()
{
using var _ = await _lock.WaitAsync();
if (_syncs.TryGetValue(typeof(T), out var ct))
{
ct.Cancel();
}
_syncs[typeof(T)] = new CancellationTokenSource();
}
public async Task Notify<T>()
{
_logger.LogInformation($"Quicksync {typeof(T).Name}");
// Needs debugging
using var _ = await _lock.WaitAsync();
if (_syncs.TryGetValue(typeof(T), out var ct))
{
ct.Cancel();
}
}
public async Task Notify(Type t)
{
_logger.LogInformation($"Quicksync {t.Name}");
// Needs debugging
using var _ = await _lock.WaitAsync();
if (_syncs.TryGetValue(t, out var ct))
{
ct.Cancel();
}
}
}
}

33
Wabbajack.Server/Services/Watchdog.cs
View File

@ -1,33 +0,0 @@
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class Watchdog : AbstractService<Watchdog, int>
{
private DiscordWebHook _discord;
public Watchdog(ILogger<Watchdog> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discordWebHook) : base(logger, settings, quickSync, TimeSpan.FromMinutes(5))
{
_discord = discordWebHook;
}
public override async Task<int> Execute()
{
var report = await _quickSync.Report();
foreach (var service in report)
{
if (service.Value.LastRunTime != default && service.Value.LastRunTime >= service.Value.Delay * 4)
{
await _discord.Send(Channel.Spam,
new DiscordMessage {Content = $"Service {service.Key.Name} has missed it's scheduled execution window. \n Current work: \n {string.Join("\n", service.Value.ActiveWork)}"});
}
}
return report.Count;
}
}
}

189
Wabbajack.Server/Startup.cs
View File

@ -1,189 +0,0 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http.Features;
using Microsoft.AspNetCore.ResponseCompression;
using Microsoft.AspNetCore.StaticFiles;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.FileProviders;
using Microsoft.Extensions.Hosting;
using Newtonsoft.Json;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.LibCefHelpers;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
namespace Wabbajack.Server
{
public class TestStartup : Startup
{
public TestStartup(IConfiguration configuration) : base(configuration)
{
}
}
public class Startup
{
public Startup(IConfiguration configuration)
{
Configuration = configuration;
}
public IConfiguration Configuration { get; }
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
Consts.UseNetworkWorkaroundMode = true;
Helpers.Init();
/*services.AddSwaggerGen(c =>
{
c.SwaggerDoc("v1", new OpenApiInfo {Title = "Wabbajack Build API", Version = "v1"});
});*/
services.AddAuthentication(options =>
{
options.DefaultAuthenticateScheme = ApiKeyAuthenticationOptions.DefaultScheme;
options.DefaultChallengeScheme = ApiKeyAuthenticationOptions.DefaultScheme;
})
.AddApiKeySupport(options => {});
services.Configure<FormOptions>(x =>
{
x.ValueLengthLimit = int.MaxValue;
x.MultipartBodyLengthLimit = int.MaxValue;
});
services.AddSingleton<AppSettings>();
services.AddSingleton<QuickSync>();
services.AddSingleton<SqlService>();
services.AddSingleton<GlobalInformation>();
services.AddSingleton<NexusPoll>();
services.AddSingleton<ArchiveMaintainer>();
services.AddSingleton<ModListDownloader>();
services.AddSingleton<NonNexusDownloadValidator>();
services.AddSingleton<ListValidator>();
services.AddSingleton<ArchiveDownloader>();
services.AddSingleton<DiscordWebHook>();
services.AddSingleton<NexusKeyMaintainance>();
services.AddSingleton<PatchBuilder>();
services.AddSingleton<CDNMirrorList>();
services.AddSingleton<NexusPermissionsUpdater>();
services.AddSingleton<MirrorUploader>();
services.AddSingleton<MirrorQueueService>();
services.AddSingleton<Watchdog>();
services.AddSingleton<DiscordFrontend>();
services.AddSingleton<AuthoredFilesCleanup>();
services.AddSingleton<MetricsKeyCache>();
services.AddResponseCompression(options =>
{
options.Providers.Add<BrotliCompressionProvider>();
options.Providers.Add<GzipCompressionProvider>();
options.MimeTypes = new[] {"application/json"};
});
services.AddMvc();
services.AddControllers()
.AddNewtonsoftJson(o =>
{
o.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore;
});
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
if (!(this is TestStartup))
app.UseHttpsRedirection();
app.UseDeveloperExceptionPage();
var provider = new FileExtensionContentTypeProvider();
provider.Mappings[".rar"] = "application/x-rar-compressed";
provider.Mappings[".7z"] = "application/x-7z-compressed";
provider.Mappings[".zip"] = "application/zip";
provider.Mappings[".wabbajack"] = "application/zip";
app.UseStaticFiles();
/*
app.UseSwagger();
app.UseSwaggerUI(c =>
{
c.SwaggerEndpoint("/swagger/v1/swagger.json", "Wabbajack Build API");
c.RoutePrefix = string.Empty;
}); */
app.UseRouting();
app.UseAuthentication();
app.UseAuthorization();
app.UseNexusPoll();
//app.UseArchiveMaintainer();
//app.UseModListDownloader();
app.UseResponseCompression();
//app.UseService<NonNexusDownloadValidator>();
//app.UseService<ListValidator>();
//app.UseService<ArchiveDownloader>();
app.UseService<DiscordWebHook>();
//app.UseService<NexusKeyMaintainance>();
//app.UseService<PatchBuilder>();
//app.UseService<CDNMirrorList>();
//app.UseService<NexusPermissionsUpdater>();
//app.UseService<MirrorUploader>();
//app.UseService<MirrorQueueService>();
app.UseService<Watchdog>();
app.UseService<DiscordFrontend>();
// Don't enable Author Files Cleanup
//app.UseService<AuthoredFilesCleanup>();
app.UseService<MetricsKeyCache>();
app.Use(next =>
{
return async context =>
{
var stopWatch = new Stopwatch();
stopWatch.Start();
context.Response.OnStarting(() =>
{
stopWatch.Stop();
var headers = context.Response.Headers;
headers.Add("Access-Control-Allow-Origin", "*");
headers.Add("Access-Control-Allow-Methods", "POST, GET");
headers.Add("Access-Control-Allow-Headers", "Accept, Origin, Content-type");
headers.Add("X-ResponseTime-Ms", stopWatch.ElapsedMilliseconds.ToString());
if (!headers.ContainsKey("Cache-Control"))
headers.Add("Cache-Control", "no-cache");
return Task.CompletedTask;
});
await next(context);
};
});
app.UseFileServer(new FileServerOptions
{
FileProvider = new PhysicalFileProvider(
Path.Combine(Directory.GetCurrentDirectory(), "public")),
StaticFileOptions = {ServeUnknownFileTypes = true},
});
app.UseEndpoints(endpoints =>
{
endpoints.MapControllers();
});
}
}
}

62
Wabbajack.Server/Wabbajack.Server.csproj
View File

@ -1,62 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net5.0-windows</TargetFramework>
<AssemblyVersion>2.5.3.9</AssemblyVersion>
<FileVersion>2.5.3.9</FileVersion>
<Copyright>Copyright © 2019-2021</Copyright>
<Description>Wabbajack Server</Description>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>
<AssemblyName>Wabbajack.Server</AssemblyName>
<RootNamespace>Wabbajack.Server</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dapper" Version="2.0.90" />
<PackageReference Include="Discord.Net.WebSocket" Version="2.4.0" />
<PackageReference Include="FluentFTP" Version="34.0.1" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="5.0.8" />
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Nettle" Version="1.3.0" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
<ProjectReference Include="..\Wabbajack.Lib\Wabbajack.Lib.csproj" />
</ItemGroup>
<ItemGroup>
<None Update="public\WABBAJACK_TEST_FILE.txt">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="appsettings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="public\metrics.html">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Controllers\Templates\AuthorControls.html">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Controllers\Templates\AuthorControls.html">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<None Remove="sheo_quotes.txt" />
<EmbeddedResource Include="sheo_quotes.txt" />
<None Remove="Controllers\Templates\TotalListTemplate.html" />
<EmbeddedResource Include="Controllers\Templates\TotalListTemplate.html" />
</ItemGroup>
<ItemGroup>
<Compile Remove="Controllers\UploadedFiles.cs" />
</ItemGroup>
</Project>

21
Wabbajack.Server/appsettings.json
View File

@ -1,21 +0,0 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"WabbajackSettings": {
"DownloadDir": "c:\\tmp\\downloads",
"ArchiveDir": "w:\\archives",
"TempFolder": "c:\\tmp",
"JobRunner": true,
"JobScheduler": false,
"RunFrontEndJobs": true,
"RunBackEndJobs": false,
"BunnyCDN_StorageZone": "wabbajacktest",
"SQLConnection": "Data Source=.\\SQLEXPRESS;Integrated Security=True;Initial Catalog=wabbajack_prod;MultipleActiveResultSets=true"
},
"AllowedHosts": "*"
}

1
Wabbajack.Server/public/WABBAJACK_TEST_FILE.txt
View File

@ -1 +0,0 @@
Cheese for Everyone!

113
Wabbajack.Server/public/metrics.html
View File

@ -1,113 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Wabbajack Metrics</title>
<script
src="https://code.jquery.com/jquery-3.5.1.min.js"
integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0="
crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/chart.js@2.8.0"></script>
<script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-colorschemes"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/underscore.js/1.9.1/underscore-min.js"></script>
</head>
<body>
<h2>Finished Install Counts</h2>
<canvas id="finished_install_count" width="800" height="600"></canvas>
<hr/>
<h2>Begin Download</h2>
<canvas id="begin_download_chart" width="800" height="600"></canvas>
<hr/>
<h2>Begin Install</h2>
<canvas id="begin_install_chart" width="800" height="600"></canvas>
<hr/>
<h2>Finished Install</h2>
<canvas id="finished_install_chart" width="800" height="600"></canvas>
<hr/>
<h2>Started Wabbajack</h2>
<canvas id="started_wabbajack_chart" width="800" height="600"></canvas>
<hr/>
<h2>Exceptions</h2>
<canvas id="exceptions_chart" width="800" height="600"></canvas>
<hr/>
<script>
var getReport = function(subject, callback) {
$.getJSON("/metrics/report/"+subject, callback)
}
var makeChart = function(ele, group) {
var result_fn = function (data) {
var data = _.filter(data, series => _.some(series.values, v => v > 1));
var labels = _.uniq(_.flatten(_.map(data, series => series.labels)));
var datasets = _.map(data, series => {
return {
label: series.seriesName,
fill: false,
data: _.last(series.values, 90)
}});
var ctx = document.getElementById(ele).getContext('2d');
var chart = new Chart(ctx, {
// The type of chart we want to create
type: 'bar',
// The data for our dataset
data: {
labels: _.last(labels, 90),
datasets: datasets},
// Configuration options go here
options: {scales: {xAxes: [{stacked:true}], yAxes: [{stacked:true}]}}
});
};
getReport(group, result_fn);
};
var makePieChart = function(ele, group) {
var result_fn = function (data) {
var data = _.filter(data, series => _.some(series.values, v => v > 2));
var labels = _.map(data, series => series.seriesName);
var datasets = {data : _.map(data, series => {
return _.reduce(series.values, (x, y) => x + y, 0)})};
console.log(datasets);
console.log(labels);
var ctx = document.getElementById(ele).getContext('2d');
var chart = new Chart(ctx, {
// The type of chart we want to create
type: 'pie',
// The data for our dataset
data: {
labels: labels,
datasets: [datasets]},
// Configuration options go here
options: {}
});
};
getReport(group, result_fn)
};
makeChart("begin_download_chart", "downloading");
makeChart("begin_install_chart", "begin_install");
makeChart("finished_install_chart", "finish_install");
makeChart("started_wabbajack_chart", "started_wabbajack");
makeChart("exceptions_chart", "Exception");
makePieChart("finished_install_count", "finish_install");
</script>
</body>
</html>

55
Wabbajack.Server/sheo_quotes.txt
View File

@ -1,55 +0,0 @@
I see you have completed my little errand. Well done. Perhaps you’ve gotten a taste of madness aswell? Do not believe madness to be a curse, mortal. For some it is the greatest of blessings. A bitter mercy perhaps, but mercy non the less. Give me the Fork of Horripilation, I believe I have something more suitable for your needs. Go now. Remember what you have seen.
Use the fork wisely, mortal. Few have wield to have not come away changed. Use the fork to strike a deathblow to the giant Bullnetch that resides near the hermit. Do this, return the Fork of Horripilation to me, and Sheogorath will reward you well.
What is it, mortal? Have you come to be of the service to Sheogorath? That in and of itself speaks toward your madness. This pleases me. Fetch the Fork of Horripliation from the mad hermit near Ald Redaynia. Take care with him. He’s not the most... stable man.
Unworthy, unworthy, unworthy! Useless mortal meat. Walking bag of dung!
Bring me a champion! Rend the flesh of my foes! A mortal champion to wade through the entrails of my enemies!
Really, do come in. It’s lovely in the Isles right now. Perfect time for a visit.
Greetings! Salutations! Welcome! Now go away. Leave. Run. Or die.
Isn't that a hoot? I love it, myself. Best part of being a Daedric Prince, really. Go ahead, try it again. He loves it!
Marvellous, marvellous! Self-immolation is a wonderful thing, isn't it? But now that we've had our fun, off to the Sacellum with you.
I suppose an introduction is in order. I'm Sheogorath, Prince of Madness. And other things. I'm not talking about them.
You should be off like the wind, solving problems and doing good deeds!
Time. Time is an artificial construct. An arbitrary system based on the idea that events occur in a linear direction at all times.
Always forward, never back. Is the concept of time correct? Is time relevant? It matters not. One way or another, I fear that our time has run out.
A new Gatekeeper! Excellent. We might be onto something with you, after all. That should keep out the stragglers.
A little busy here! I'm trying to decide what to have for dinner. Oh, how I love eating. One of my favorite things to do.
It's Jyggalag's time, and not a good time at all. You're going to help me stop it. First, though, you need to get your feet wet.
Another Daedric Prince. Not a nice one. I don't think ANY of the other Princes like him, actually. I mean, Malacath is more popular at parties.
The Daedric Prince of Order. Or biscuits... No. Order. And not in a good way. Bleak. Colorless. Dead. Boring, boring, boring.
The Greymarch comes, and Jyggalag walks. Or runs. Never skips, sidles, or struts. Mostly, he just destroys everything around him.
Once you understand what My Realm is, you might understand why it's important to keep it intact.
Two halves, two rulers, two places. Meet and greet. Do what they will, so you know what they're about.
Ask? ASK? I don't ask. I tell. This is My Realm, remember? My creation, My place, My rules.
Wonderful! Time for a celebration... Cheese for everyone!
Makes all of my subjects uneasy. Tense. Homicidal. Some of them, at least. We need to get that Torch relit, before the place falls apart.
You're going to stop the Greymarch by becoming Me. Or a version of Me. You'll be powerful. Powerful enough to stop Jyggalag.
You know what would be a good sign? "Free Sweetrolls!" Who wouldn't like that?
You'll be my champion. You'll grow powerful. You'll grow to be me. Prince of Madness, a new Sheogorath. Or you'll die trying. I love that about you.
Oh, don't forget to make use of dear Haskill. Between you and me, if he's not summoned three or four times a day, I don't think he feels appreciated.
I hate indecision! Or maybe I don't. Make up your mind, or I'll have your skin made into a hat -- one of those arrowcatchers. I love those hats!
So, which is it? What will it be? Mania? Dementia? The suspense is killing me. Or you, if I have to keep waiting.
Except where the backbone is an actual backbone. Ever been to Malacath's realm...? Nasty stuff. But, back to the business at hand.
Happens every time. The Greymarch starts, Order appears, and I become Jyggalag and wipe out My whole Realm.
Flee while you can, mortal. When we next meet I will not know you, and I will slay you like the others.
Ah... New Sheoth. My home away from places that aren't my home. The current location is much better than some of the prior ones. Don't you think?
The Isles, the Isles. A wonderful place! Except when it's horrible. Then it's horribly wonderful. Good for a visit. Or for an eternity.
Time to save the Realm! Rescue the damsel! Slay the beast! Or die trying. Your help is required.
Daedra are the embodiment of change. Change and permanency. I'm no different, except in the ways that I am.
Was it Molag? No, no... Little Tim, the toymaker's son? The ghost of King Lysandus? Or was it... Yes! Stanley, that talking grapefruit from Passwall.
Reeaaaallllyyyy?
Well? Spit it out, mortal. I haven't got an eternity! Oh, wait! I do.
I am a part of you, little mortal. I am a shadow in your subconscious, a blemish on your fragile little psyche. You know me. You just don't know it.
Sheogorath, Daedric Prince of Madness. At your service.
Yaaawwwwnn....
Oh, pardon me. Were you saying something? I do apologize, it's just that I find myself suddenly and irrevocably...
Bored!
I mean, really. Here you stand, before Sheogorath himself, Daedric Prince of Madness, and all you deem fit to do is... deliver a message? How sad.
Now you. You can call me Ann Marie.
Oh... lovely. Now all my dear Pelagius has to worry about are the several hundred legitimate threats...
Ah, wonderful, wonderful! Why waste all that hatred on yourself when it can so easily be directed at others!
Mortal? Insufferable.
Yes, yes, you're entirely brilliant. Conquering madness and all that. Blah blah blah.
Ah, so now my dear Pelagius can hate himself for being legitimately afraid of things that actually threaten his existence...
Conquering paranoia should be a snap after that ordeal, hmm?
Welcome to the deceptively verdant mind of the Emperor Pelagius III. That's right! You're in the head of a dead, homicidally insane monarch.
The Wabbajack! Huh? Huh? Didn't see that coming, did you?

2
Wabbajack.Test/GitHubTests.cs
View File

@ -36,7 +36,7 @@ namespace Wabbajack.Test
await client.UpdateList("ci_tester", update);
var updated = await client.GetData(Client.List.CI);
var lst = updated.Lists.FirstOrDefault(l => l.Links.MachineURL == "ci_test");
var lst = updated.Lists.FirstOrDefault(l => l.NamespacedName == "ci_test");
var newMeta = lst!.DownloadMetadata!;
Assert.Equal(meta.Hash, newMeta.Hash);
Assert.Equal(meta.Size, newMeta.Size);

20
Wabbajack.sln
View File

@ -40,10 +40,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Common.Test", "Wa
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.App.Test", "Wabbajack.App.Test\Wabbajack.App.Test.csproj", "{44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Server", "Wabbajack.Server\Wabbajack.Server.csproj", "{3E11B700-8405-433D-BF47-6C356087A7C2}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Server.Test", "Wabbajack.Server.Test\Wabbajack.Server.Test.csproj", "{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.ImageHashing", "Wabbajack.ImageHashing\Wabbajack.ImageHashing.csproj", "{0C893E5F-1FD8-463E-AC3F-D020213ACD3B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.ImageHashing.Test", "Wabbajack.ImageHashing.Test\Wabbajack.ImageHashing.Test.csproj", "{96892D68-7400-4297-864C-39D0673775DE}"
@ -150,22 +146,6 @@ Global
{44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}.Debug|x64.Build.0 = Debug|x64
{44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}.Release|Any CPU.ActiveCfg = Release|x64
{44E30B97-D4A8-40A6-81D5-5CAB1F3D45CB}.Release|x64.ActiveCfg = Release|x64
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|x64.ActiveCfg = Debug|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|x64.Build.0 = Debug|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|Any CPU.Build.0 = Release|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|x64.ActiveCfg = Release|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|x64.Build.0 = Release|Any CPU
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Debug|x64.ActiveCfg = Debug|Any CPU
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Debug|x64.Build.0 = Debug|Any CPU
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|Any CPU.Build.0 = Release|Any CPU
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|x64.ActiveCfg = Release|Any CPU
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|x64.Build.0 = Release|Any CPU
{0C893E5F-1FD8-463E-AC3F-D020213ACD3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0C893E5F-1FD8-463E-AC3F-D020213ACD3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0C893E5F-1FD8-463E-AC3F-D020213ACD3B}.Debug|x64.ActiveCfg = Debug|Any CPU

14
Wabbajack/View Models/Gallery/ModListMetadataVM.cs
View File

@ -84,7 +84,7 @@ namespace Wabbajack
{
_parent = parent;
Metadata = metadata;
Location = LauncherUpdater.CommonFolder.Value.Combine("downloaded_mod_lists", Metadata.Links.MachineURL + (string)Consts.ModListExtension);
Location = LauncherUpdater.CommonFolder.Value.Combine("downloaded_mod_lists", Metadata.NamespacedName.Replace("/", "_@@_") + (string)Consts.ModListExtension);
ModListTagList = new List<ModListTag>();
Metadata.tags.ForEach(tag =>
@ -98,7 +98,7 @@ namespace Wabbajack
VersionText = "Modlist version : " + Metadata.Version;
IsBroken = metadata.ValidationSummary.HasFailures || metadata.ForceDown;
//https://www.wabbajack.org/#/modlists/info?machineURL=eldersouls
OpenWebsiteCommand = ReactiveCommand.Create(() => Utils.OpenWebsite(new Uri($"https://www.wabbajack.org/#/modlists/info?machineURL={Metadata.Links.MachineURL}")));
OpenWebsiteCommand = ReactiveCommand.Create(() => Utils.OpenWebsite(new Uri($"https://www.wabbajack.org/#/modlists/info?machineURL={Metadata.NamespacedName}")));
IsLoadingIdle = new Subject<bool>();
@ -108,7 +108,7 @@ namespace Wabbajack
IsLoadingIdle.OnNext(false);
try
{
var status = await ClientAPIEx.GetDetailedStatus(metadata.Links.MachineURL);
var status = await ClientAPIEx.GetDetailedStatus(metadata.NamespacedName);
var coll = _parent.MWVM.ModListContentsVM.Value.Status;
coll.Clear();
coll.AddRange(status.Archives);
@ -222,18 +222,18 @@ namespace Wabbajack
try
{
IsDownloading = true;
Utils.Log($"Starting Download of {Metadata.Links.MachineURL}");
Utils.Log($"Starting Download of {Metadata.NamespacedName}");
var downloader = DownloadDispatcher.ResolveArchive(Metadata.Links.Download);
var result = await downloader.Download(
new Archive(state: null!)
{
Name = Metadata.Title, Size = Metadata.DownloadMetadata?.Size ?? 0
}, Location);
Utils.Log($"Done downloading {Metadata.Links.MachineURL}");
Utils.Log($"Done downloading {Metadata.NamespacedName}");
// Want to rehash to current file, even if failed?
await Location.FileHashCachedAsync();
Utils.Log($"Done hashing {Metadata.Links.MachineURL}");
Utils.Log($"Done hashing {Metadata.NamespacedName}");
await Metadata.ToJsonAsync(Location.WithExtension(Consts.ModlistMetadataExtension));
@ -241,7 +241,7 @@ namespace Wabbajack
}
catch (Exception ex)
{
Utils.Error(ex, $"Error Downloading of {Metadata.Links.MachineURL}");
Utils.Error(ex, $"Error Downloading of {Metadata.NamespacedName}");
tcs.SetException(ex);
}
finally