mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
Server has been moved to a different repo
This commit is contained in:
Timothy Baldridge
parent
f080464c19
commit
0ee3917106
@ -1,169 +0,0 @@
|
||||
using System.Security.Claims;
|
||||
using System.Text.Encodings.Web;
|
||||
using Microsoft.AspNetCore.Authentication;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Wabbajack.DTOs.JsonConverters;
|
||||
using Wabbajack.Networking.GitHub;
|
||||
using Wabbajack.Networking.GitHub.DTOs;
|
||||
using Wabbajack.Server.DataModels;
|
||||
using Wabbajack.Server.DTOs;
|
||||
|
||||
namespace Wabbajack.BuildServer;
|
||||
|
||||
public class ApiKeyAuthenticationOptions : AuthenticationSchemeOptions
|
||||
{
|
||||
public const string DefaultScheme = "API Key";
|
||||
public string AuthenticationType = DefaultScheme;
|
||||
public string Scheme => DefaultScheme;
|
||||
}
|
||||
|
||||
public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthenticationOptions>
|
||||
{
|
||||
private const string ProblemDetailsContentType = "application/problem+json";
|
||||
public const string ApiKeyHeaderName = "X-Api-Key";
|
||||
private readonly DTOSerializer _dtos;
|
||||
private readonly AppSettings _settings;
|
||||
private readonly AuthorKeys _authorKeys;
|
||||
private readonly Metrics _metricsStore;
|
||||
private readonly TarLog _tarLog;
|
||||
private readonly Client _githubClient;
|
||||
private readonly MemoryCache _githubCache;
|
||||
|
||||
public ApiKeyAuthenticationHandler(
|
||||
IOptionsMonitor<ApiKeyAuthenticationOptions> options,
|
||||
AuthorKeys authorKeys,
|
||||
Client githubClient,
|
||||
ILoggerFactory logger,
|
||||
UrlEncoder encoder,
|
||||
ISystemClock clock,
|
||||
DTOSerializer dtos,
|
||||
Metrics metricsStore,
|
||||
TarLog tarlog,
|
||||
AppSettings settings) : base(options, logger, encoder, clock)
|
||||
{
|
||||
|
||||
_tarLog = tarlog;
|
||||
_metricsStore = metricsStore;
|
||||
_dtos = dtos;
|
||||
_authorKeys = authorKeys;
|
||||
_settings = settings;
|
||||
_githubClient = githubClient;
|
||||
_githubCache = new MemoryCache(new MemoryCacheOptions());
|
||||
}
|
||||
|
||||
protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
|
||||
{
|
||||
var metricsKey = Request.Headers[_settings.MetricsKeyHeader].FirstOrDefault();
|
||||
// Never needed this, disabled for now
|
||||
//await LogRequest(metricsKey);
|
||||
if (metricsKey != default)
|
||||
{
|
||||
if (await _tarLog.Contains(metricsKey))
|
||||
{
|
||||
await _metricsStore.Ingest(new Metric
|
||||
{
|
||||
Subject = metricsKey,
|
||||
Action = "tarlog",
|
||||
MetricsKey = metricsKey,
|
||||
UserAgent = Request.Headers.UserAgent,
|
||||
Ip = Request.HttpContext.Connection.RemoteIpAddress?.ToString() ?? ""
|
||||
});
|
||||
await Task.Delay(TimeSpan.FromSeconds(20));
|
||||
throw new Exception("Error, lipsum timeout of the cross distant cloud.");
|
||||
}
|
||||
}
|
||||
|
||||
var authorKey = Request.Headers[ApiKeyHeaderName].FirstOrDefault();
|
||||
|
||||
if (authorKey == null)
|
||||
Request.Cookies.TryGetValue(ApiKeyHeaderName, out authorKey);
|
||||
|
||||
|
||||
if (authorKey == null && metricsKey == null) return AuthenticateResult.NoResult();
|
||||
|
||||
if (authorKey != null)
|
||||
{
|
||||
|
||||
var owner = await _authorKeys.AuthorForKey(authorKey);
|
||||
if (owner == null)
|
||||
{
|
||||
var ghUser = await GetGithubUserInfo(authorKey);
|
||||
|
||||
if (ghUser == null)
|
||||
return AuthenticateResult.Fail("Invalid author key");
|
||||
|
||||
owner = "github/" + ghUser.Login;
|
||||
}
|
||||
|
||||
if (await _tarLog.Contains(owner))
|
||||
return AuthenticateResult.Fail("Banned author key");
|
||||
|
||||
var claims = new List<Claim> {new(ClaimTypes.Name, owner)};
|
||||
|
||||
claims.Add(new Claim(ClaimTypes.Role, "Author"));
|
||||
claims.Add(new Claim(ClaimTypes.Role, "User"));
|
||||
|
||||
var identity = new ClaimsIdentity(claims, Options.AuthenticationType);
|
||||
var identities = new List<ClaimsIdentity> {identity};
|
||||
var principal = new ClaimsPrincipal(identities);
|
||||
var ticket = new AuthenticationTicket(principal, Options.Scheme);
|
||||
|
||||
return AuthenticateResult.Success(ticket);
|
||||
}
|
||||
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(metricsKey))
|
||||
{
|
||||
var claims = new List<Claim> {new(ClaimTypes.Role, "User")};
|
||||
|
||||
|
||||
var identity = new ClaimsIdentity(claims, Options.AuthenticationType);
|
||||
var identities = new List<ClaimsIdentity> {identity};
|
||||
var principal = new ClaimsPrincipal(identities);
|
||||
var ticket = new AuthenticationTicket(principal, Options.Scheme);
|
||||
|
||||
return AuthenticateResult.Success(ticket);
|
||||
}
|
||||
|
||||
return AuthenticateResult.NoResult();
|
||||
}
|
||||
|
||||
protected async Task<UserInfo?> GetGithubUserInfo(string authToken)
|
||||
{
|
||||
if (_githubCache.TryGetValue<UserInfo>(authToken, out var value)) return value;
|
||||
|
||||
var info = await _githubClient.GetUserInfoFromPAT(authToken);
|
||||
if (info != null)
|
||||
_githubCache.Set(authToken, info,
|
||||
new MemoryCacheEntryOptions().SetSlidingExpiration(TimeSpan.FromHours(6)));
|
||||
|
||||
return info;
|
||||
}
|
||||
|
||||
protected override async Task HandleChallengeAsync(AuthenticationProperties properties)
|
||||
{
|
||||
Response.StatusCode = 401;
|
||||
Response.ContentType = ProblemDetailsContentType;
|
||||
await Response.WriteAsync("Unauthorized");
|
||||
}
|
||||
|
||||
protected override async Task HandleForbiddenAsync(AuthenticationProperties properties)
|
||||
{
|
||||
Response.StatusCode = 403;
|
||||
Response.ContentType = ProblemDetailsContentType;
|
||||
await Response.WriteAsync("forbidden");
|
||||
}
|
||||
}
|
||||
|
||||
public static class ApiKeyAuthorizationHandlerExtensions
|
||||
{
|
||||
public static AuthenticationBuilder AddApiKeySupport(this AuthenticationBuilder authenticationBuilder,
|
||||
Action<ApiKeyAuthenticationOptions> options)
|
||||
{
|
||||
return authenticationBuilder.AddScheme<ApiKeyAuthenticationOptions, ApiKeyAuthenticationHandler>(
|
||||
ApiKeyAuthenticationOptions.DefaultScheme, options);
|
||||
}
|
||||
}
|
||||
@ -1,61 +0,0 @@
|
||||
using Amazon.S3;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Wabbajack.Paths;
|
||||
|
||||
namespace Wabbajack.BuildServer;
|
||||
|
||||
public class AppSettings
|
||||
{
|
||||
public AppSettings(IConfiguration config)
|
||||
{
|
||||
config.Bind("WabbajackSettings", this);
|
||||
}
|
||||
public bool TestMode { get; set; }
|
||||
public bool RunBackendNexusRoutines { get; set; } = true;
|
||||
public string AuthorAPIKeyFile { get; set; }
|
||||
|
||||
public string TarKeyFile { get; set; }
|
||||
public string WabbajackBuildServerUri { get; set; } = "https://build.wabbajack.org/";
|
||||
public string MetricsKeyHeader { get; set; } = "x-metrics-key";
|
||||
public string TempFolder { get; set; }
|
||||
|
||||
public string ProxyFolder { get; set; }
|
||||
public AbsolutePath ProxyPath => (AbsolutePath) ProxyFolder;
|
||||
public AbsolutePath TempPath => (AbsolutePath) TempFolder;
|
||||
public string SpamWebHook { get; set; } = "";
|
||||
public string HamWebHook { get; set; } = "";
|
||||
|
||||
public string DiscordKey { get; set; }
|
||||
|
||||
public string PatchesFilesFolder { get; set; }
|
||||
public string MirrorFilesFolder { get; set; }
|
||||
public string NexusCacheFolder { get; set; }
|
||||
public string MetricsFolder { get; set; } = "";
|
||||
public string TarLogPath { get; set; }
|
||||
public string GitHubKey { get; set; } = "";
|
||||
|
||||
public CouchDBSetting CesiDB { get; set; }
|
||||
public CouchDBSetting MetricsDB { get; set; }
|
||||
|
||||
public S3Settings S3 { get; set; }
|
||||
}
|
||||
|
||||
public class S3Settings
|
||||
{
|
||||
public string AccessKey { get; set; }
|
||||
public string SecretKey { get; set; }
|
||||
public string ServiceUrl { get; set; }
|
||||
|
||||
public string AuthoredFilesBucket { get; set; }
|
||||
public string ProxyFilesBucket { get; set; }
|
||||
|
||||
public string AuthoredFilesBucketCache { get; set; }
|
||||
}
|
||||
|
||||
public class CouchDBSetting
|
||||
{
|
||||
public Uri Endpoint { get; set; }
|
||||
public string Database { get; set; }
|
||||
public string Username { get; set; }
|
||||
public string Password { get; set; }
|
||||
}
|
||||
@ -1,15 +0,0 @@
|
||||
namespace Wabbajack.Server;
|
||||
|
||||
public class Badge
|
||||
{
|
||||
public Badge(string _label, string _message)
|
||||
{
|
||||
label = _label;
|
||||
message = _message;
|
||||
}
|
||||
|
||||
public int schemaVersion { get; set; } = 1;
|
||||
public string label { get; set; }
|
||||
public string message { get; set; }
|
||||
public string color { get; set; }
|
||||
}
|
||||
@ -1,179 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Security.Claims;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using FluentFTP.Helpers;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Nettle;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.DTOs;
|
||||
using Wabbajack.DTOs.GitHub;
|
||||
using Wabbajack.DTOs.JsonConverters;
|
||||
using Wabbajack.Networking.GitHub;
|
||||
using Wabbajack.Paths.IO;
|
||||
using Wabbajack.RateLimiter;
|
||||
using Wabbajack.Server.DataModels;
|
||||
using Wabbajack.Server.Extensions;
|
||||
using Wabbajack.Server.Services;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers;
|
||||
|
||||
[Authorize(Roles = "Author")]
|
||||
[Route("/author_controls")]
|
||||
public class AuthorControls : ControllerBase
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
private readonly DTOSerializer _dtos;
|
||||
private readonly Client _gitHubClient;
|
||||
private readonly QuickSync _quickSync;
|
||||
private readonly AppSettings _settings;
|
||||
private readonly ILogger<AuthorControls> _logger;
|
||||
private readonly AuthorFiles _authorFiles;
|
||||
private readonly IResource<HttpClient> _limiter;
|
||||
|
||||
public AuthorControls(ILogger<AuthorControls> logger, QuickSync quickSync, HttpClient client,
|
||||
AppSettings settings, DTOSerializer dtos, AuthorFiles authorFiles,
|
||||
Client gitHubClient, IResource<HttpClient> limiter)
|
||||
{
|
||||
_logger = logger;
|
||||
_quickSync = quickSync;
|
||||
_client = client;
|
||||
_settings = settings;
|
||||
_dtos = dtos;
|
||||
_gitHubClient = gitHubClient;
|
||||
_authorFiles = authorFiles;
|
||||
_limiter = limiter;
|
||||
}
|
||||
|
||||
[Route("login/{authorKey}")]
|
||||
[AllowAnonymous]
|
||||
public async Task<IActionResult> Login(string authorKey)
|
||||
{
|
||||
Response.Cookies.Append(ApiKeyAuthenticationHandler.ApiKeyHeaderName, authorKey);
|
||||
return Redirect($"{_settings.WabbajackBuildServerUri}author_controls/home");
|
||||
}
|
||||
|
||||
[Route("lists")]
|
||||
[HttpGet]
|
||||
public async Task<IActionResult> AuthorLists()
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
var lists = (await LoadLists())
|
||||
.Where(l => l.Maintainers.Contains(user))
|
||||
.Select(l => l.NamespacedName)
|
||||
.ToArray();
|
||||
|
||||
return Ok(lists);
|
||||
}
|
||||
|
||||
public async Task<ModlistMetadata[]> LoadLists()
|
||||
{
|
||||
var repos = await LoadRepositories();
|
||||
|
||||
return await repos.PMapAll(async url =>
|
||||
{
|
||||
try
|
||||
{
|
||||
return (await _client.GetFromJsonAsync<ModlistMetadata[]>(_limiter,
|
||||
new HttpRequestMessage(HttpMethod.Get, url.Value),
|
||||
_dtos.Options))!.Select(meta =>
|
||||
{
|
||||
meta.RepositoryName = url.Key;
|
||||
return meta;
|
||||
});
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
_logger.LogError(ex, "While loading repository {Name} from {Url}", url.Key, url.Value);
|
||||
return Enumerable.Empty<ModlistMetadata>();
|
||||
}
|
||||
})
|
||||
.SelectMany(x => x)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
public async Task<Dictionary<string, Uri>> LoadRepositories()
|
||||
{
|
||||
var repositories = await _client.GetFromJsonAsync<Dictionary<string, Uri>>(_limiter,
|
||||
new HttpRequestMessage(HttpMethod.Get,
|
||||
"https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/repositories.json"), _dtos.Options);
|
||||
return repositories!;
|
||||
}
|
||||
|
||||
[Route("whoami")]
|
||||
[HttpGet]
|
||||
public async Task<IActionResult> GetWhoAmI()
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
return Ok(user!);
|
||||
}
|
||||
|
||||
|
||||
[Route("lists/download_metadata")]
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> PostDownloadMetadata()
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
var data = await _dtos.DeserializeAsync<UpdateRequest>(Request.Body);
|
||||
try
|
||||
{
|
||||
await _gitHubClient.UpdateList(user, data);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "During posting of download_metadata");
|
||||
return BadRequest(ex);
|
||||
}
|
||||
|
||||
return Ok(data);
|
||||
}
|
||||
|
||||
private static async Task<string> HomePageTemplate(object o)
|
||||
{
|
||||
var data = await KnownFolders.EntryPoint.Combine(@"Controllers\Templates\AuthorControls.html")
|
||||
.ReadAllTextAsync();
|
||||
var func = NettleEngine.GetCompiler().Compile(data);
|
||||
return func(o);
|
||||
}
|
||||
|
||||
[Route("home")]
|
||||
[Authorize("")]
|
||||
public async Task<IActionResult> HomePage()
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
var files = _authorFiles.AllDefinitions
|
||||
.Where(af => af.Definition.Author == user)
|
||||
.Select(af => new
|
||||
{
|
||||
Size = af.Definition.Size.FileSizeToString(),
|
||||
OriginalSize = af.Definition.Size,
|
||||
Name = af.Definition.OriginalFileName,
|
||||
MangledName = af.Definition.MungedName,
|
||||
UploadedDate = af.Updated
|
||||
})
|
||||
.OrderBy(f => f.Name)
|
||||
.ThenBy(f => f.UploadedDate)
|
||||
.ToList();
|
||||
|
||||
var result = HomePageTemplate(new
|
||||
{
|
||||
User = user,
|
||||
TotalUsage = files.Select(f => f.OriginalSize).Sum().ToFileSizeString(),
|
||||
WabbajackFiles = files.Where(f => f.Name.EndsWith(Ext.Wabbajack.ToString())),
|
||||
OtherFiles = files.Where(f => !f.Name.EndsWith(Ext.Wabbajack.ToString()))
|
||||
});
|
||||
|
||||
return new ContentResult
|
||||
{
|
||||
ContentType = "text/html",
|
||||
StatusCode = (int) HttpStatusCode.OK,
|
||||
Content = await result
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -1,183 +0,0 @@
|
||||
using System.Net;
|
||||
using System.Security.Claims;
|
||||
using Humanizer;
|
||||
using Humanizer.Localisation;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Primitives;
|
||||
using Nettle;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.DTOs.CDN;
|
||||
using Wabbajack.DTOs.JsonConverters;
|
||||
using Wabbajack.Hashing.xxHash64;
|
||||
using Wabbajack.Server.DataModels;
|
||||
using Wabbajack.Server.DTOs;
|
||||
using Wabbajack.Server.Services;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers;
|
||||
|
||||
[Authorize(Roles = "Author")]
|
||||
[Route("/authored_files")]
|
||||
public class AuthoredFiles : ControllerBase
|
||||
{
|
||||
private readonly DTOSerializer _dtos;
|
||||
|
||||
private readonly DiscordWebHook _discord;
|
||||
private readonly ILogger<AuthoredFiles> _logger;
|
||||
private readonly AppSettings _settings;
|
||||
private readonly AuthorFiles _authoredFiles;
|
||||
private readonly Func<object,string> _authoredFilesTemplate;
|
||||
|
||||
|
||||
public AuthoredFiles(ILogger<AuthoredFiles> logger, AuthorFiles authorFiles, AppSettings settings, DiscordWebHook discord,
|
||||
DTOSerializer dtos)
|
||||
{
|
||||
_logger = logger;
|
||||
_settings = settings;
|
||||
_discord = discord;
|
||||
_dtos = dtos;
|
||||
_authoredFiles = authorFiles;
|
||||
using var stream = typeof(AuthoredFiles).Assembly
|
||||
.GetManifestResourceStream("Wabbajack.Server.Resources.Reports.AuthoredFiles.html");
|
||||
_authoredFilesTemplate = NettleEngine.GetCompiler().Compile(stream.ReadAllText());
|
||||
}
|
||||
|
||||
[HttpPut]
|
||||
[Route("{serverAssignedUniqueId}/part/{index}")]
|
||||
public async Task<IActionResult> UploadFilePart(CancellationToken token, string serverAssignedUniqueId, long index)
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
var definition = await _authoredFiles.ReadDefinitionForServerId(serverAssignedUniqueId);
|
||||
if (definition.Author != user)
|
||||
return Forbid("File Id does not match authorized user");
|
||||
_logger.Log(LogLevel.Information,
|
||||
$"Uploading File part {definition.OriginalFileName} - ({index} / {definition.Parts.Length})");
|
||||
|
||||
var part = definition.Parts[index];
|
||||
|
||||
await using var ms = new MemoryStream();
|
||||
await Request.Body.CopyToLimitAsync(ms, (int) part.Size, token);
|
||||
ms.Position = 0;
|
||||
if (ms.Length != part.Size)
|
||||
return BadRequest($"Couldn't read enough data for part {part.Size} vs {ms.Length}");
|
||||
|
||||
var hash = await ms.Hash(token);
|
||||
if (hash != part.Hash)
|
||||
return BadRequest(
|
||||
$"Hashes don't match for index {index}. Sizes ({ms.Length} vs {part.Size}). Hashes ({hash} vs {part.Hash}");
|
||||
|
||||
ms.Position = 0;
|
||||
await _authoredFiles.WritePart(definition.MungedName, (int) index, ms);
|
||||
return Ok(part.Hash.ToBase64());
|
||||
}
|
||||
|
||||
[HttpPut]
|
||||
[Route("create")]
|
||||
public async Task<IActionResult> CreateUpload()
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
|
||||
var definition = (await _dtos.DeserializeAsync<FileDefinition>(Request.Body))!;
|
||||
|
||||
_logger.Log(LogLevel.Information, "Creating File upload {originalFileName}", definition.OriginalFileName);
|
||||
|
||||
definition.ServerAssignedUniqueId = Guid.NewGuid().ToString();
|
||||
definition.Author = user;
|
||||
await _authoredFiles.WriteDefinition(definition);
|
||||
|
||||
await _discord.Send(Channel.Ham,
|
||||
new DiscordMessage
|
||||
{
|
||||
Content =
|
||||
$"{user} has started uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})"
|
||||
});
|
||||
|
||||
return Ok(definition.ServerAssignedUniqueId);
|
||||
}
|
||||
|
||||
[HttpPut]
|
||||
[Route("{serverAssignedUniqueId}/finish")]
|
||||
public async Task<IActionResult> CreateUpload(string serverAssignedUniqueId)
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
var definition = await _authoredFiles.ReadDefinitionForServerId(serverAssignedUniqueId);
|
||||
if (definition.Author != user)
|
||||
return Forbid("File Id does not match authorized user");
|
||||
_logger.Log(LogLevel.Information, $"Finalizing file upload {definition.OriginalFileName}");
|
||||
|
||||
await _discord.Send(Channel.Ham,
|
||||
new DiscordMessage
|
||||
{
|
||||
Content =
|
||||
$"{user} has finished uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})"
|
||||
});
|
||||
|
||||
var host = _settings.TestMode ? "test-files" : "authored-files";
|
||||
return Ok($"https://{host}.wabbajack.org/{definition.MungedName}");
|
||||
}
|
||||
|
||||
[HttpDelete]
|
||||
[Route("{serverAssignedUniqueId}")]
|
||||
public async Task<IActionResult> DeleteUpload(string serverAssignedUniqueId)
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
var definition = _authoredFiles.AllDefinitions
|
||||
.First(f => f.Definition.ServerAssignedUniqueId == serverAssignedUniqueId)
|
||||
.Definition;
|
||||
if (definition.Author != user)
|
||||
return Forbid("File Id does not match authorized user");
|
||||
await _discord.Send(Channel.Ham,
|
||||
new DiscordMessage
|
||||
{
|
||||
Content =
|
||||
$"{user} is deleting {definition.MungedName}, {definition.Size.ToFileSizeString()} to be freed"
|
||||
});
|
||||
_logger.Log(LogLevel.Information, $"Deleting upload {definition.OriginalFileName}");
|
||||
|
||||
await _authoredFiles.DeleteFile(definition);
|
||||
return Ok();
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[AllowAnonymous]
|
||||
[Route("")]
|
||||
public async Task<ContentResult> UploadedFilesGet()
|
||||
{
|
||||
var files = _authoredFiles.AllDefinitions
|
||||
.ToArray();
|
||||
var response = _authoredFilesTemplate(new
|
||||
{
|
||||
Files = files.OrderByDescending(f => f.Updated).ToArray(),
|
||||
UsedSpace = _authoredFiles.UsedSpace.Bytes().Humanize("#.##"),
|
||||
});
|
||||
return new ContentResult
|
||||
{
|
||||
ContentType = "text/html",
|
||||
StatusCode = (int) HttpStatusCode.OK,
|
||||
Content = response,
|
||||
};
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[AllowAnonymous]
|
||||
[Route("direct_link/{mungedName}")]
|
||||
public async Task DirectLink(string mungedName)
|
||||
{
|
||||
mungedName = _authoredFiles.DecodeName(mungedName);
|
||||
var definition = await _authoredFiles.ReadDefinition(mungedName);
|
||||
Response.Headers.ContentDisposition =
|
||||
new StringValues($"attachment; filename={definition.OriginalFileName}");
|
||||
Response.Headers.ContentType = new StringValues("application/octet-stream");
|
||||
Response.Headers.ContentLength = definition.Size;
|
||||
Response.Headers.ETag = definition.MungedName + "_direct";
|
||||
|
||||
foreach (var part in definition.Parts.OrderBy(p => p.Index))
|
||||
{
|
||||
await _authoredFiles.StreamForPart(mungedName, (int)part.Index, async stream =>
|
||||
{
|
||||
await stream.CopyToAsync(Response.Body);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,106 +0,0 @@
|
||||
using cesi.DTOs;
|
||||
using CouchDB.Driver;
|
||||
using CouchDB.Driver.Views;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.DTOs.JsonConverters;
|
||||
using Wabbajack.DTOs.Texture;
|
||||
using Wabbajack.DTOs.Vfs;
|
||||
using Wabbajack.Hashing.xxHash64;
|
||||
using Wabbajack.Paths;
|
||||
using Wabbajack.VFS;
|
||||
|
||||
namespace Wabbajack.Server.Controllers;
|
||||
|
||||
[Route("/cesi")]
|
||||
public class Cesi : ControllerBase
|
||||
{
|
||||
private readonly ILogger<Cesi> _logger;
|
||||
private readonly ICouchDatabase<Analyzed> _db;
|
||||
private readonly DTOSerializer _dtos;
|
||||
|
||||
public Cesi(ILogger<Cesi> logger, ICouchDatabase<Analyzed> db, DTOSerializer serializer)
|
||||
{
|
||||
_logger = logger;
|
||||
_db = db;
|
||||
_dtos = serializer;
|
||||
}
|
||||
|
||||
[HttpGet("entry/{hash}")]
|
||||
public async Task<IActionResult> Entry(string hash)
|
||||
{
|
||||
return Ok(await _db.FindAsync(hash));
|
||||
}
|
||||
|
||||
[HttpGet("vfs/{hash}")]
|
||||
public async Task<IActionResult> Vfs(string hash)
|
||||
{
|
||||
var entry = await _db.FindAsync(ReverseHash(hash));
|
||||
if (entry == null) return NotFound(new {Message = "Entry not found", Hash = hash, ReverseHash = ReverseHash(hash)});
|
||||
|
||||
|
||||
var indexed = new IndexedVirtualFile
|
||||
{
|
||||
Hash = Hash.FromHex(ReverseHash(entry.xxHash64)),
|
||||
Size = entry.Size,
|
||||
ImageState = GetImageState(entry),
|
||||
Children = await GetChildrenState(entry),
|
||||
};
|
||||
|
||||
|
||||
return Ok(_dtos.Serialize(indexed, true));
|
||||
}
|
||||
|
||||
private async Task<List<IndexedVirtualFile>> GetChildrenState(Analyzed entry)
|
||||
{
|
||||
if (entry.Archive == null) return new List<IndexedVirtualFile>();
|
||||
|
||||
var children = await _db.GetViewAsync<string, Analyzed>("Indexes", "ArchiveContents", new CouchViewOptions<string>
|
||||
{
|
||||
IncludeDocs = true,
|
||||
Key = entry.xxHash64
|
||||
});
|
||||
|
||||
var indexed = children.ToLookup(d => d.Document.xxHash64, v => v.Document);
|
||||
|
||||
return await entry.Archive.Entries.SelectAsync(async e =>
|
||||
{
|
||||
var found = indexed[e.Value].First();
|
||||
return new IndexedVirtualFile
|
||||
{
|
||||
Name = e.Key.ToRelativePath(),
|
||||
Size = found.Size,
|
||||
Hash = Hash.FromHex(ReverseHash(found.xxHash64)),
|
||||
ImageState = GetImageState(found),
|
||||
Children = await GetChildrenState(found),
|
||||
};
|
||||
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
private ImageState? GetImageState(Analyzed entry)
|
||||
{
|
||||
if (entry.DDS == null) return null;
|
||||
return new ImageState
|
||||
{
|
||||
Width = entry.DDS.Width,
|
||||
Height = entry.DDS.Height,
|
||||
Format = Enum.Parse<DXGI_FORMAT>(entry.DDS.Format),
|
||||
PerceptualHash = new PHash(entry.DDS.PHash.FromHex())
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
private Hash ReverseHash(Hash hash)
|
||||
{
|
||||
return Hash.FromHex(hash.ToArray().Reverse().ToArray().ToHex());
|
||||
}
|
||||
private string ReverseHash(string hash)
|
||||
{
|
||||
return hash.FromHex().Reverse().ToArray().ToHex();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,52 +0,0 @@
|
||||
using System.Security.Claims;
|
||||
using System.Text;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Networking.GitHub;
|
||||
using Wabbajack.Server.DTOs;
|
||||
using Wabbajack.Server.Services;
|
||||
|
||||
namespace Wabbajack.Server.Controllers;
|
||||
|
||||
[Authorize(Roles = "Author")]
|
||||
[Route("/github")]
|
||||
public class Github : ControllerBase
|
||||
{
|
||||
private readonly Client _client;
|
||||
private readonly ILogger<Github> _logger;
|
||||
private readonly DiscordWebHook _discord;
|
||||
|
||||
public Github(ILogger<Github> logger, Client client, DiscordWebHook discord)
|
||||
{
|
||||
_client = client;
|
||||
_logger = logger;
|
||||
_discord = discord;
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
public async Task GetContent([FromQuery] string owner, [FromQuery] string repo, [FromQuery] string path)
|
||||
{
|
||||
var (sha, content) = await _client.GetData(owner, repo, path);
|
||||
Response.StatusCode = 200;
|
||||
Response.Headers.Add("x-content-sha", sha);
|
||||
await Response.WriteAsync(content);
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> SetContent([FromQuery] string owner, [FromQuery] string repo, [FromQuery] string path, [FromQuery] string oldSha)
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name)!;
|
||||
_logger.LogInformation("Updating {Owner}/{Repo}/{Path} on behalf of {User}", owner, repo, path, user);
|
||||
|
||||
await _discord.Send(Channel.Ham,
|
||||
new DiscordMessage {Content = $"Updating {owner}/{repo}/{path} on behalf of {user}"});
|
||||
|
||||
var content = Encoding.UTF8.GetString(await Request.Body.ReadAllAsync());
|
||||
await _client.PutData(owner, repo, path, $"Update on behalf of {user}", content, oldSha);
|
||||
return Ok();
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.Server;
|
||||
using Wabbajack.Server.DTOs;
|
||||
using Wabbajack.Server.Services;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers;
|
||||
|
||||
[Route("/heartbeat")]
|
||||
public class Heartbeat : ControllerBase
|
||||
{
|
||||
private static readonly DateTime _startTime;
|
||||
|
||||
private readonly GlobalInformation _globalInformation;
|
||||
static Heartbeat()
|
||||
{
|
||||
_startTime = DateTime.Now;
|
||||
}
|
||||
|
||||
public Heartbeat(ILogger<Heartbeat> logger, GlobalInformation globalInformation,
|
||||
QuickSync quickSync)
|
||||
{
|
||||
_globalInformation = globalInformation;
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
public async Task<IActionResult> GetHeartbeat()
|
||||
{
|
||||
return Ok(new HeartbeatResult
|
||||
{
|
||||
Uptime = DateTime.Now - _startTime,
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -1,178 +0,0 @@
|
||||
using System.Reflection;
|
||||
using System.Text.Json;
|
||||
using Chronic.Core;
|
||||
using CouchDB.Driver;
|
||||
using CouchDB.Driver.Views;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Nettle;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.DTOs.ServerResponses;
|
||||
using Wabbajack.Server.DataModels;
|
||||
using Wabbajack.Server.DTOs;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Route("/metrics")]
|
||||
public class MetricsController : ControllerBase
|
||||
{
|
||||
private static readonly Func<object, string> ReportTemplate = NettleEngine.GetCompiler().Compile(@"
|
||||
<html><body>
|
||||
<h2>Tar Report for {{$.key}}</h2>
|
||||
<h3>Ban Status: {{$.status}}</h3>
|
||||
<table>
|
||||
{{each $.log }}
|
||||
<tr>
|
||||
<td>{{$.Timestamp}}</td>
|
||||
<td>{{$.Path}}</td>
|
||||
<td>{{$.Key}}</td>
|
||||
</tr>
|
||||
{{/each}}
|
||||
</table>
|
||||
</body></html>
|
||||
");
|
||||
|
||||
private static Func<object, string> _totalListTemplate;
|
||||
private readonly AppSettings _settings;
|
||||
private ILogger<MetricsController> _logger;
|
||||
private readonly Metrics _metricsStore;
|
||||
private readonly ICouchDatabase<Metric> _db;
|
||||
|
||||
public MetricsController(ILogger<MetricsController> logger, Metrics metricsStore,
|
||||
AppSettings settings, ICouchDatabase<Metric> db)
|
||||
{
|
||||
_logger = logger;
|
||||
_settings = settings;
|
||||
_metricsStore = metricsStore;
|
||||
_db = db;
|
||||
}
|
||||
|
||||
|
||||
private static Func<object, string> TotalListTemplate
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_totalListTemplate == null)
|
||||
{
|
||||
var resource = Assembly.GetExecutingAssembly()
|
||||
.GetManifestResourceStream("Wabbajack.Server.Controllers.Templates.TotalListTemplate.html")!
|
||||
.ReadAllText();
|
||||
_totalListTemplate = NettleEngine.GetCompiler().Compile(resource);
|
||||
}
|
||||
|
||||
return _totalListTemplate;
|
||||
}
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("{subject}/{value}")]
|
||||
public async Task<Result> LogMetricAsync(string subject, string value)
|
||||
{
|
||||
var date = DateTime.UtcNow;
|
||||
var metricsKey = Request.Headers[_settings.MetricsKeyHeader].FirstOrDefault();
|
||||
|
||||
// Used in tests
|
||||
if (value is "Default" or "untitled" || subject == "failed_download" || Guid.TryParse(value, out _))
|
||||
return new Result {Timestamp = date};
|
||||
|
||||
await _db.AddAsync(new Metric
|
||||
{
|
||||
Timestamp = date,
|
||||
Action = subject,
|
||||
Subject = value,
|
||||
MetricsKey = metricsKey,
|
||||
UserAgent = Request.Headers.UserAgent.FirstOrDefault() ?? "<unknown>",
|
||||
Ip = Request.Headers["cf-connecting-ip"].FirstOrDefault() ??
|
||||
(Request.HttpContext.Connection.RemoteIpAddress?.ToString() ?? "")
|
||||
});
|
||||
|
||||
return new Result {Timestamp = date};
|
||||
}
|
||||
|
||||
private static byte[] EOL = {(byte)'\n'};
|
||||
private static byte[] LBRACKET = {(byte)'['};
|
||||
private static byte[] RBRACKET = {(byte)']'};
|
||||
private static byte[] COMMA = {(byte) ','};
|
||||
|
||||
[HttpGet]
|
||||
[Route("dump")]
|
||||
public async Task GetMetrics([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to, [FromQuery] string? subject)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("report")]
|
||||
[ResponseCache(Duration = 60 * 10, VaryByQueryKeys = new [] {"action", "from", "to"})]
|
||||
public async Task GetReport([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to)
|
||||
{
|
||||
var parser = new Parser();
|
||||
|
||||
to ??= "now";
|
||||
|
||||
var toDate = parser.Parse(to).Start!.Value.TruncateToDate();
|
||||
|
||||
var groupFilterStart = parser.Parse("three days ago").Start!.Value.TruncateToDate();
|
||||
toDate = new DateTime(toDate.Year, toDate.Month, toDate.Day);
|
||||
|
||||
var prefetch = (await GetByAction(action, groupFilterStart, toDate))
|
||||
.Select(d => d.Subject)
|
||||
.ToHashSet();
|
||||
|
||||
var fromDate = parser.Parse(from).Start!.Value.TruncateToDate();
|
||||
|
||||
var counts = (await GetByAction(action, fromDate, toDate))
|
||||
.Where(r => prefetch.Contains(r.Subject))
|
||||
.ToDictionary(kv => (kv.Date, kv.Subject), kv => kv.Count);
|
||||
|
||||
Response.Headers.ContentType = "application/json";
|
||||
var row = new Dictionary<string, object>();
|
||||
|
||||
Response.Body.Write(LBRACKET);
|
||||
for (var d = fromDate; d <= toDate; d = d.AddDays(1))
|
||||
{
|
||||
row["_Timestamp"] = d;
|
||||
foreach (var group in prefetch)
|
||||
{
|
||||
if (counts.TryGetValue((d, group), out var found))
|
||||
row[group] = found;
|
||||
else
|
||||
row[group] = 0;
|
||||
}
|
||||
await JsonSerializer.SerializeAsync(Response.Body, row);
|
||||
Response.Body.Write(EOL);
|
||||
if (d != toDate)
|
||||
Response.Body.Write(COMMA);
|
||||
}
|
||||
|
||||
Response.Body.Write(RBRACKET);
|
||||
|
||||
}
|
||||
|
||||
|
||||
private async Task<IReadOnlyList<(DateTime Date, string Subject, long Count)>> GetByAction(string action, DateTime from, DateTime to)
|
||||
{
|
||||
var records = await _db.GetViewAsync<object?[], long>("Indexes", "ActionDaySubject",
|
||||
new CouchViewOptions<object?[]>
|
||||
{
|
||||
StartKey = new object?[]{action, from.Year, from.Month, from.Day, null},
|
||||
EndKey = new object?[]{action, to.Year, to.Month, to.Day, new()},
|
||||
Reduce = true,
|
||||
GroupLevel = 10,
|
||||
Group = true
|
||||
});
|
||||
|
||||
var results = records
|
||||
.Where(r => r.Key.Length >= 4 && r.Key[4] != null)
|
||||
.Select(r =>
|
||||
(new DateTime((int)(long)r.Key[1]!, (int)(long)r.Key[2]!, (int)(long)r.Key[3]!), (string)r.Key[4]!, r.Value));
|
||||
return results.ToList();
|
||||
}
|
||||
|
||||
|
||||
public class Result
|
||||
{
|
||||
public DateTime Timestamp { get; set; }
|
||||
}
|
||||
}
|
||||
@ -1,221 +0,0 @@
|
||||
|
||||
|
||||
using System.IO.Compression;
|
||||
using System.Net;
|
||||
using System.Security.Claims;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Primitives;
|
||||
using Wabbajack.BuildServer;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.DTOs.CDN;
|
||||
using Wabbajack.DTOs.JsonConverters;
|
||||
using Wabbajack.Hashing.xxHash64;
|
||||
using Wabbajack.Paths;
|
||||
using Wabbajack.Paths.IO;
|
||||
using Wabbajack.Server.DataModels;
|
||||
using Wabbajack.Server.DTOs;
|
||||
using Wabbajack.Server.Services;
|
||||
|
||||
namespace Wabbajack.Server.Controllers;
|
||||
|
||||
[Authorize(Roles = "Author")]
|
||||
[Route("/mirrored_files")]
|
||||
public class MirroredFiles : ControllerBase
|
||||
{
|
||||
private readonly DTOSerializer _dtos;
|
||||
|
||||
private readonly DiscordWebHook _discord;
|
||||
private readonly ILogger<MirroredFiles> _logger;
|
||||
private readonly AppSettings _settings;
|
||||
|
||||
public AbsolutePath MirrorFilesLocation => _settings.MirrorFilesFolder.ToAbsolutePath();
|
||||
|
||||
|
||||
public MirroredFiles(ILogger<MirroredFiles> logger, AppSettings settings, DiscordWebHook discord,
|
||||
DTOSerializer dtos)
|
||||
{
|
||||
_logger = logger;
|
||||
_settings = settings;
|
||||
_discord = discord;
|
||||
_dtos = dtos;
|
||||
}
|
||||
|
||||
[HttpPut]
|
||||
[Route("{hashAsHex}/part/{index}")]
|
||||
public async Task<IActionResult> UploadFilePart(CancellationToken token, string hashAsHex, long index)
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
var definition = await ReadDefinition(hashAsHex);
|
||||
if (definition.Author != user)
|
||||
return Forbid("File Id does not match authorized user");
|
||||
_logger.Log(LogLevel.Information,
|
||||
$"Uploading File part {definition.OriginalFileName} - ({index} / {definition.Parts.Length})");
|
||||
|
||||
var part = definition.Parts[index];
|
||||
|
||||
await using var ms = new MemoryStream();
|
||||
await Request.Body.CopyToLimitAsync(ms, (int) part.Size, token);
|
||||
ms.Position = 0;
|
||||
if (ms.Length != part.Size)
|
||||
return BadRequest($"Couldn't read enough data for part {part.Size} vs {ms.Length}");
|
||||
|
||||
var hash = await ms.Hash(token);
|
||||
if (hash != part.Hash)
|
||||
return BadRequest(
|
||||
$"Hashes don't match for index {index}. Sizes ({ms.Length} vs {part.Size}). Hashes ({hash} vs {part.Hash}");
|
||||
|
||||
ms.Position = 0;
|
||||
await using var partStream = await CreatePart(hashAsHex, (int)index);
|
||||
await ms.CopyToAsync(partStream, token);
|
||||
return Ok(part.Hash.ToBase64());
|
||||
}
|
||||
|
||||
[HttpPut]
|
||||
[Route("create/{hashAsHex}")]
|
||||
public async Task<IActionResult> CreateUpload(string hashAsHex)
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
|
||||
var definition = (await _dtos.DeserializeAsync<FileDefinition>(Request.Body))!;
|
||||
|
||||
_logger.Log(LogLevel.Information, "Creating File upload {Hash}", hashAsHex);
|
||||
|
||||
definition.ServerAssignedUniqueId = hashAsHex;
|
||||
definition.Author = user;
|
||||
await WriteDefinition(definition);
|
||||
|
||||
await _discord.Send(Channel.Ham,
|
||||
new DiscordMessage
|
||||
{
|
||||
Content =
|
||||
$"{user} has started mirroring {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})"
|
||||
});
|
||||
|
||||
return Ok(definition.ServerAssignedUniqueId);
|
||||
}
|
||||
|
||||
[HttpPut]
|
||||
[Route("{hashAsHex}/finish")]
|
||||
public async Task<IActionResult> FinishUpload(string hashAsHex)
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
var definition = await ReadDefinition(hashAsHex);
|
||||
if (definition.Author != user)
|
||||
return Forbid("File Id does not match authorized user");
|
||||
_logger.Log(LogLevel.Information, "Finalizing file upload {Hash}", hashAsHex);
|
||||
|
||||
await _discord.Send(Channel.Ham,
|
||||
new DiscordMessage
|
||||
{
|
||||
Content =
|
||||
$"{user} has finished uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})"
|
||||
});
|
||||
|
||||
var host = _settings.TestMode ? "test-files" : "authored-files";
|
||||
return Ok($"https://{host}.wabbajack.org/{definition.MungedName}");
|
||||
}
|
||||
|
||||
[HttpDelete]
|
||||
[Route("{hashAsHex}")]
|
||||
public async Task<IActionResult> DeleteMirror(string hashAsHex)
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name);
|
||||
var definition = await ReadDefinition(hashAsHex);
|
||||
|
||||
await _discord.Send(Channel.Ham,
|
||||
new DiscordMessage
|
||||
{
|
||||
Content =
|
||||
$"{user} is deleting {hashAsHex}, {definition.Size.ToFileSizeString()} to be freed"
|
||||
});
|
||||
_logger.Log(LogLevel.Information, "Deleting upload {Hash}", hashAsHex);
|
||||
|
||||
RootPath(hashAsHex).DeleteDirectory();
|
||||
return Ok();
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[AllowAnonymous]
|
||||
[Route("")]
|
||||
public async Task<IActionResult> MirroredFilesGet()
|
||||
{
|
||||
var files = await AllMirroredFiles();
|
||||
foreach (var file in files)
|
||||
file.Parts = Array.Empty<PartDefinition>();
|
||||
return Ok(_dtos.Serialize(files));
|
||||
}
|
||||
|
||||
|
||||
public IEnumerable<AbsolutePath> AllDefinitions => MirrorFilesLocation.EnumerateFiles("definition.json.gz");
|
||||
public async Task<FileDefinition[]> AllMirroredFiles()
|
||||
{
|
||||
var defs = new List<FileDefinition>();
|
||||
foreach (var file in AllDefinitions)
|
||||
{
|
||||
defs.Add(await ReadDefinition(file));
|
||||
}
|
||||
return defs.ToArray();
|
||||
}
|
||||
|
||||
public async Task<FileDefinition> ReadDefinition(string hashAsHex)
|
||||
{
|
||||
return await ReadDefinition(RootPath(hashAsHex).Combine("definition.json.gz"));
|
||||
}
|
||||
|
||||
private async Task<FileDefinition> ReadDefinition(AbsolutePath file)
|
||||
{
|
||||
var gz = new GZipStream(new MemoryStream(await file.ReadAllBytesAsync()), CompressionMode.Decompress);
|
||||
var definition = (await _dtos.DeserializeAsync<FileDefinition>(gz))!;
|
||||
return definition;
|
||||
}
|
||||
|
||||
public async Task WriteDefinition(FileDefinition definition)
|
||||
{
|
||||
var path = RootPath(definition.Hash.ToHex()).Combine("definition.json.gz");
|
||||
path.Parent.CreateDirectory();
|
||||
path.Parent.Combine("parts").CreateDirectory();
|
||||
|
||||
await using var ms = new MemoryStream();
|
||||
await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
|
||||
{
|
||||
await _dtos.Serialize(definition, gz);
|
||||
}
|
||||
|
||||
await path.WriteAllBytesAsync(ms.ToArray());
|
||||
}
|
||||
|
||||
public AbsolutePath RootPath(string hashAsHex)
|
||||
{
|
||||
// Make sure it's a true hash before splicing into the path
|
||||
return MirrorFilesLocation.Combine(Hash.FromHex(hashAsHex).ToHex());
|
||||
}
|
||||
|
||||
|
||||
[HttpGet]
|
||||
[AllowAnonymous]
|
||||
[Route("direct_link/{hashAsHex}")]
|
||||
public async Task DirectLink(string hashAsHex)
|
||||
{
|
||||
var definition = await ReadDefinition(hashAsHex);
|
||||
Response.Headers.ContentDisposition =
|
||||
new StringValues($"attachment; filename={definition.OriginalFileName}");
|
||||
Response.Headers.ContentType = new StringValues("application/octet-stream");
|
||||
foreach (var part in definition.Parts)
|
||||
{
|
||||
await using var partStream = await StreamForPart(hashAsHex, (int)part.Index);
|
||||
await partStream.CopyToAsync(Response.Body);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<Stream> StreamForPart(string hashAsHex, int part)
|
||||
{
|
||||
return RootPath(hashAsHex).Combine("parts", part.ToString()).Open(FileMode.Open);
|
||||
}
|
||||
|
||||
public async Task<Stream> CreatePart(string hashAsHex, int part)
|
||||
{
|
||||
return RootPath(hashAsHex).Combine("parts", part.ToString()).Open(FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
}
|
||||
}
|
||||
@ -1,29 +0,0 @@
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.DTOs;
|
||||
using Wabbajack.DTOs.JsonConverters;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers;
|
||||
|
||||
[Authorize(Roles = "User")]
|
||||
[ApiController]
|
||||
[Route("/mod_files")]
|
||||
public class ModFilesForHash : ControllerBase
|
||||
{
|
||||
private readonly DTOSerializer _dtos;
|
||||
private ILogger<ModFilesForHash> _logger;
|
||||
|
||||
public ModFilesForHash(ILogger<ModFilesForHash> logger, DTOSerializer dtos)
|
||||
{
|
||||
_logger = logger;
|
||||
_dtos = dtos;
|
||||
}
|
||||
|
||||
[HttpGet("by_hash/{hashAsHex}")]
|
||||
public async Task<IActionResult> GetByHash(string hashAsHex)
|
||||
{
|
||||
var empty = Array.Empty<Archive>();
|
||||
return Ok(_dtos.Serialize(empty));
|
||||
}
|
||||
}
|
||||
@ -1,94 +0,0 @@
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.DTOs.JsonConverters;
|
||||
using Wabbajack.Networking.NexusApi.DTOs;
|
||||
using Wabbajack.Paths.IO;
|
||||
using Wabbajack.Server.Services;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers;
|
||||
|
||||
//[Authorize]
|
||||
[ApiController]
|
||||
[Authorize(Roles = "User")]
|
||||
[Route("/v1/games/")]
|
||||
public class NexusCache : ControllerBase
|
||||
{
|
||||
private readonly ILogger<NexusCache> _logger;
|
||||
private readonly HttpClient _client;
|
||||
private readonly DTOSerializer _dtos;
|
||||
private readonly NexusCacheManager _cache;
|
||||
|
||||
public NexusCache(ILogger<NexusCache> logger, HttpClient client, NexusCacheManager cache, DTOSerializer dtos)
|
||||
{
|
||||
_logger = logger;
|
||||
_client = client;
|
||||
_cache = cache;
|
||||
_dtos = dtos;
|
||||
}
|
||||
|
||||
private async Task<T> ForwardRequest<T>(HttpRequest src, CancellationToken token)
|
||||
{
|
||||
_logger.LogInformation("Nexus Cache Forwarding: {path}", src.Path);
|
||||
var request = new HttpRequestMessage(HttpMethod.Get, (Uri?)new Uri("https://api.nexusmods.com/" + src.Path));
|
||||
request.Headers.Add("apikey", (string?)src.Headers["apikey"]);
|
||||
request.Headers.Add("User-Agent", (string?)src.Headers.UserAgent);
|
||||
using var response = await _client.SendAsync(request, token);
|
||||
return (await JsonSerializer.DeserializeAsync<T>(await response.Content.ReadAsStreamAsync(token), _dtos.Options, token))!;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Looks up the mod details for a given Gamename/ModId pair. If the entry is not found in the cache it will
|
||||
/// be requested from the server (using the caller's Nexus API key if provided).
|
||||
/// </summary>
|
||||
/// <param name="db"></param>
|
||||
/// <param name="GameName">The Nexus game name</param>
|
||||
/// <param name="ModId">The Nexus mod id</param>
|
||||
/// <returns>A Mod Info result</returns>
|
||||
[HttpGet]
|
||||
[Route("{GameName}/mods/{ModId}.json")]
|
||||
public async Task GetModInfo(string GameName, long ModId, CancellationToken token)
|
||||
{
|
||||
var key = $"modinfo_{GameName}_{ModId}";
|
||||
await ReturnCachedResult<ModInfo>(key, token);
|
||||
}
|
||||
|
||||
private async Task ReturnCachedResult<T>(string key, CancellationToken token)
|
||||
{
|
||||
key = key.ToLowerInvariant();
|
||||
var cached = await _cache.GetCache<T>(key, token);
|
||||
if (cached == null)
|
||||
{
|
||||
var returned = await ForwardRequest<T>(Request, token);
|
||||
await _cache.SaveCache(key, returned, token);
|
||||
Response.StatusCode = 200;
|
||||
Response.ContentType = "application/json";
|
||||
await JsonSerializer.SerializeAsync(Response.Body, returned, _dtos.Options, cancellationToken: token);
|
||||
return;
|
||||
}
|
||||
|
||||
await JsonSerializer.SerializeAsync(Response.Body, cached, _dtos.Options, cancellationToken: token);
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("{GameName}/mods/{ModId}/files.json")]
|
||||
public async Task GetModFiles(string GameName, long ModId, CancellationToken token)
|
||||
{
|
||||
var key = $"modfiles_{GameName}_{ModId}";
|
||||
await ReturnCachedResult<ModFiles>(key, token);
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("{GameName}/mods/{ModId}/files/{FileId}.json")]
|
||||
public async Task GetModFile(string GameName, long ModId, long FileId, CancellationToken token)
|
||||
{
|
||||
var key = $"modfile_{GameName}_{ModId}_{FileId}";
|
||||
await ReturnCachedResult<ModFile>(key, token);
|
||||
}
|
||||
}
|
||||
@ -1,75 +0,0 @@
|
||||
using System.Security.Claims;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Wabbajack.BuildServer;
|
||||
using Wabbajack.DTOs.JsonConverters;
|
||||
using Wabbajack.Hashing.xxHash64;
|
||||
using Wabbajack.Paths;
|
||||
using Wabbajack.Paths.IO;
|
||||
using Wabbajack.Server.DTOs;
|
||||
using Wabbajack.Server.Services;
|
||||
|
||||
namespace Wabbajack.Server.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Authorize(Roles = "Author")]
|
||||
[Route("/patches")]
|
||||
public class Patches : ControllerBase
|
||||
{
|
||||
private readonly AppSettings _settings;
|
||||
private readonly DiscordWebHook _discord;
|
||||
private readonly DTOSerializer _dtos;
|
||||
|
||||
public Patches(AppSettings appSettings, DiscordWebHook discord, DTOSerializer dtos)
|
||||
{
|
||||
_settings = appSettings;
|
||||
_discord = discord;
|
||||
_dtos = dtos;
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> WritePart(CancellationToken token, [FromQuery] string name, [FromQuery] long start)
|
||||
{
|
||||
var path = GetPath(name);
|
||||
if (!path.FileExists())
|
||||
{
|
||||
|
||||
var user = User.FindFirstValue(ClaimTypes.Name)!;
|
||||
await _discord.Send(Channel.Ham,
|
||||
new DiscordMessage {Content = $"{user} is uploading a new forced-healing patch file"});
|
||||
}
|
||||
|
||||
await using var file = path.Open(FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read);
|
||||
file.Position = start;
|
||||
var hash = await Request.Body.HashingCopy(file, token);
|
||||
await file.FlushAsync(token);
|
||||
return Ok(hash.ToHex());
|
||||
}
|
||||
|
||||
private AbsolutePath GetPath(string name)
|
||||
{
|
||||
return _settings.PatchesFilesFolder.ToAbsolutePath().Combine(name);
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("list")]
|
||||
public async Task<IActionResult> ListPatches(CancellationToken token)
|
||||
{
|
||||
var root = _settings.PatchesFilesFolder.ToAbsolutePath();
|
||||
var files = root.EnumerateFiles()
|
||||
.ToDictionary(f => f.RelativeTo(root).ToString(), f => f.Size());
|
||||
return Ok(_dtos.Serialize(files));
|
||||
}
|
||||
|
||||
[HttpDelete]
|
||||
public async Task<IActionResult> DeletePart([FromQuery] string name)
|
||||
{
|
||||
var user = User.FindFirstValue(ClaimTypes.Name)!;
|
||||
await _discord.Send(Channel.Ham,
|
||||
new DiscordMessage {Content = $"{user} is deleting a new forced-healing patch file"});
|
||||
|
||||
GetPath(name).Delete();
|
||||
return Ok(name);
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,228 +0,0 @@
|
||||
using System.Text;
|
||||
using Amazon.Runtime;
|
||||
using Amazon.S3;
|
||||
using Amazon.S3.Model;
|
||||
using FluentFTP.Helpers;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Net.Http.Headers;
|
||||
using Wabbajack.BuildServer;
|
||||
using Wabbajack.Downloaders;
|
||||
using Wabbajack.Downloaders.Interfaces;
|
||||
using Wabbajack.DTOs;
|
||||
using Wabbajack.DTOs.DownloadStates;
|
||||
using Wabbajack.Hashing.xxHash64;
|
||||
using Wabbajack.Paths;
|
||||
using Wabbajack.Paths.IO;
|
||||
using Wabbajack.RateLimiter;
|
||||
using Wabbajack.VFS;
|
||||
|
||||
namespace Wabbajack.Server.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Route("/proxy")]
|
||||
public class Proxy : ControllerBase
|
||||
{
|
||||
private readonly ILogger<Proxy> _logger;
|
||||
private readonly DownloadDispatcher _dispatcher;
|
||||
private readonly TemporaryFileManager _tempFileManager;
|
||||
private readonly AppSettings _appSettings;
|
||||
private readonly FileHashCache _hashCache;
|
||||
private readonly IAmazonS3 _s3;
|
||||
private readonly string _bucket;
|
||||
|
||||
private string _redirectUrl = "https://proxy.wabbajack.org/";
|
||||
private readonly IResource<DownloadDispatcher> _resource;
|
||||
|
||||
public Proxy(ILogger<Proxy> logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager,
|
||||
FileHashCache hashCache, AppSettings appSettings, IAmazonS3 s3, IResource<DownloadDispatcher> resource)
|
||||
{
|
||||
_logger = logger;
|
||||
_dispatcher = dispatcher;
|
||||
_tempFileManager = tempFileManager;
|
||||
_appSettings = appSettings;
|
||||
_hashCache = hashCache;
|
||||
_s3 = s3;
|
||||
_bucket = _appSettings.S3.ProxyFilesBucket;
|
||||
_resource = resource;
|
||||
}
|
||||
|
||||
[HttpHead]
|
||||
public async Task<IActionResult> ProxyHead(CancellationToken token, [FromQuery] Uri uri, [FromQuery] string? name,
|
||||
[FromQuery] string? hash)
|
||||
{
|
||||
var cacheName = (await Encoding.UTF8.GetBytes(uri.ToString()).Hash()).ToHex();
|
||||
return new RedirectResult(_redirectUrl + cacheName);
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
public async Task<IActionResult> ProxyGet(CancellationToken token, [FromQuery] Uri uri, [FromQuery] string? name, [FromQuery] string? hash)
|
||||
{
|
||||
|
||||
Hash hashResult = default;
|
||||
var shouldMatch = hash != null ? Hash.FromHex(hash) : default;
|
||||
|
||||
_logger.LogInformation("Got proxy request for {Uri}", uri);
|
||||
var state = _dispatcher.Parse(uri);
|
||||
var cacheName = (await Encoding.UTF8.GetBytes(uri.ToString()).Hash()).ToHex();
|
||||
var cacheFile = await GetCacheEntry(cacheName);
|
||||
|
||||
if (state == null)
|
||||
{
|
||||
return BadRequest(new {Type = "Could not get state from Uri", Uri = uri.ToString()});
|
||||
}
|
||||
|
||||
var archive = new Archive
|
||||
{
|
||||
Name = name ?? "",
|
||||
State = state,
|
||||
Hash = shouldMatch
|
||||
|
||||
};
|
||||
|
||||
var downloader = _dispatcher.Downloader(archive);
|
||||
if (downloader is not IProxyable)
|
||||
{
|
||||
return BadRequest(new {Type = "Downloader is not IProxyable", Downloader = downloader.GetType().FullName});
|
||||
}
|
||||
|
||||
if (cacheFile != null && (DateTime.UtcNow - cacheFile.LastModified) > TimeSpan.FromHours(4))
|
||||
{
|
||||
try
|
||||
{
|
||||
var verify = await _dispatcher.Verify(archive, token);
|
||||
if (verify)
|
||||
await TouchCacheEntry(cacheName);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogInformation(ex, "When trying to verify cached file ({Hash}) {Url}",
|
||||
cacheFile.Hash, uri);
|
||||
await TouchCacheEntry(cacheName);
|
||||
}
|
||||
}
|
||||
|
||||
if (cacheFile != null && (DateTime.Now - cacheFile.LastModified) > TimeSpan.FromHours(24))
|
||||
{
|
||||
try
|
||||
{
|
||||
await DeleteCacheEntry(cacheName);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "When trying to delete expired file");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
var redirectUrl = _redirectUrl + cacheName + "?response-content-disposition=attachment;filename=" + (name ?? "unknown");
|
||||
if (cacheFile != null)
|
||||
{
|
||||
if (hash != default)
|
||||
{
|
||||
if (cacheFile.Hash != shouldMatch)
|
||||
return BadRequest(new {Type = "Unmatching Hashes", Expected = shouldMatch.ToHex(), Found = hashResult.ToHex()});
|
||||
}
|
||||
return new RedirectResult(redirectUrl);
|
||||
}
|
||||
|
||||
_logger.LogInformation("Downloading proxy request for {Uri}", uri);
|
||||
|
||||
var tempFile = _tempFileManager.CreateFile(deleteOnDispose:false);
|
||||
|
||||
var proxyDownloader = _dispatcher.Downloader(archive) as IProxyable;
|
||||
|
||||
using var job = await _resource.Begin("Downloading file", 0, token);
|
||||
hashResult = await proxyDownloader!.Download(archive, tempFile.Path, job, token);
|
||||
|
||||
|
||||
if (hash != default && hashResult != shouldMatch)
|
||||
{
|
||||
if (tempFile.Path.FileExists())
|
||||
tempFile.Path.Delete();
|
||||
return NotFound();
|
||||
}
|
||||
|
||||
await PutCacheEntry(tempFile.Path, cacheName, hashResult);
|
||||
|
||||
_logger.LogInformation("Returning proxy request for {Uri}", uri);
|
||||
return new RedirectResult(redirectUrl);
|
||||
}
|
||||
|
||||
private async Task<CacheStatus?> GetCacheEntry(string name)
|
||||
{
|
||||
GetObjectMetadataResponse info;
|
||||
try
|
||||
{
|
||||
info = await _s3.GetObjectMetadataAsync(new GetObjectMetadataRequest()
|
||||
{
|
||||
BucketName = _bucket,
|
||||
Key = name,
|
||||
});
|
||||
}
|
||||
catch (Exception _)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (info.HttpStatusCode == System.Net.HttpStatusCode.NotFound)
|
||||
return null;
|
||||
|
||||
if (info.Metadata["WJ-Hash"] == null)
|
||||
return null;
|
||||
|
||||
if (!Hash.TryGetFromHex(info.Metadata["WJ-Hash"], out var hash))
|
||||
return null;
|
||||
|
||||
return new CacheStatus
|
||||
{
|
||||
LastModified = info.LastModified,
|
||||
Size = info.ContentLength,
|
||||
Hash = hash
|
||||
};
|
||||
}
|
||||
|
||||
private async Task TouchCacheEntry(string name)
|
||||
{
|
||||
await _s3.CopyObjectAsync(new CopyObjectRequest()
|
||||
{
|
||||
SourceBucket = _bucket,
|
||||
DestinationBucket = _bucket,
|
||||
SourceKey = name,
|
||||
DestinationKey = name,
|
||||
MetadataDirective = S3MetadataDirective.REPLACE,
|
||||
});
|
||||
}
|
||||
|
||||
private async Task PutCacheEntry(AbsolutePath path, string name, Hash hash)
|
||||
{
|
||||
var obj = new PutObjectRequest
|
||||
{
|
||||
BucketName = _bucket,
|
||||
Key = name,
|
||||
FilePath = path.ToString(),
|
||||
ContentType = "application/octet-stream",
|
||||
DisablePayloadSigning = true
|
||||
};
|
||||
obj.Metadata.Add("WJ-Hash", hash.ToHex());
|
||||
await _s3.PutObjectAsync(obj);
|
||||
}
|
||||
|
||||
private async Task DeleteCacheEntry(string name)
|
||||
{
|
||||
await _s3.DeleteObjectAsync(new DeleteObjectRequest
|
||||
{
|
||||
BucketName = _bucket,
|
||||
Key = name
|
||||
});
|
||||
}
|
||||
|
||||
record CacheStatus
|
||||
{
|
||||
public DateTime LastModified { get; init; }
|
||||
public long Size { get; init; }
|
||||
|
||||
public Hash Hash { get; init; }
|
||||
}
|
||||
}
|
||||
@ -1,73 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Author Controls - {{$.User}} - {{$.TotalUsage}}</title>
|
||||
</head>
|
||||
<body>
|
||||
<h2>Author Controls - {{$.User}} - {{$.TotalUsage}}</h2>
|
||||
<br/>
|
||||
<h3>Wabbajack Files</h3>
|
||||
<table>
|
||||
<tr>
|
||||
<td><b>Commands</b></td>
|
||||
<td><b>Name</b></td>
|
||||
<td><b>Size</b></td>
|
||||
<td><b>Finished Uploading</b></td>
|
||||
<td><b>Unique Name</b></td>
|
||||
</tr>
|
||||
{{each $.WabbajackFiles }}
|
||||
<tr>
|
||||
<td>
|
||||
<button onclick="deleteFile('{{@Escape($.MangledName)}}');">Delete</button>
|
||||
</td>
|
||||
<td>{{$.Name}}</td>
|
||||
<td>{{$.Size}}</td>
|
||||
<td>{{$.UploadedDate}}</td>
|
||||
<td>{{$.MangledName}}</td>
|
||||
|
||||
</tr>
|
||||
{{/each}}
|
||||
</table>
|
||||
|
||||
<h3>Other Files</h3>
|
||||
<table>
|
||||
<tr>
|
||||
<td><b>Commands</b></td>
|
||||
<td><b>Name</b></td>
|
||||
<td><b>Size</b></td>
|
||||
<td><b>Finished Uploading</b></td>
|
||||
<td><b>Unique Name</b></td>
|
||||
|
||||
</tr>
|
||||
{{each $.OtherFiles }}
|
||||
<tr>
|
||||
<td>
|
||||
<button onclick="deleteFile('{{@Escape($.MangledName)}}');">Delete</button>
|
||||
</td>
|
||||
<td>{{$.Name}}</td>
|
||||
<td>{{$.Size}}</td>
|
||||
<td>{{$.UploadedDate}}</td>
|
||||
<td>{{$.MangledName}}</td>
|
||||
</tr>
|
||||
{{/each}}
|
||||
</table>
|
||||
<script lang="javascript">
|
||||
|
||||
if (!Array.prototype.last) {
|
||||
Array.prototype.last = function () {
|
||||
return this[this.length - 1];
|
||||
};
|
||||
}
|
||||
|
||||
function deleteFile(mangled) {
|
||||
if (window.confirm("Are you sure you want to delete: " + mangled)) {
|
||||
fetch("/authored_files/" + mangled.split("_").last(), {method: "DELETE"})
|
||||
.then(r => location.reload());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@ -1,21 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Total Installs</title>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<h2>{{$.Title}} - Total: {{$.Total}}</h2>
|
||||
|
||||
<table>
|
||||
{{each $.Items }}
|
||||
<tr>
|
||||
<td>{{$.Count}}</td>
|
||||
<td>{{$.Title}}</td>
|
||||
</tr>
|
||||
{{/each}}
|
||||
</table>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@ -1,48 +0,0 @@
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.Paths.IO;
|
||||
using Wabbajack.Server.DataLayer;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers;
|
||||
|
||||
[Authorize]
|
||||
[Route("/users")]
|
||||
public class Users : ControllerBase
|
||||
{
|
||||
private ILogger<Users> _logger;
|
||||
private readonly AppSettings _settings;
|
||||
private readonly SqlService _sql;
|
||||
|
||||
public Users(ILogger<Users> logger, SqlService sql, AppSettings settings)
|
||||
{
|
||||
_settings = settings;
|
||||
_logger = logger;
|
||||
_sql = sql;
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("add/{Name}")]
|
||||
public async Task<string> AddUser(string Name)
|
||||
{
|
||||
return await _sql.AddLogin(Name);
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("export")]
|
||||
public async Task<string> Export()
|
||||
{
|
||||
var mainFolder = _settings.TempPath.Combine("exported_users");
|
||||
mainFolder.CreateDirectory();
|
||||
|
||||
foreach (var (owner, key) in await _sql.GetAllUserKeys())
|
||||
{
|
||||
var folder = mainFolder.Combine(owner);
|
||||
folder.CreateDirectory();
|
||||
await folder.Combine(_settings.AuthorAPIKeyFile).WriteAllTextAsync(key);
|
||||
}
|
||||
|
||||
return "done";
|
||||
}
|
||||
}
|
||||
@ -1,76 +0,0 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Wabbajack.Server.DTOs;
|
||||
|
||||
public class DiscordMessage
|
||||
{
|
||||
[JsonPropertyName("username")]
|
||||
public string? UserName { get; set; }
|
||||
|
||||
[JsonPropertyName("avatar_url")]
|
||||
public Uri? AvatarUrl { get; set; }
|
||||
|
||||
[JsonPropertyName("content")]
|
||||
public string? Content { get; set; }
|
||||
|
||||
[JsonPropertyName("embeds")]
|
||||
public DiscordEmbed[]? Embeds { get; set; }
|
||||
}
|
||||
|
||||
public class DiscordEmbed
|
||||
{
|
||||
[JsonPropertyName("title")] public string Title { get; set; }
|
||||
|
||||
[JsonPropertyName("color")] public int Color { get; set; }
|
||||
|
||||
[JsonPropertyName("author")] public DiscordAuthor Author { get; set; }
|
||||
|
||||
[JsonPropertyName("url")] public Uri Url { get; set; }
|
||||
|
||||
[JsonPropertyName("description")] public string Description { get; set; }
|
||||
|
||||
[JsonPropertyName("fields")] public DiscordField Field { get; set; }
|
||||
|
||||
[JsonPropertyName("thumbnail")] public DiscordThumbnail Thumbnail { get; set; }
|
||||
|
||||
[JsonPropertyName("image")] public DiscordImage Image { get; set; }
|
||||
|
||||
[JsonPropertyName("footer")] public DiscordFooter Footer { get; set; }
|
||||
|
||||
[JsonPropertyName("timestamp")] public DateTime Timestamp { get; set; } = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
public class DiscordAuthor
|
||||
{
|
||||
[JsonPropertyName("name")] public string Name { get; set; }
|
||||
|
||||
[JsonPropertyName("url")] public Uri Url { get; set; }
|
||||
|
||||
[JsonPropertyName("icon_url")] public Uri IconUrl { get; set; }
|
||||
}
|
||||
|
||||
public class DiscordField
|
||||
{
|
||||
[JsonPropertyName("name")] public string Name { get; set; }
|
||||
|
||||
[JsonPropertyName("value")] public string Value { get; set; }
|
||||
|
||||
[JsonPropertyName("inline")] public bool Inline { get; set; }
|
||||
}
|
||||
|
||||
public class DiscordThumbnail
|
||||
{
|
||||
[JsonPropertyName("Url")] public Uri Url { get; set; }
|
||||
}
|
||||
|
||||
public class DiscordImage
|
||||
{
|
||||
[JsonPropertyName("Url")] public Uri Url { get; set; }
|
||||
}
|
||||
|
||||
public class DiscordFooter
|
||||
{
|
||||
[JsonPropertyName("text")] public string Text { get; set; }
|
||||
|
||||
[JsonPropertyName("icon_url")] public Uri icon_url { get; set; }
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
using System;
|
||||
|
||||
namespace Wabbajack.Server.DTOs;
|
||||
|
||||
public class HeartbeatResult
|
||||
{
|
||||
public TimeSpan Uptime { get; set; }
|
||||
public TimeSpan LastNexusUpdate { get; set; }
|
||||
|
||||
public TimeSpan LastListValidation { get; set; }
|
||||
}
|
||||
@ -1,15 +0,0 @@
|
||||
using System;
|
||||
using CouchDB.Driver.Types;
|
||||
using Microsoft.Extensions.Primitives;
|
||||
|
||||
namespace Wabbajack.Server.DTOs;
|
||||
|
||||
public class Metric : CouchDocument
|
||||
{
|
||||
public DateTime Timestamp { get; set; }
|
||||
public string Action { get; set; }
|
||||
public string Subject { get; set; }
|
||||
public string MetricsKey { get; set; }
|
||||
public string UserAgent { get; set; }
|
||||
public string Ip { get; set; }
|
||||
}
|
||||
@ -1,279 +0,0 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.IO.Compression;
|
||||
using System.Web;
|
||||
using Amazon.S3;
|
||||
using Amazon.S3.Model;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.IO;
|
||||
using Wabbajack.BuildServer;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.DTOs.CDN;
|
||||
using Wabbajack.DTOs.JsonConverters;
|
||||
using Wabbajack.Paths;
|
||||
using Wabbajack.Paths.IO;
|
||||
|
||||
|
||||
namespace Wabbajack.Server.DataModels;
|
||||
|
||||
public class AuthorFiles
|
||||
{
|
||||
private readonly ILogger<AuthorFiles> _logger;
|
||||
private readonly AppSettings _settings;
|
||||
private readonly DTOSerializer _dtos;
|
||||
private ConcurrentDictionary<string, FileDefinition> _byServerId = new();
|
||||
private readonly IAmazonS3 _s3;
|
||||
private readonly ConcurrentDictionary<string,FileDefinitionMetadata> _fileCache;
|
||||
private readonly string _bucketName;
|
||||
private ConcurrentDictionary<RelativePath, long> _allObjects = new();
|
||||
private HashSet<RelativePath> _mangledNames;
|
||||
private readonly RecyclableMemoryStreamManager _streamPool;
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly AbsolutePath _cacheFile;
|
||||
|
||||
private Uri _baseUri => new($"https://authored-files.wabbajack.org/");
|
||||
|
||||
public AuthorFiles(ILogger<AuthorFiles> logger, AppSettings settings, DTOSerializer dtos, IAmazonS3 s3, HttpClient client)
|
||||
{
|
||||
_httpClient = client;
|
||||
_s3 = s3;
|
||||
_logger = logger;
|
||||
_settings = settings;
|
||||
_dtos = dtos;
|
||||
_fileCache = new ConcurrentDictionary<string, FileDefinitionMetadata>();
|
||||
_bucketName = settings.S3.AuthoredFilesBucket;
|
||||
_ = PrimeCache();
|
||||
_streamPool = new RecyclableMemoryStreamManager();
|
||||
_cacheFile = _settings.S3.AuthoredFilesBucketCache.ToAbsolutePath();
|
||||
}
|
||||
|
||||
private async Task PrimeCache()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!_cacheFile.FileExists())
|
||||
{
|
||||
var allObjects = await AllObjects().ToArrayAsync();
|
||||
foreach (var obje in allObjects)
|
||||
{
|
||||
_allObjects.TryAdd(obje.Key.ToRelativePath(), obje.LastModified.ToFileTimeUtc());
|
||||
}
|
||||
SaveBucketCacheFile(_cacheFile);
|
||||
}
|
||||
else
|
||||
{
|
||||
LoadBucketCacheFile(_cacheFile);
|
||||
}
|
||||
|
||||
|
||||
_mangledNames = _allObjects
|
||||
.Where(f => f.Key.EndsWith("definition.json.gz"))
|
||||
.Select(f => f.Key.Parent)
|
||||
.ToHashSet();
|
||||
|
||||
await Parallel.ForEachAsync(_mangledNames, async (name, _) =>
|
||||
{
|
||||
if (!_allObjects.TryGetValue(name.Combine("definition.json.gz"), out var value))
|
||||
return;
|
||||
|
||||
_logger.LogInformation("Priming {Name}", name);
|
||||
var definition = await PrimeDefinition(name);
|
||||
var metadata = new FileDefinitionMetadata()
|
||||
{
|
||||
Definition = definition,
|
||||
Updated = DateTime.FromFileTimeUtc(value)
|
||||
};
|
||||
_fileCache.TryAdd(definition.MungedName, metadata);
|
||||
_byServerId.TryAdd(definition.ServerAssignedUniqueId!, definition);
|
||||
});
|
||||
|
||||
_logger.LogInformation("Finished priming cache, {Count} files {Size} GB cached", _fileCache.Count,
|
||||
_fileCache.Sum(s => s.Value.Definition.Size) / (1024 * 1024 * 1024));
|
||||
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogCritical(ex, "Failed to prime cache");
|
||||
}
|
||||
}
|
||||
|
||||
private void SaveBucketCacheFile(AbsolutePath cacheFile)
|
||||
{
|
||||
using var file = cacheFile.Open(FileMode.Create, FileAccess.Write);
|
||||
using var sw = new StreamWriter(file);
|
||||
foreach(var entry in _allObjects)
|
||||
{
|
||||
sw.WriteLine($"{entry.Key}||{entry.Value}");
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadBucketCacheFile(AbsolutePath cacheFile)
|
||||
{
|
||||
using var file = cacheFile.Open(FileMode.Open, FileAccess.Read);
|
||||
using var sr = new StreamReader(file);
|
||||
while (!sr.EndOfStream)
|
||||
{
|
||||
var line = sr.ReadLine();
|
||||
var parts = line!.Split("||");
|
||||
_allObjects.TryAdd(parts[0].ToRelativePath(), long.Parse(parts[1]));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<FileDefinition> PrimeDefinition(RelativePath name)
|
||||
{
|
||||
return await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () =>
|
||||
{
|
||||
var uri = _baseUri + $"{name}/definition.json.gz";
|
||||
using var response = await _httpClient.GetAsync(uri);
|
||||
return await ReadDefinition(await response.Content.ReadAsStreamAsync());
|
||||
});
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<S3Object> AllObjects()
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
var total = 0;
|
||||
_logger.Log(LogLevel.Information, "Listing all objects in S3");
|
||||
var results = await _s3.ListObjectsV2Async(new ListObjectsV2Request()
|
||||
{
|
||||
BucketName = _bucketName,
|
||||
});
|
||||
TOP:
|
||||
total += results.S3Objects.Count;
|
||||
_logger.Log(LogLevel.Information, "Got {S3ObjectsCount} objects, {Total} total", results.S3Objects.Count, total);
|
||||
foreach (var result in results.S3Objects)
|
||||
{
|
||||
yield return result;
|
||||
}
|
||||
|
||||
if (results.IsTruncated)
|
||||
{
|
||||
results = await _s3.ListObjectsV2Async(new ListObjectsV2Request
|
||||
{
|
||||
ContinuationToken = results.NextContinuationToken,
|
||||
BucketName = _bucketName,
|
||||
});
|
||||
goto TOP;
|
||||
}
|
||||
_logger.LogInformation("Finished listing all objects in S3 in {Elapsed}", sw.Elapsed);
|
||||
}
|
||||
|
||||
public IEnumerable<FileDefinitionMetadata> AllDefinitions => _fileCache.Values;
|
||||
|
||||
/// <summary>
|
||||
/// Used space in bytes
|
||||
/// </summary>
|
||||
public long UsedSpace => _fileCache.Sum(s => s.Value.Definition.Size);
|
||||
|
||||
public async Task StreamForPart(string mungedName, int part, Func<Stream, Task> func)
|
||||
{
|
||||
var definition = _fileCache[mungedName].Definition;
|
||||
|
||||
if (part >= definition.Parts.Length)
|
||||
throw new ArgumentOutOfRangeException(nameof(part));
|
||||
|
||||
var uri = _baseUri + $"{mungedName}/parts/{part}";
|
||||
using var response = await _httpClient.GetAsync(uri);
|
||||
await func(await response.Content.ReadAsStreamAsync());
|
||||
}
|
||||
|
||||
public async Task WritePart(string mungedName, int part, Stream ms)
|
||||
{
|
||||
await _s3.PutObjectAsync(new PutObjectRequest
|
||||
{
|
||||
BucketName = _bucketName,
|
||||
Key = mungedName.ToRelativePath().Combine("parts", part.ToString()).ToString().Replace("\\", "/"),
|
||||
InputStream = ms,
|
||||
DisablePayloadSigning = true,
|
||||
ContentType = "application/octet-stream"
|
||||
});
|
||||
}
|
||||
|
||||
public async Task WriteDefinition(FileDefinition definition)
|
||||
{
|
||||
await using var ms = new MemoryStream();
|
||||
await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
|
||||
{
|
||||
await _dtos.Serialize(definition, gz);
|
||||
}
|
||||
ms.Position = 0;
|
||||
|
||||
await _s3.PutObjectAsync(new PutObjectRequest
|
||||
{
|
||||
BucketName = _bucketName,
|
||||
Key = definition.MungedName.ToRelativePath().Combine("definition.json.gz").ToString().Replace("\\", "/"),
|
||||
InputStream = ms,
|
||||
DisablePayloadSigning = true,
|
||||
ContentType = "application/octet-stream"
|
||||
});
|
||||
_fileCache.TryAdd(definition.MungedName, new FileDefinitionMetadata
|
||||
{
|
||||
Definition = definition,
|
||||
Updated = DateTime.UtcNow
|
||||
});
|
||||
_byServerId.TryAdd(definition.ServerAssignedUniqueId!, definition);
|
||||
}
|
||||
|
||||
public async Task<FileDefinition> ReadDefinition(string mungedName)
|
||||
{
|
||||
return _fileCache[mungedName].Definition;
|
||||
}
|
||||
|
||||
public bool IsDefinition(string mungedName)
|
||||
{
|
||||
return _fileCache.ContainsKey(mungedName);
|
||||
}
|
||||
|
||||
|
||||
private async Task<FileDefinition> ReadDefinition(Stream stream)
|
||||
{
|
||||
var gz = new GZipStream(stream, CompressionMode.Decompress);
|
||||
var definition = (await _dtos.DeserializeAsync<FileDefinition>(gz))!;
|
||||
return definition;
|
||||
}
|
||||
|
||||
public class FileDefinitionMetadata
|
||||
{
|
||||
public FileDefinition Definition { get; set; }
|
||||
public DateTime Updated { get; set; }
|
||||
public string HumanSize => Definition.Size.ToFileSizeString();
|
||||
}
|
||||
|
||||
public async Task DeleteFile(FileDefinition definition)
|
||||
{
|
||||
var allFiles = _allObjects.Where(f => f.Key.TopParent.ToString() == definition.MungedName)
|
||||
.Select(f => f.Key).ToList();
|
||||
foreach (var batch in allFiles.Batch(512))
|
||||
{
|
||||
var batchedArray = batch.ToHashSet();
|
||||
_logger.LogInformation("Deleting {Count} files for prefix {Prefix}", batchedArray.Count, definition.MungedName);
|
||||
await _s3.DeleteObjectsAsync(new DeleteObjectsRequest
|
||||
{
|
||||
BucketName = _bucketName,
|
||||
|
||||
Objects = batchedArray.Select(f => new KeyVersion
|
||||
{
|
||||
Key = f.ToString().Replace("\\", "/")
|
||||
}).ToList()
|
||||
});
|
||||
foreach (var key in batchedArray)
|
||||
{
|
||||
_allObjects.TryRemove(key, out _);
|
||||
}
|
||||
}
|
||||
|
||||
_byServerId.TryRemove(definition.ServerAssignedUniqueId!, out _);
|
||||
_fileCache.TryRemove(definition.MungedName, out _);
|
||||
}
|
||||
|
||||
public async ValueTask<FileDefinition> ReadDefinitionForServerId(string serverAssignedUniqueId)
|
||||
{
|
||||
return _byServerId[serverAssignedUniqueId];
|
||||
}
|
||||
|
||||
public string DecodeName(string mungedName)
|
||||
{
|
||||
var decoded = HttpUtility.UrlDecode(mungedName);
|
||||
return IsDefinition(decoded) ? decoded : mungedName;
|
||||
}
|
||||
}
|
||||
@ -1,28 +0,0 @@
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.BuildServer;
|
||||
using Wabbajack.Paths;
|
||||
using Wabbajack.Paths.IO;
|
||||
|
||||
namespace Wabbajack.Server.DataModels;
|
||||
|
||||
public class AuthorKeys
|
||||
{
|
||||
private readonly AppSettings _settings;
|
||||
private AbsolutePath AuthorKeysPath => _settings.AuthorAPIKeyFile.ToAbsolutePath();
|
||||
|
||||
public AuthorKeys(AppSettings settings)
|
||||
{
|
||||
_settings = settings;
|
||||
}
|
||||
|
||||
public async Task<string?> AuthorForKey(string key)
|
||||
{
|
||||
await foreach (var line in AuthorKeysPath.ReadAllLinesAsync())
|
||||
{
|
||||
var parts = line.Split("\t");
|
||||
if (parts[0].Trim() == key)
|
||||
return parts[1].Trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -1,138 +0,0 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Concurrent;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Chronic.Core;
|
||||
using Microsoft.Toolkit.HighPerformance;
|
||||
using Wabbajack.BuildServer;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.DTOs.JsonConverters;
|
||||
using Wabbajack.DTOs.ServerResponses;
|
||||
using Wabbajack.Hashing.xxHash64;
|
||||
using Wabbajack.Paths;
|
||||
using Wabbajack.Paths.IO;
|
||||
using Wabbajack.Server.DTOs;
|
||||
|
||||
namespace Wabbajack.Server.DataModels;
|
||||
|
||||
public class Metrics
|
||||
{
|
||||
private readonly AppSettings _settings;
|
||||
public SemaphoreSlim _lock = new(1);
|
||||
private readonly DTOSerializer _dtos;
|
||||
|
||||
public Metrics(AppSettings settings, DTOSerializer dtos)
|
||||
{
|
||||
_settings = settings;
|
||||
_dtos = dtos;
|
||||
}
|
||||
|
||||
public async Task Ingest(Metric metric)
|
||||
{
|
||||
using var _ = await _lock.Lock();
|
||||
var data = Encoding.UTF8.GetBytes(_dtos.Serialize(metric));
|
||||
var metricsFile = _settings.MetricsFolder.ToAbsolutePath().Combine(DateTime.Now.ToString("yyyy_MM_dd") + ".json");
|
||||
await using var fs = metricsFile.Open(FileMode.Append, FileAccess.Write, FileShare.Read);
|
||||
fs.Write(data);
|
||||
fs.Write(Encoding.UTF8.GetBytes("\n"));
|
||||
}
|
||||
|
||||
private IEnumerable<DateTime> GetDates(DateTime fromDate, DateTime toDate)
|
||||
{
|
||||
for (var d = new DateTime(fromDate.Year, fromDate.Month, fromDate.Day); d <= toDate; d = d.AddDays(1))
|
||||
{
|
||||
yield return d;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public async IAsyncEnumerable<MetricResult> GetRecords(DateTime fromDate, DateTime toDate, string action)
|
||||
{
|
||||
ulong GetMetricKey(string key)
|
||||
{
|
||||
var hash = new xxHashAlgorithm(0);
|
||||
Span<byte> bytes = stackalloc byte[key.Length];
|
||||
Encoding.ASCII.GetBytes(key, bytes);
|
||||
return hash.HashBytes(bytes);
|
||||
}
|
||||
|
||||
foreach (var file in GetFiles(fromDate, toDate))
|
||||
{
|
||||
await foreach (var line in file.ReadAllLinesAsync())
|
||||
{
|
||||
var metric = _dtos.Deserialize<Metric>(line)!;
|
||||
if (metric.Action != action) continue;
|
||||
if (metric.Timestamp >= fromDate && metric.Timestamp <= toDate)
|
||||
{
|
||||
yield return new MetricResult
|
||||
{
|
||||
Timestamp = metric.Timestamp,
|
||||
Subject = metric.Subject,
|
||||
Action = metric.Action,
|
||||
MetricKey = GetMetricKey(metric.MetricsKey),
|
||||
UserAgent = metric.UserAgent,
|
||||
GroupingSubject = GetGroupingSubject(metric.Subject)
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public ParallelQuery<MetricResult> GetRecordsParallel(DateTime fromDate, DateTime toDate, string action)
|
||||
{
|
||||
ulong GetMetricKey(string key)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(key)) return 0;
|
||||
var hash = new xxHashAlgorithm(0);
|
||||
Span<byte> bytes = stackalloc byte[key.Length];
|
||||
Encoding.ASCII.GetBytes(key, bytes);
|
||||
return hash.HashBytes(bytes);
|
||||
}
|
||||
|
||||
var rows = GetFiles(fromDate, toDate).AsParallel()
|
||||
.SelectMany(file => file.ReadAllLines())
|
||||
.Select(row => _dtos.Deserialize<Metric>(row)!)
|
||||
.Where(m => m.Action == action)
|
||||
.Where(m => m.Timestamp >= fromDate && m.Timestamp <= toDate)
|
||||
.Select(m => new MetricResult
|
||||
{
|
||||
Timestamp = m.Timestamp,
|
||||
Subject = m.Subject,
|
||||
Action = m.Action,
|
||||
MetricKey = GetMetricKey(m.MetricsKey),
|
||||
UserAgent = m.UserAgent,
|
||||
GroupingSubject = GetGroupingSubject(m.Subject)
|
||||
});
|
||||
return rows;
|
||||
}
|
||||
|
||||
private Regex groupingRegex = new("^[^0-9]*");
|
||||
private string GetGroupingSubject(string metricSubject)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = groupingRegex.Match(metricSubject).Groups[0].ToString();
|
||||
return string.IsNullOrEmpty(result) ? metricSubject : result;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return metricSubject;
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<AbsolutePath> GetFiles(DateTime fromDate, DateTime toDate)
|
||||
{
|
||||
var folder = _settings.MetricsFolder.ToAbsolutePath();
|
||||
foreach (var day in GetDates(fromDate, toDate))
|
||||
{
|
||||
var file = folder.Combine(day.ToString("yyyy_MM_dd") + ".json");
|
||||
if (file.FileExists())
|
||||
yield return file;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,48 +0,0 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer;
|
||||
using Wabbajack.Paths;
|
||||
using Wabbajack.Paths.IO;
|
||||
|
||||
namespace Wabbajack.Server.DataModels;
|
||||
|
||||
public class TarLog
|
||||
{
|
||||
private Task<HashSet<string>> _tarKeys;
|
||||
private readonly AppSettings _settings;
|
||||
private readonly ILogger<TarLog> _logger;
|
||||
|
||||
public TarLog(AppSettings settings, ILogger<TarLog> logger)
|
||||
{
|
||||
_settings = settings;
|
||||
_logger = logger;
|
||||
Load();
|
||||
}
|
||||
|
||||
private void Load()
|
||||
{
|
||||
if (_settings.TarKeyFile.ToAbsolutePath().FileExists())
|
||||
{
|
||||
_tarKeys = Task.Run(async () =>
|
||||
{
|
||||
var keys = await _settings.TarKeyFile.ToAbsolutePath()
|
||||
.ReadAllLinesAsync()
|
||||
.Select(line => line.Trim())
|
||||
.ToHashSetAsync();
|
||||
_logger.LogInformation("Loaded {Count} tar keys", keys.Count);
|
||||
return keys;
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
_tarKeys = Task.Run(async () => new HashSet<string>());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public async Task<bool> Contains(string metricsKey)
|
||||
{
|
||||
return (await _tarKeys).Contains(metricsKey);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,47 +0,0 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Web;
|
||||
using Nettle.Compiler;
|
||||
using Nettle.Functions;
|
||||
|
||||
namespace Wabbajack.Server.Extensions;
|
||||
|
||||
public static class NettleFunctions
|
||||
{
|
||||
public static INettleCompiler RegisterWJFunctions(this INettleCompiler compiler)
|
||||
{
|
||||
compiler.RegisterFunction(new Escape());
|
||||
return compiler;
|
||||
}
|
||||
|
||||
private sealed class UrlEncode : FunctionBase
|
||||
{
|
||||
public UrlEncode() : base()
|
||||
{
|
||||
DefineRequiredParameter("text", "text to encode", typeof(string));
|
||||
}
|
||||
|
||||
protected override object GenerateOutput(TemplateContext context, params object[] parameterValues)
|
||||
{
|
||||
var value = GetParameterValue<string>("text", parameterValues);
|
||||
return HttpUtility.UrlEncode(value);
|
||||
}
|
||||
|
||||
public override string Description => "URL encodes a string";
|
||||
}
|
||||
|
||||
private sealed class Escape : FunctionBase
|
||||
{
|
||||
public Escape() : base()
|
||||
{
|
||||
DefineRequiredParameter("text", "text to escape", typeof(string));
|
||||
}
|
||||
|
||||
protected override object GenerateOutput(TemplateContext context, params object[] parameterValues)
|
||||
{
|
||||
var value = GetParameterValue<string>("text", parameterValues);
|
||||
return Regex.Escape(value).Replace("'", "\\'");
|
||||
}
|
||||
|
||||
public override string Description => "Escapes a string";
|
||||
}
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
using System;
|
||||
|
||||
namespace Wabbajack.Server;
|
||||
|
||||
public class GlobalInformation
|
||||
{
|
||||
public TimeSpan NexusAPIPollRate = TimeSpan.FromMinutes(15);
|
||||
public TimeSpan NexusRSSPollRate = TimeSpan.FromMinutes(1);
|
||||
public DateTime LastNexusSyncUTC { get; set; }
|
||||
public TimeSpan TimeSinceLastNexusSync => DateTime.UtcNow - LastNexusSyncUTC;
|
||||
}
|
||||
@ -1,31 +0,0 @@
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
|
||||
namespace Wabbajack.Server;
|
||||
|
||||
public class Program
|
||||
{
|
||||
public static void Main(string[] args)
|
||||
{
|
||||
var testMode = args.Contains("TESTMODE");
|
||||
CreateHostBuilder(args, testMode).Build().Run();
|
||||
}
|
||||
|
||||
public static IHostBuilder CreateHostBuilder(string[] args, bool testMode)
|
||||
{
|
||||
return Host.CreateDefaultBuilder(args)
|
||||
.ConfigureWebHostDefaults(webBuilder =>
|
||||
{
|
||||
webBuilder.UseStartup<Startup>()
|
||||
.UseKestrel(options =>
|
||||
{
|
||||
options.AllowSynchronousIO = true;
|
||||
options.Listen(IPAddress.Any, 5000);
|
||||
options.Limits.MaxRequestBodySize = null;
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -1,45 +0,0 @@
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Authored Files Report</title>
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@4.3.1/dist/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
|
||||
<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.12.1/css/jquery.dataTables.css">
|
||||
<script type="text/javascript" charset="utf8" src="https://code.jquery.com/jquery-3.5.1.js"></script>
|
||||
<script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.12.1/js/jquery.dataTables.js"></script>
|
||||
<script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.12.1/js/dataTables.bootstrap4.min.js"></script>
|
||||
<script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/plug-ins/1.12.1/sorting/file-size.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<H1>Authored Files:</H1>
|
||||
<H3>{{$.UsedSpace}}</H3>
|
||||
<table id="inlined-data" class="table table-striped table-bordered" style="width:100%" >
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Size</th>
|
||||
<th>Author</th>
|
||||
<th>Updated</th>
|
||||
<th>Direct Link</th>
|
||||
</tr>
|
||||
</thead>
|
||||
{{each $.Files }}
|
||||
<tr>
|
||||
<td><a href='https://authored-files.wabbajack.org/{{$.Definition.MungedName}}'>{{$.Definition.OriginalFileName}}</a></td>
|
||||
<td>{{$.HumanSize}}</td>
|
||||
<td>{{$.Definition.Author}}</td>
|
||||
<td>{{$.Updated}}</td>
|
||||
<td><a href='https://workers.wabbajack.workers.dev/authored_files/stream/{{$.Definition.MungedName}}'>(Slow) HTTP Direct Link</a></td>
|
||||
</tr>
|
||||
{{/each}}
|
||||
</table>
|
||||
|
||||
<script>
|
||||
$(document).ready( function () {
|
||||
$("#inlined-data").DataTable({
|
||||
columnDefs: [
|
||||
{ type: 'file-size', targets: 1},
|
||||
]
|
||||
});
|
||||
} );
|
||||
</script>
|
||||
</body></html>
|
||||
@ -1,107 +0,0 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer;
|
||||
|
||||
namespace Wabbajack.Server.Services;
|
||||
|
||||
public interface IStartable
|
||||
{
|
||||
public Task Start();
|
||||
}
|
||||
|
||||
public interface IReportingService
|
||||
{
|
||||
public TimeSpan Delay { get; }
|
||||
public DateTime LastStart { get; }
|
||||
public DateTime LastEnd { get; }
|
||||
|
||||
public (string, DateTime)[] ActiveWorkStatus { get; }
|
||||
}
|
||||
|
||||
public abstract class AbstractService<TP, TR> : IStartable, IReportingService
|
||||
{
|
||||
protected ILogger<TP> _logger;
|
||||
protected QuickSync _quickSync;
|
||||
protected AppSettings _settings;
|
||||
|
||||
public AbstractService(ILogger<TP> logger, AppSettings settings, QuickSync quickSync, TimeSpan delay)
|
||||
{
|
||||
_settings = settings;
|
||||
Delay = delay;
|
||||
_logger = logger;
|
||||
_quickSync = quickSync;
|
||||
}
|
||||
|
||||
public TimeSpan Delay { get; }
|
||||
|
||||
public DateTime LastStart { get; private set; }
|
||||
public DateTime LastEnd { get; private set; }
|
||||
public (string, DateTime)[] ActiveWorkStatus { get; private set; } = { };
|
||||
|
||||
public async Task Start()
|
||||
{
|
||||
await Setup();
|
||||
await _quickSync.Register(this);
|
||||
|
||||
while (true)
|
||||
{
|
||||
await _quickSync.ResetToken<TP>();
|
||||
try
|
||||
{
|
||||
_logger.LogInformation($"Running: {GetType().Name}");
|
||||
ActiveWorkStatus = Array.Empty<(string, DateTime)>();
|
||||
LastStart = DateTime.UtcNow;
|
||||
await Execute();
|
||||
LastEnd = DateTime.UtcNow;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Running Service Loop");
|
||||
}
|
||||
|
||||
var token = await _quickSync.GetToken<TP>();
|
||||
try
|
||||
{
|
||||
await Task.Delay(Delay, token);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public virtual async Task Setup()
|
||||
{
|
||||
}
|
||||
|
||||
public abstract Task<TR> Execute();
|
||||
|
||||
protected void ReportStarting(string value)
|
||||
{
|
||||
lock (this)
|
||||
{
|
||||
ActiveWorkStatus = ActiveWorkStatus.Append((value, DateTime.UtcNow)).ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
protected void ReportEnding(string value)
|
||||
{
|
||||
lock (this)
|
||||
{
|
||||
ActiveWorkStatus = ActiveWorkStatus.Where(x => x.Item1 != value).ToArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static class AbstractServiceExtensions
|
||||
{
|
||||
public static void UseService<T>(this IApplicationBuilder b)
|
||||
{
|
||||
var poll = (IStartable) b.ApplicationServices.GetRequiredService(typeof(T));
|
||||
poll.Start();
|
||||
}
|
||||
}
|
||||
@ -1,104 +0,0 @@
|
||||
using Discord;
|
||||
using Discord.WebSocket;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer;
|
||||
|
||||
namespace Wabbajack.Server.Services;
|
||||
|
||||
public class DiscordBackend
|
||||
{
|
||||
private readonly AppSettings _settings;
|
||||
private readonly ILogger<DiscordBackend> _logger;
|
||||
private readonly DiscordSocketClient _client;
|
||||
private readonly NexusCacheManager _nexusCacheManager;
|
||||
|
||||
public DiscordBackend(ILogger<DiscordBackend> logger, AppSettings settings, NexusCacheManager nexusCacheManager)
|
||||
{
|
||||
_settings = settings;
|
||||
_logger = logger;
|
||||
_nexusCacheManager = nexusCacheManager;
|
||||
_client = new DiscordSocketClient(new DiscordSocketConfig()
|
||||
{
|
||||
|
||||
});
|
||||
_client.Log += LogAsync;
|
||||
_client.Ready += ReadyAsync;
|
||||
_client.MessageReceived += MessageReceivedAsync;
|
||||
Task.Run(async () =>
|
||||
{
|
||||
await _client.LoginAsync(TokenType.Bot, settings.DiscordKey);
|
||||
await _client.StartAsync();
|
||||
});
|
||||
}
|
||||
|
||||
private async Task MessageReceivedAsync(SocketMessage arg)
|
||||
{
|
||||
_logger.LogInformation(arg.Content);
|
||||
|
||||
if (arg.Content.StartsWith("!dervenin"))
|
||||
{
|
||||
var parts = arg.Content.Split(" ", StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts[0] != "!dervenin")
|
||||
return;
|
||||
|
||||
if (parts.Length == 1)
|
||||
{
|
||||
await ReplyTo(arg, "Wat?");
|
||||
}
|
||||
|
||||
if (parts[1] == "purge-nexus-cache")
|
||||
{
|
||||
if (parts.Length != 3)
|
||||
{
|
||||
await ReplyTo(arg, "Welp you did that wrong, gotta give me a mod-id or url");
|
||||
return;
|
||||
}
|
||||
var rows = await _nexusCacheManager.Purge(parts[2]);
|
||||
await ReplyTo(arg, $"Purged {rows} rows");
|
||||
}
|
||||
|
||||
if (parts[1] == "nft")
|
||||
{
|
||||
await ReplyTo(arg, "No Fucking Thanks.");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReplyTo(SocketMessage socketMessage, string message)
|
||||
{
|
||||
await socketMessage.Channel.SendMessageAsync(message);
|
||||
}
|
||||
|
||||
|
||||
private async Task ReadyAsync()
|
||||
{
|
||||
}
|
||||
|
||||
private async Task LogAsync(LogMessage arg)
|
||||
{
|
||||
switch (arg.Severity)
|
||||
{
|
||||
case LogSeverity.Info:
|
||||
_logger.LogInformation(arg.Message);
|
||||
break;
|
||||
case LogSeverity.Warning:
|
||||
_logger.LogWarning(arg.Message);
|
||||
break;
|
||||
case LogSeverity.Critical:
|
||||
_logger.LogCritical(arg.Message);
|
||||
break;
|
||||
case LogSeverity.Error:
|
||||
_logger.LogError(arg.Exception, arg.Message);
|
||||
break;
|
||||
case LogSeverity.Verbose:
|
||||
_logger.LogTrace(arg.Message);
|
||||
break;
|
||||
case LogSeverity.Debug:
|
||||
_logger.LogDebug(arg.Message);
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,83 +0,0 @@
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.DTOs.JsonConverters;
|
||||
using Wabbajack.Server.DTOs;
|
||||
|
||||
namespace Wabbajack.Server.Services;
|
||||
|
||||
public enum Channel
|
||||
{
|
||||
// High volume messaging, really only useful for internal devs
|
||||
Spam,
|
||||
|
||||
// Low volume messages designed for admins
|
||||
Ham
|
||||
}
|
||||
|
||||
public class DiscordWebHook : AbstractService<DiscordWebHook, int>
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
private readonly DTOSerializer _dtos;
|
||||
private readonly Random _random = new();
|
||||
|
||||
public DiscordWebHook(ILogger<DiscordWebHook> logger, AppSettings settings, QuickSync quickSync, HttpClient client,
|
||||
DTOSerializer dtos) : base(logger, settings, quickSync, TimeSpan.FromHours(1))
|
||||
{
|
||||
_settings = settings;
|
||||
_logger = logger;
|
||||
_client = client;
|
||||
_dtos = dtos;
|
||||
|
||||
Task.Run(async () =>
|
||||
{
|
||||
|
||||
var message = new DiscordMessage
|
||||
{
|
||||
Content = $"\"{await GetQuote()}\" - Sheogorath (as he brings the server online)"
|
||||
};
|
||||
await Send(Channel.Ham, message);
|
||||
await Send(Channel.Spam, message);
|
||||
});
|
||||
}
|
||||
|
||||
public async Task Send(Channel channel, DiscordMessage message)
|
||||
{
|
||||
try
|
||||
{
|
||||
var url = channel switch
|
||||
{
|
||||
Channel.Spam => _settings.SpamWebHook,
|
||||
Channel.Ham => _settings.HamWebHook,
|
||||
_ => null
|
||||
};
|
||||
if (string.IsNullOrWhiteSpace(url)) return;
|
||||
|
||||
await _client.PostAsync(url,
|
||||
new StringContent(_dtos.Serialize(message), Encoding.UTF8, "application/json"));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "While sending discord message");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> GetQuote()
|
||||
{
|
||||
var lines =
|
||||
await Assembly.GetExecutingAssembly()!.GetManifestResourceStream("Wabbajack.Server.sheo_quotes.txt")!
|
||||
.ReadLinesAsync()
|
||||
.ToList();
|
||||
return lines[_random.Next(lines.Count)].Trim();
|
||||
}
|
||||
|
||||
public override async Task<int> Execute()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@ -1,172 +0,0 @@
|
||||
using System.Text.Json;
|
||||
using K4os.Compression.LZ4.Internal;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.DTOs;
|
||||
using Wabbajack.DTOs.JsonConverters;
|
||||
using Wabbajack.Networking.NexusApi;
|
||||
using Wabbajack.Networking.NexusApi.DTOs;
|
||||
using Wabbajack.Paths;
|
||||
using Wabbajack.Paths.IO;
|
||||
using Wabbajack.Server.DTOs;
|
||||
|
||||
namespace Wabbajack.Server.Services;
|
||||
|
||||
public class NexusCacheManager
|
||||
{
|
||||
private readonly ILogger<NexusCacheManager> _loggger;
|
||||
private readonly DTOSerializer _dtos;
|
||||
private readonly AppSettings _configuration;
|
||||
private readonly AbsolutePath _cacheFolder;
|
||||
private readonly SemaphoreSlim _lockObject;
|
||||
private readonly NexusApi _nexusAPI;
|
||||
private readonly Timer _timer;
|
||||
private readonly DiscordWebHook _discord;
|
||||
|
||||
public NexusCacheManager(ILogger<NexusCacheManager> logger, DTOSerializer dtos, AppSettings configuration, NexusApi nexusApi, DiscordWebHook discord)
|
||||
{
|
||||
_loggger = logger;
|
||||
_dtos = dtos;
|
||||
_configuration = configuration;
|
||||
_cacheFolder = configuration.NexusCacheFolder.ToAbsolutePath();
|
||||
_lockObject = new SemaphoreSlim(1);
|
||||
_nexusAPI = nexusApi;
|
||||
_discord = discord;
|
||||
|
||||
if (configuration.RunBackendNexusRoutines)
|
||||
{
|
||||
_timer = new Timer(_ => UpdateNexusCacheAPI().FireAndForget(), null, TimeSpan.FromSeconds(2),
|
||||
TimeSpan.FromHours(4));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private AbsolutePath CacheFile(string key)
|
||||
{
|
||||
return _cacheFolder.Combine(key).WithExtension(Ext.Json);
|
||||
}
|
||||
|
||||
|
||||
private bool HaveCache(string key)
|
||||
{
|
||||
return CacheFile(key).FileExists();
|
||||
}
|
||||
|
||||
public async Task SaveCache<T>(string key, T value, CancellationToken token)
|
||||
{
|
||||
var ms = new MemoryStream();
|
||||
await JsonSerializer.SerializeAsync(ms, value, _dtos.Options, token);
|
||||
await ms.FlushAsync(token);
|
||||
var data = ms.ToArray();
|
||||
await _lockObject.WaitAsync(token);
|
||||
try
|
||||
{
|
||||
await CacheFile(key).WriteAllBytesAsync(data, token: token);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lockObject.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<T?> GetCache<T>(string key, CancellationToken token)
|
||||
{
|
||||
if (!HaveCache(key)) return default;
|
||||
|
||||
var file = CacheFile(key);
|
||||
await _lockObject.WaitAsync(token);
|
||||
byte[] data;
|
||||
try
|
||||
{
|
||||
data = await file.ReadAllBytesAsync(token);
|
||||
}
|
||||
catch (FileNotFoundException)
|
||||
{
|
||||
return default;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lockObject.Release();
|
||||
}
|
||||
return await JsonSerializer.DeserializeAsync<T>(new MemoryStream(data), _dtos.Options, token);
|
||||
}
|
||||
|
||||
public async Task UpdateNexusCacheAPI()
|
||||
{
|
||||
var gameTasks = GameRegistry.Games.Values
|
||||
.Where(g => g.NexusName != null)
|
||||
.SelectAsync(async game =>
|
||||
{
|
||||
var mods = await _nexusAPI.GetUpdates(game.Game, CancellationToken.None);
|
||||
return (game, mods);
|
||||
});
|
||||
|
||||
|
||||
var purgeList = new List<(string Key, DateTime Date)>();
|
||||
|
||||
await foreach (var (game, mods) in gameTasks)
|
||||
{
|
||||
foreach (var mod in mods.Item1)
|
||||
{
|
||||
var date = Math.Max(mod.LastestModActivity, mod.LatestFileUpdate).AsUnixTime();
|
||||
purgeList.Add(($"_{game.Game.MetaData().NexusName!.ToLowerInvariant()}_{mod.ModId}_", date));
|
||||
}
|
||||
}
|
||||
|
||||
// This is O(m * n) where n and m are 15,000 items, we really should improve this
|
||||
var files = (from file in _cacheFolder.EnumerateFiles().AsParallel()
|
||||
from entry in purgeList
|
||||
where file.FileName.ToString().Contains(entry.Key)
|
||||
where file.LastModifiedUtc() < entry.Date
|
||||
select file).ToHashSet();
|
||||
|
||||
foreach (var file in files)
|
||||
{
|
||||
await PurgeCacheEntry(file);
|
||||
}
|
||||
|
||||
await _discord.Send(Channel.Ham, new DiscordMessage
|
||||
{
|
||||
Content = $"Cleared {files.Count} Nexus cache entries due to updates"
|
||||
});
|
||||
}
|
||||
|
||||
private async Task PurgeCacheEntry(AbsolutePath file)
|
||||
{
|
||||
await _lockObject.WaitAsync();
|
||||
try
|
||||
{
|
||||
if (file.FileExists()) file.Delete();
|
||||
}
|
||||
catch (FileNotFoundException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lockObject.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<int> Purge(string mod)
|
||||
{
|
||||
if (Uri.TryCreate(mod, UriKind.Absolute, out var url))
|
||||
{
|
||||
mod = Enumerable.Last(url.AbsolutePath.Split("/", StringSplitOptions.RemoveEmptyEntries));
|
||||
}
|
||||
|
||||
var count = 0;
|
||||
if (!int.TryParse(mod, out var mod_id)) return count;
|
||||
|
||||
foreach (var file in _cacheFolder.EnumerateFiles())
|
||||
{
|
||||
if (!file.FileName.ToString().Contains($"_{mod_id}")) continue;
|
||||
|
||||
await PurgeCacheEntry(file);
|
||||
count++;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
}
|
||||
@ -1,69 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.Common;
|
||||
|
||||
namespace Wabbajack.Server.Services;
|
||||
|
||||
public class QuickSync
|
||||
{
|
||||
private readonly AsyncLock _lock = new();
|
||||
private readonly ILogger<QuickSync> _logger;
|
||||
private readonly Dictionary<Type, IReportingService> _services = new();
|
||||
private readonly Dictionary<Type, CancellationTokenSource> _syncs = new();
|
||||
|
||||
public QuickSync(ILogger<QuickSync> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<Dictionary<Type, (TimeSpan Delay, TimeSpan LastRunTime, (string, DateTime)[] ActiveWork)>>
|
||||
Report()
|
||||
{
|
||||
using var _ = await _lock.WaitAsync();
|
||||
return _services.ToDictionary(s => s.Key,
|
||||
s => (s.Value.Delay, DateTime.UtcNow - s.Value.LastEnd, s.Value.ActiveWorkStatus));
|
||||
}
|
||||
|
||||
public async Task Register<T>(T service)
|
||||
where T : IReportingService
|
||||
{
|
||||
using var _ = await _lock.WaitAsync();
|
||||
_services[service.GetType()] = service;
|
||||
}
|
||||
|
||||
public async Task<CancellationToken> GetToken<T>()
|
||||
{
|
||||
using var _ = await _lock.WaitAsync();
|
||||
if (_syncs.TryGetValue(typeof(T), out var result)) return result.Token;
|
||||
var token = new CancellationTokenSource();
|
||||
_syncs[typeof(T)] = token;
|
||||
return token.Token;
|
||||
}
|
||||
|
||||
public async Task ResetToken<T>()
|
||||
{
|
||||
using var _ = await _lock.WaitAsync();
|
||||
if (_syncs.TryGetValue(typeof(T), out var ct)) ct.Cancel();
|
||||
_syncs[typeof(T)] = new CancellationTokenSource();
|
||||
}
|
||||
|
||||
public async Task Notify<T>()
|
||||
{
|
||||
_logger.LogInformation($"Quicksync {typeof(T).Name}");
|
||||
// Needs debugging
|
||||
using var _ = await _lock.WaitAsync();
|
||||
if (_syncs.TryGetValue(typeof(T), out var ct)) ct.Cancel();
|
||||
}
|
||||
|
||||
public async Task Notify(Type t)
|
||||
{
|
||||
_logger.LogInformation($"Quicksync {t.Name}");
|
||||
// Needs debugging
|
||||
using var _ = await _lock.WaitAsync();
|
||||
if (_syncs.TryGetValue(t, out var ct)) ct.Cancel();
|
||||
}
|
||||
}
|
||||
@ -1,278 +0,0 @@
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Net.Http;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading.Tasks;
|
||||
using Amazon.Runtime;
|
||||
using Amazon.S3;
|
||||
using Amazon.Util.Internal;
|
||||
using cesi.DTOs;
|
||||
using CouchDB.Driver;
|
||||
using CouchDB.Driver.Options;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Http.Features;
|
||||
using Microsoft.AspNetCore.ResponseCompression;
|
||||
using Microsoft.AspNetCore.StaticFiles;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.FileProviders;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Nettle;
|
||||
using Nettle.Compiler;
|
||||
using Newtonsoft.Json;
|
||||
using Octokit;
|
||||
using Wabbajack.BuildServer;
|
||||
using Wabbajack.Configuration;
|
||||
using Wabbajack.Downloaders;
|
||||
using Wabbajack.Downloaders.VerificationCache;
|
||||
using Wabbajack.DTOs;
|
||||
using Wabbajack.DTOs.JsonConverters;
|
||||
using Wabbajack.DTOs.Logins;
|
||||
using Wabbajack.Networking.GitHub;
|
||||
using Wabbajack.Networking.Http;
|
||||
using Wabbajack.Networking.Http.Interfaces;
|
||||
using Wabbajack.Networking.NexusApi;
|
||||
using Wabbajack.Paths;
|
||||
using Wabbajack.RateLimiter;
|
||||
using Wabbajack.Server.DataModels;
|
||||
using Wabbajack.Server.Extensions;
|
||||
using Wabbajack.Server.Services;
|
||||
using Wabbajack.Services.OSIntegrated.TokenProviders;
|
||||
using Wabbajack.Networking.WabbajackClientApi;
|
||||
using Wabbajack.Paths.IO;
|
||||
using Wabbajack.VFS;
|
||||
using YamlDotNet.Serialization.NamingConventions;
|
||||
using Client = Wabbajack.Networking.GitHub.Client;
|
||||
using Metric = Wabbajack.Server.DTOs.Metric;
|
||||
using SettingsManager = Wabbajack.Services.OSIntegrated.SettingsManager;
|
||||
|
||||
namespace Wabbajack.Server;
|
||||
|
||||
public class TestStartup : Startup
|
||||
{
|
||||
public TestStartup(IConfiguration configuration) : base(configuration)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
public class Startup
|
||||
{
|
||||
public Startup(IConfiguration configuration)
|
||||
{
|
||||
Configuration = configuration;
|
||||
}
|
||||
|
||||
public IConfiguration Configuration { get; }
|
||||
|
||||
// This method gets called by the runtime. Use this method to add services to the container.
|
||||
public void ConfigureServices(IServiceCollection services)
|
||||
{
|
||||
services.AddAuthentication(options =>
|
||||
{
|
||||
options.DefaultAuthenticateScheme = ApiKeyAuthenticationOptions.DefaultScheme;
|
||||
options.DefaultChallengeScheme = ApiKeyAuthenticationOptions.DefaultScheme;
|
||||
})
|
||||
.AddApiKeySupport(options => { });
|
||||
|
||||
services.Configure<FormOptions>(x =>
|
||||
{
|
||||
x.ValueLengthLimit = int.MaxValue;
|
||||
x.MultipartBodyLengthLimit = int.MaxValue;
|
||||
});
|
||||
|
||||
services.AddSingleton<AppSettings>();
|
||||
services.AddSingleton<QuickSync>();
|
||||
services.AddSingleton<GlobalInformation>();
|
||||
services.AddSingleton<DiscordWebHook>();
|
||||
services.AddSingleton<Metrics>();
|
||||
services.AddSingleton<HttpClient>();
|
||||
services.AddSingleton<AuthorFiles>();
|
||||
services.AddSingleton<AuthorKeys>();
|
||||
services.AddSingleton<Client>();
|
||||
services.AddSingleton<NexusCacheManager>();
|
||||
services.AddSingleton<NexusApi>();
|
||||
services.AddSingleton<DiscordBackend>();
|
||||
services.AddSingleton<TarLog>();
|
||||
services.AddAllSingleton<IHttpDownloader, SingleThreadedDownloader>();
|
||||
services.AddDownloadDispatcher(useLoginDownloaders:false, useProxyCache:false);
|
||||
services.AddSingleton<IAmazonS3>(s =>
|
||||
{
|
||||
var appSettings = s.GetRequiredService<AppSettings>();
|
||||
var settings = new BasicAWSCredentials(appSettings.S3.AccessKey,
|
||||
appSettings.S3.SecretKey);
|
||||
return new AmazonS3Client(settings, new AmazonS3Config
|
||||
{
|
||||
ServiceURL = appSettings.S3.ServiceUrl,
|
||||
});
|
||||
});
|
||||
services.AddTransient(s =>
|
||||
{
|
||||
var settings = s.GetRequiredService<AppSettings>();
|
||||
return new TemporaryFileManager(settings.TempPath.Combine(Environment.ProcessId + "_" + Guid.NewGuid()));
|
||||
});
|
||||
services.AddSingleton<IVerificationCache, NullCache>();
|
||||
|
||||
services.AddAllSingleton<ITokenProvider<WabbajackApiState>, WabbajackApiTokenProvider>();
|
||||
services.AddAllSingleton<IResource, IResource<DownloadDispatcher>>(s => new Resource<DownloadDispatcher>("Downloads", 12));
|
||||
services.AddAllSingleton<IResource, IResource<FileHashCache>>(s => new Resource<FileHashCache>("File Hashing", 12));
|
||||
services.AddAllSingleton<IResource, IResource<Wabbajack.Networking.WabbajackClientApi.Client>>(s =>
|
||||
new Resource<Wabbajack.Networking.WabbajackClientApi.Client>("Wabbajack Client", 4));
|
||||
|
||||
services.AddSingleton(s =>
|
||||
new FileHashCache(KnownFolders.AppDataLocal.Combine("Wabbajack", "GlobalHashCache.sqlite"),
|
||||
s.GetService<IResource<FileHashCache>>()!));
|
||||
|
||||
services.AddAllSingleton<ITokenProvider<NexusApiState>, NexusApiTokenProvider>();
|
||||
services.AddAllSingleton<IResource, IResource<HttpClient>>(s => new Resource<HttpClient>("Web Requests", 12));
|
||||
// Application Info
|
||||
|
||||
var version =
|
||||
$"{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Patch}{ThisAssembly.Git.SemVer.DashLabel}";
|
||||
services.AddSingleton(s => new ApplicationInfo
|
||||
{
|
||||
ApplicationSlug = "Wabbajack",
|
||||
ApplicationName = Environment.ProcessPath?.ToAbsolutePath().FileName.ToString() ?? "Wabbajack",
|
||||
ApplicationSha = ThisAssembly.Git.Sha,
|
||||
Platform = RuntimeInformation.ProcessArchitecture.ToString(),
|
||||
OperatingSystemDescription = RuntimeInformation.OSDescription,
|
||||
RuntimeIdentifier = RuntimeInformation.RuntimeIdentifier,
|
||||
OSVersion = Environment.OSVersion.VersionString,
|
||||
Version = version
|
||||
});
|
||||
|
||||
|
||||
services.AddResponseCaching();
|
||||
services.AddSingleton(s =>
|
||||
{
|
||||
var settings = s.GetService<AppSettings>()!;
|
||||
if (string.IsNullOrWhiteSpace(settings.GitHubKey))
|
||||
return new GitHubClient(new ProductHeaderValue("wabbajack"));
|
||||
|
||||
var creds = new Credentials(settings.GitHubKey);
|
||||
return new GitHubClient(new ProductHeaderValue("wabbajack")) {Credentials = creds};
|
||||
});
|
||||
services.AddDTOSerializer();
|
||||
services.AddDTOConverters();
|
||||
|
||||
services.AddSingleton(s => new Wabbajack.Services.OSIntegrated.Configuration
|
||||
{
|
||||
EncryptedDataLocation = KnownFolders.WabbajackAppLocal.Combine("encrypted"),
|
||||
ModListsDownloadLocation = KnownFolders.EntryPoint.Combine("downloaded_mod_lists"),
|
||||
SavedSettingsLocation = KnownFolders.WabbajackAppLocal.Combine("saved_settings"),
|
||||
LogLocation = KnownFolders.LauncherAwarePath.Combine("logs"),
|
||||
ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache")
|
||||
});
|
||||
|
||||
|
||||
services.AddSingleton<SettingsManager>();
|
||||
services.AddSingleton<MainSettings>(s => Wabbajack.Services.OSIntegrated.ServiceExtensions.GetAppSettings(s, MainSettings.SettingsFileName));
|
||||
|
||||
services.AddResponseCompression(options =>
|
||||
{
|
||||
options.Providers.Add<BrotliCompressionProvider>();
|
||||
options.Providers.Add<GzipCompressionProvider>();
|
||||
options.MimeTypes = new[] {"application/json"};
|
||||
});
|
||||
|
||||
// CouchDB
|
||||
services.AddSingleton(s =>
|
||||
{
|
||||
var settings = s.GetRequiredService<AppSettings>().CesiDB;
|
||||
var client = new CouchClient(settings.Endpoint, b =>
|
||||
{
|
||||
b.UseBasicAuthentication(settings.Username, settings.Password);
|
||||
b.SetPropertyCase(PropertyCaseType.None);
|
||||
b.SetJsonNullValueHandling(NullValueHandling.Ignore);
|
||||
});
|
||||
return client.GetDatabase<Analyzed>(settings.Database);
|
||||
});
|
||||
|
||||
services.AddSingleton(s =>
|
||||
{
|
||||
var settings = s.GetRequiredService<AppSettings>().MetricsDB;
|
||||
var client = new CouchClient(settings.Endpoint, b =>
|
||||
{
|
||||
b.UseBasicAuthentication(settings.Username, settings.Password);
|
||||
b.SetPropertyCase(PropertyCaseType.None);
|
||||
b.SetJsonNullValueHandling(NullValueHandling.Ignore);
|
||||
});
|
||||
return client.GetDatabase<Metric>(settings.Database);
|
||||
});
|
||||
|
||||
services.AddMvc();
|
||||
services
|
||||
.AddControllers()
|
||||
.AddJsonOptions(j =>
|
||||
{
|
||||
j.JsonSerializerOptions.PropertyNamingPolicy = null;
|
||||
j.JsonSerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault;
|
||||
});
|
||||
|
||||
NettleEngine.GetCompiler().RegisterWJFunctions();
|
||||
}
|
||||
|
||||
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
|
||||
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
|
||||
{
|
||||
if (env.IsDevelopment()) app.UseDeveloperExceptionPage();
|
||||
|
||||
app.UseDeveloperExceptionPage();
|
||||
|
||||
var provider = new FileExtensionContentTypeProvider();
|
||||
provider.Mappings[".rar"] = "application/x-rar-compressed";
|
||||
provider.Mappings[".7z"] = "application/x-7z-compressed";
|
||||
provider.Mappings[".zip"] = "application/zip";
|
||||
provider.Mappings[".wabbajack"] = "application/zip";
|
||||
app.UseStaticFiles();
|
||||
|
||||
app.UseRouting();
|
||||
|
||||
app.UseAuthentication();
|
||||
app.UseAuthorization();
|
||||
app.UseResponseCompression();
|
||||
|
||||
app.UseService<DiscordWebHook>();
|
||||
|
||||
app.UseResponseCaching();
|
||||
|
||||
app.Use(next =>
|
||||
{
|
||||
return async context =>
|
||||
{
|
||||
var stopWatch = new Stopwatch();
|
||||
stopWatch.Start();
|
||||
context.Response.OnStarting(() =>
|
||||
{
|
||||
stopWatch.Stop();
|
||||
var headers = context.Response.Headers;
|
||||
headers.Add("Access-Control-Allow-Origin", "*");
|
||||
headers.Add("Access-Control-Allow-Methods", "POST, GET");
|
||||
headers.Add("Access-Control-Allow-Headers", "Accept, Origin, Content-type");
|
||||
headers.Add("X-ResponseTime-Ms", stopWatch.ElapsedMilliseconds.ToString());
|
||||
if (!headers.ContainsKey("Cache-Control"))
|
||||
headers.Add("Cache-Control", "no-cache");
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
await next(context);
|
||||
};
|
||||
});
|
||||
|
||||
app.UseFileServer(new FileServerOptions
|
||||
{
|
||||
FileProvider = new PhysicalFileProvider(
|
||||
Path.Combine(Directory.GetCurrentDirectory(), "public")),
|
||||
StaticFileOptions = {ServeUnknownFileTypes = true}
|
||||
});
|
||||
|
||||
app.UseEndpoints(endpoints => { endpoints.MapControllers(); });
|
||||
|
||||
// Trigger the internal update code
|
||||
app.ApplicationServices.GetRequiredService<NexusCacheManager>();
|
||||
app.ApplicationServices.GetRequiredService<DiscordBackend>();
|
||||
|
||||
app.ApplicationServices.GetRequiredService<AuthorFiles>();
|
||||
}
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
using Wabbajack.Networking.Http.Interfaces;
|
||||
|
||||
namespace Wabbajack.Server.TokenProviders;
|
||||
|
||||
public interface IDiscordToken : ITokenProvider<string>
|
||||
{
|
||||
}
|
||||
@ -1,78 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<OutputType>Exe</OutputType>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<NoWarn>CS8600,CS8601,CS8618,CS8604,CS1998</NoWarn>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AWSSDK.S3" Version="3.7.205.9" />
|
||||
<PackageReference Include="cesi.DTOs" Version="1.0.0" />
|
||||
<PackageReference Include="Chronic.Core" Version="0.4.0" />
|
||||
<PackageReference Include="Dapper" Version="2.0.123" />
|
||||
<PackageReference Include="Discord.Net.WebSocket" Version="3.8.1" />
|
||||
<PackageReference Include="FluentFTP" Version="42.0.1" />
|
||||
<PackageReference Include="GitInfo" Version="2.2.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.11" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.3" />
|
||||
<PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="2.3.2" />
|
||||
<PackageReference Include="Nettle" Version="1.3.0" />
|
||||
<PackageReference Include="System.Data.SqlClient" Version="4.8.5" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Update="public\WABBAJACK_TEST_FILE.txt">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="appsettings.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="public\metrics.html">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="Controllers\Templates\AuthorControls.html">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="Controllers\Templates\AuthorControls.html">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Remove="sheo_quotes.txt" />
|
||||
<EmbeddedResource Include="sheo_quotes.txt" />
|
||||
<None Remove="Controllers\Templates\TotalListTemplate.html" />
|
||||
<EmbeddedResource Include="Controllers\Templates\TotalListTemplate.html" />
|
||||
<None Remove="Resources\Reports\AuthoredFiles.html" />
|
||||
<EmbeddedResource Include="Resources\Reports\AuthoredFiles.html" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Compile Remove="Controllers\UploadedFiles.cs" />
|
||||
<Compile Remove="Services\ListValidator.cs" />
|
||||
<Compile Remove="Controllers\Users.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
|
||||
<ProjectReference Include="..\Wabbajack.Compiler\Wabbajack.Compiler.csproj" />
|
||||
<ProjectReference Include="..\Wabbajack.Downloaders.Dispatcher\Wabbajack.Downloaders.Dispatcher.csproj" />
|
||||
<ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
|
||||
<ProjectReference Include="..\Wabbajack.Hashing.xxHash64\Wabbajack.Hashing.xxHash64.csproj" />
|
||||
<ProjectReference Include="..\Wabbajack.Networking.GitHub\Wabbajack.Networking.GitHub.csproj" />
|
||||
<ProjectReference Include="..\Wabbajack.Networking.NexusApi\Wabbajack.Networking.NexusApi.csproj" />
|
||||
<ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
|
||||
<ProjectReference Include="..\Wabbajack.Paths\Wabbajack.Paths.csproj" />
|
||||
<ProjectReference Include="..\Wabbajack.Services.OSIntegrated\Wabbajack.Services.OSIntegrated.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
</Project>
|
||||
@ -1,42 +0,0 @@
|
||||
{
|
||||
"Logging": {
|
||||
"LogLevel": {
|
||||
"Default": "Information",
|
||||
"Microsoft": "Warning",
|
||||
"Microsoft.Hosting.Lifetime": "Information"
|
||||
}
|
||||
},
|
||||
"WabbajackSettings": {
|
||||
"RunBackendNexusRoutines": false,
|
||||
"TempFolder": "c:\\tmp\\server_temp",
|
||||
"MetricsFolder": "c:\\tmp\\server_metrics",
|
||||
"AuthoredFilesFolder": "c:\\tmp\\server_authored_files",
|
||||
"AuthorAPIKeyFile": "c:\\tmp\\author_keys.txt",
|
||||
"PatchesFilesFolder": "c:\\tmp\\patches",
|
||||
"MirrorFilesFolder": "c:\\tmp\\mirrors",
|
||||
"NexusCacheFolder": "c:\\tmp\\nexus-cache",
|
||||
"ProxyFolder": "c:\\tmp\\proxy",
|
||||
"GitHubKey": "",
|
||||
"CesiDB": {
|
||||
"Endpoint": "http://localhost:15984",
|
||||
"Database": "cesi",
|
||||
"Username": "cesi",
|
||||
"Password": "password"
|
||||
},
|
||||
"MetricsDB": {
|
||||
"Endpoint": "http://localhost:15984",
|
||||
"Database": "metrics",
|
||||
"Username": "wabbajack",
|
||||
"Password": "password"
|
||||
},
|
||||
"S3": {
|
||||
"AccessKey": "<>",
|
||||
"SecretKey": "<>",
|
||||
"ServiceUrl": "<>",
|
||||
"ProxyFilesBucket": "proxy-files",
|
||||
"AuthoredFilesBucket": "authored-files",
|
||||
"AuthoredFilesBucketCache": "c:\\tmp\\bucket-cache.txt"
|
||||
}
|
||||
},
|
||||
"AllowedHosts": "*"
|
||||
}
|
||||
@ -1 +0,0 @@
|
||||
Cheese for Everyone!
|
||||
@ -1,117 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Wabbajack Metrics</title>
|
||||
<script
|
||||
crossorigin="anonymous"
|
||||
integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0="
|
||||
src="https://code.jquery.com/jquery-3.5.1.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js@2.8.0"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-colorschemes"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/underscore.js/1.9.1/underscore-min.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<h2>Finished Install Counts</h2>
|
||||
<canvas height="600" id="finished_install_count" width="800"></canvas>
|
||||
<hr/>
|
||||
|
||||
<h2>Begin Download</h2>
|
||||
<canvas height="600" id="begin_download_chart" width="800"></canvas>
|
||||
<hr/>
|
||||
|
||||
<h2>Begin Install</h2>
|
||||
<canvas height="600" id="begin_install_chart" width="800"></canvas>
|
||||
<hr/>
|
||||
|
||||
<h2>Finished Install</h2>
|
||||
<canvas height="600" id="finished_install_chart" width="800"></canvas>
|
||||
<hr/>
|
||||
|
||||
<h2>Started Wabbajack</h2>
|
||||
<canvas height="600" id="started_wabbajack_chart" width="800"></canvas>
|
||||
<hr/>
|
||||
|
||||
<h2>Exceptions</h2>
|
||||
<canvas height="600" id="exceptions_chart" width="800"></canvas>
|
||||
<hr/>
|
||||
|
||||
|
||||
<script>
|
||||
|
||||
var getReport = function (subject, from, callback) {
|
||||
$.getJSON("/metrics/report/?action=" + subject + "&from=" + from + "&to=now", callback)
|
||||
}
|
||||
|
||||
var makeChart = function (ele, group) {
|
||||
|
||||
var result_fn = function (data) {
|
||||
var labels = _.map(data, f => f._Timestamp);
|
||||
var datasets = _.map(_.filter(Object.keys(data[0]), key => key !== "_Timestamp"), key => {
|
||||
return {
|
||||
label: key,
|
||||
fill: false,
|
||||
data: _.map(data, row => row[key])
|
||||
}
|
||||
});
|
||||
var ctx = document.getElementById(ele).getContext('2d');
|
||||
var chart = new Chart(ctx, {
|
||||
// The type of chart we want to create
|
||||
type: 'bar',
|
||||
|
||||
// The data for our dataset
|
||||
data: {
|
||||
labels: _.last(labels, 90),
|
||||
datasets: datasets
|
||||
},
|
||||
|
||||
// Configuration options go here
|
||||
options: {scales: {xAxes: [{stacked: true}], yAxes: [{stacked: true}]}}
|
||||
});
|
||||
};
|
||||
getReport(group, "30 days ago", result_fn);
|
||||
};
|
||||
|
||||
|
||||
var makePieChart = function (ele, group) {
|
||||
|
||||
var result_fn = function (data) {
|
||||
const counts = {};
|
||||
for (var row of data) {
|
||||
for (var key of Object.keys(row)) {
|
||||
if (key === "_Timestamp") continue;
|
||||
counts[key] = (counts[key] || 0) + row[key]
|
||||
}
|
||||
}
|
||||
|
||||
var ctx = document.getElementById(ele).getContext('2d');
|
||||
var chart = new Chart(ctx, {
|
||||
// The type of chart we want to create
|
||||
type: 'pie',
|
||||
|
||||
// The data for our dataset
|
||||
data: {
|
||||
labels: Object.keys(counts),
|
||||
datasets: [{
|
||||
label: "Data",
|
||||
data: Object.values(counts)
|
||||
}]
|
||||
},
|
||||
|
||||
// Configuration options go here
|
||||
options: {}
|
||||
});
|
||||
};
|
||||
getReport(group, "10 years ago", result_fn)
|
||||
};
|
||||
|
||||
makeChart("begin_download_chart", "downloading");
|
||||
makeChart("begin_install_chart", "begin_install");
|
||||
makeChart("finished_install_chart", "finish_install");
|
||||
makeChart("started_wabbajack_chart", "started_wabbajack");
|
||||
makeChart("exceptions_chart", "Exception");
|
||||
makePieChart("finished_install_count", "finish_install");
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@ -1,55 +0,0 @@
|
||||
I see you have completed my little errand. Well done. Perhaps you’ve gotten a taste of madness aswell? Do not believe madness to be a curse, mortal. For some it is the greatest of blessings. A bitter mercy perhaps, but mercy non the less. Give me the Fork of Horripilation, I believe I have something more suitable for your needs. Go now. Remember what you have seen.
|
||||
Use the fork wisely, mortal. Few have wield to have not come away changed. Use the fork to strike a deathblow to the giant Bullnetch that resides near the hermit. Do this, return the Fork of Horripilation to me, and Sheogorath will reward you well.
|
||||
What is it, mortal? Have you come to be of the service to Sheogorath? That in and of itself speaks toward your madness. This pleases me. Fetch the Fork of Horripliation from the mad hermit near Ald Redaynia. Take care with him. He’s not the most... stable man.
|
||||
Unworthy, unworthy, unworthy! Useless mortal meat. Walking bag of dung!
|
||||
Bring me a champion! Rend the flesh of my foes! A mortal champion to wade through the entrails of my enemies!
|
||||
Really, do come in. It’s lovely in the Isles right now. Perfect time for a visit.
|
||||
Greetings! Salutations! Welcome! Now go away. Leave. Run. Or die.
|
||||
Isn't that a hoot? I love it, myself. Best part of being a Daedric Prince, really. Go ahead, try it again. He loves it!
|
||||
Marvellous, marvellous! Self-immolation is a wonderful thing, isn't it? But now that we've had our fun, off to the Sacellum with you.
|
||||
I suppose an introduction is in order. I'm Sheogorath, Prince of Madness. And other things. I'm not talking about them.
|
||||
You should be off like the wind, solving problems and doing good deeds!
|
||||
Time. Time is an artificial construct. An arbitrary system based on the idea that events occur in a linear direction at all times.
|
||||
Always forward, never back. Is the concept of time correct? Is time relevant? It matters not. One way or another, I fear that our time has run out.
|
||||
A new Gatekeeper! Excellent. We might be onto something with you, after all. That should keep out the stragglers.
|
||||
A little busy here! I'm trying to decide what to have for dinner. Oh, how I love eating. One of my favorite things to do.
|
||||
It's Jyggalag's time, and not a good time at all. You're going to help me stop it. First, though, you need to get your feet wet.
|
||||
Another Daedric Prince. Not a nice one. I don't think ANY of the other Princes like him, actually. I mean, Malacath is more popular at parties.
|
||||
The Daedric Prince of Order. Or biscuits... No. Order. And not in a good way. Bleak. Colorless. Dead. Boring, boring, boring.
|
||||
The Greymarch comes, and Jyggalag walks. Or runs. Never skips, sidles, or struts. Mostly, he just destroys everything around him.
|
||||
Once you understand what My Realm is, you might understand why it's important to keep it intact.
|
||||
Two halves, two rulers, two places. Meet and greet. Do what they will, so you know what they're about.
|
||||
Ask? ASK? I don't ask. I tell. This is My Realm, remember? My creation, My place, My rules.
|
||||
Wonderful! Time for a celebration... Cheese for everyone!
|
||||
Makes all of my subjects uneasy. Tense. Homicidal. Some of them, at least. We need to get that Torch relit, before the place falls apart.
|
||||
You're going to stop the Greymarch by becoming Me. Or a version of Me. You'll be powerful. Powerful enough to stop Jyggalag.
|
||||
You know what would be a good sign? "Free Sweetrolls!" Who wouldn't like that?
|
||||
You'll be my champion. You'll grow powerful. You'll grow to be me. Prince of Madness, a new Sheogorath. Or you'll die trying. I love that about you.
|
||||
Oh, don't forget to make use of dear Haskill. Between you and me, if he's not summoned three or four times a day, I don't think he feels appreciated.
|
||||
I hate indecision! Or maybe I don't. Make up your mind, or I'll have your skin made into a hat -- one of those arrowcatchers. I love those hats!
|
||||
So, which is it? What will it be? Mania? Dementia? The suspense is killing me. Or you, if I have to keep waiting.
|
||||
Except where the backbone is an actual backbone. Ever been to Malacath's realm...? Nasty stuff. But, back to the business at hand.
|
||||
Happens every time. The Greymarch starts, Order appears, and I become Jyggalag and wipe out My whole Realm.
|
||||
Flee while you can, mortal. When we next meet I will not know you, and I will slay you like the others.
|
||||
Ah... New Sheoth. My home away from places that aren't my home. The current location is much better than some of the prior ones. Don't you think?
|
||||
The Isles, the Isles. A wonderful place! Except when it's horrible. Then it's horribly wonderful. Good for a visit. Or for an eternity.
|
||||
Time to save the Realm! Rescue the damsel! Slay the beast! Or die trying. Your help is required.
|
||||
Daedra are the embodiment of change. Change and permanency. I'm no different, except in the ways that I am.
|
||||
Was it Molag? No, no... Little Tim, the toymaker's son? The ghost of King Lysandus? Or was it... Yes! Stanley, that talking grapefruit from Passwall.
|
||||
Reeaaaallllyyyy?
|
||||
Well? Spit it out, mortal. I haven't got an eternity! Oh, wait! I do.
|
||||
I am a part of you, little mortal. I am a shadow in your subconscious, a blemish on your fragile little psyche. You know me. You just don't know it.
|
||||
Sheogorath, Daedric Prince of Madness. At your service.
|
||||
Yaaawwwwnn....
|
||||
Oh, pardon me. Were you saying something? I do apologize, it's just that I find myself suddenly and irrevocably...
|
||||
Bored!
|
||||
I mean, really. Here you stand, before Sheogorath himself, Daedric Prince of Madness, and all you deem fit to do is... deliver a message? How sad.
|
||||
Now you. You can call me Ann Marie.
|
||||
Oh... lovely. Now all my dear Pelagius has to worry about are the several hundred legitimate threats...
|
||||
Ah, wonderful, wonderful! Why waste all that hatred on yourself when it can so easily be directed at others!
|
||||
Mortal? Insufferable.
|
||||
Yes, yes, you're entirely brilliant. Conquering madness and all that. Blah blah blah.
|
||||
Ah, so now my dear Pelagius can hate himself for being legitimately afraid of things that actually threaten his existence...
|
||||
Conquering paranoia should be a snap after that ordeal, hmm?
|
||||
Welcome to the deceptively verdant mind of the Emperor Pelagius III. That's right! You're in the head of a dead, homicidally insane monarch.
|
||||
The Wabbajack! Huh? Huh? Didn't see that coming, did you?
|
||||
@ -75,8 +75,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Compiler.Test", "
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Downloaders.WabbajackCDN", "Wabbajack.Downloaders.WabbajackCDN\Wabbajack.Downloaders.WabbajackCDN.csproj", "{0210A092-4A69-479F-8FF4-120921B5758E}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Server", "Wabbajack.Server\Wabbajack.Server.csproj", "{8F7FB1BE-4E1E-4798-AC9E-AA2E18CCDD42}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Networking.GitHub", "Wabbajack.Networking.GitHub\Wabbajack.Networking.GitHub.csproj", "{549D01FD-8E26-42C7-ABEA-4BAF6E24E754}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Services.OSIntegrated", "Wabbajack.Services.OSIntegrated\Wabbajack.Services.OSIntegrated.csproj", "{45E48012-6C58-4C3D-843F-C6EED02868B7}"
|
||||
@ -299,10 +297,6 @@ Global
|
||||
{0210A092-4A69-479F-8FF4-120921B5758E}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{0210A092-4A69-479F-8FF4-120921B5758E}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{0210A092-4A69-479F-8FF4-120921B5758E}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{8F7FB1BE-4E1E-4798-AC9E-AA2E18CCDD42}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{8F7FB1BE-4E1E-4798-AC9E-AA2E18CCDD42}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{8F7FB1BE-4E1E-4798-AC9E-AA2E18CCDD42}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{8F7FB1BE-4E1E-4798-AC9E-AA2E18CCDD42}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{549D01FD-8E26-42C7-ABEA-4BAF6E24E754}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{549D01FD-8E26-42C7-ABEA-4BAF6E24E754}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{549D01FD-8E26-42C7-ABEA-4BAF6E24E754}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
|
||||
Loading…
Reference in New Issue
Block a user