Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #849 from wabbajack-tools/small-server-fixes-and-discord

Browse Source 
Small server fixes and discord
This commit is contained in:
Timothy Baldridge 2020-05-16 09:05:51 -07:00 committed by GitHub
commit 404cdc7183
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 570 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -1,5 +1,8 @@
### Changelog
#### Version - 2.0.5.1 - 5/16/2020
* Close automatically Wabbajack when opening the CLI, resolves the "RocksDB is in use" errors
#### Version - 2.0.5.0 - 5/14/2020
* Make the CDN downloads multi-threaded
* Optimize installation of included files

4
Wabbajack.Common/Json.cs
View File

@ -63,9 +63,9 @@ namespace Wabbajack.Common
obj.ToJson(fs);
}
public static string ToJson<T>(this T obj)
public static string ToJson<T>(this T obj, bool useGenericSettings = false)
{
return JsonConvert.SerializeObject(obj, JsonSettings);
return JsonConvert.SerializeObject(obj, useGenericSettings ? GenericJsonSettings : JsonSettings);
}
public static T FromJson<T>(this AbsolutePath filename)

13
Wabbajack.Lib/AInstaller.cs
View File

@ -292,11 +292,18 @@ namespace Wabbajack.Lib
public async Task HashArchives()
{
var hashResults = await
DownloadFolder.EnumerateFiles()
Utils.Log("Looking for files to hash");
var toHash = DownloadFolder.EnumerateFiles()
.Concat(Game.GameLocation().EnumerateFiles())
.Where(e => e.Extension != Consts.HashFileExtension)
.PMap(Queue, async e => (await e.FileHashCachedAsync(), e));
.ToList();
Utils.Log($"Found {toHash} files to hash");
var hashResults = await
toHash
.PMap(Queue, async e => (await e.FileHashCachedAsync(), e));
HashedArchives.SetTo(hashResults
.OrderByDescending(e => e.Item2.LastModified)
.GroupBy(e => e.Item1)

3
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -121,7 +121,8 @@ namespace Wabbajack.Lib.Downloaders
$"Authenticating for the Nexus failed. A nexus account is required to automatically download mods."));
return;
}
await Metrics.Send("nexus_login", _client.ApiKey!);
if (!await _client.IsPremium())
{

26
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -157,8 +157,11 @@ namespace Wabbajack.Lib.NexusApi
{
var url = "https://api.nexusmods.com/v1/users/validate.json";
using var response = await HttpClient.GetAsync(url);
return (int.Parse(response.Headers.GetValues("X-RL-Daily-Remaining").First()),
var result = (int.Parse(response.Headers.GetValues("X-RL-Daily-Remaining").First()),
int.Parse(response.Headers.GetValues("X-RL-Hourly-Remaining").First()));
_dailyRemaining = result.Item1;
_hourlyRemaining = result.Item2;
return result;
}
#endregion
@ -177,6 +180,13 @@ namespace Wabbajack.Lib.NexusApi
return _dailyRemaining;
}
}
protected set
{
lock (RemainingLock)
{
_dailyRemaining = value;
}
}
}
private int _hourlyRemaining;
@ -189,10 +199,18 @@ namespace Wabbajack.Lib.NexusApi
return _hourlyRemaining;
}
}
protected set
{
lock (RemainingLock)
{
_hourlyRemaining = value;
}
}
}
private void UpdateRemaining(HttpResponseMessage response)
protected virtual async Task UpdateRemaining(HttpResponseMessage response)
{
try
{
@ -221,7 +239,7 @@ namespace Wabbajack.Lib.NexusApi
#endregion
private NexusApiClient(string? apiKey = null)
protected NexusApiClient(string? apiKey = null)
{
ApiKey = apiKey;
@ -250,7 +268,7 @@ namespace Wabbajack.Lib.NexusApi
try
{
using var response = await HttpClient.GetAsync(url);
UpdateRemaining(response);
await UpdateRemaining(response);
if (!response.IsSuccessStatusCode)
{
Utils.Log($"Nexus call failed: {response.RequestMessage.RequestUri}");

12
Wabbajack.Server.Test/NexusCacheTests.cs
View File

@ -60,7 +60,7 @@ namespace Wabbajack.BuildServer.Test
}
[Fact]
public async Task CanQueryAndFindNexusModfilesSlow()
public async Task CanQueryAndFindNexusModfilesFast()
{
var startTime = DateTime.UtcNow;
var sql = Fixture.GetService<SqlService>();
@ -68,8 +68,7 @@ namespace Wabbajack.BuildServer.Test
await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow);
await sql.DeleteNexusModInfosUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow);
var result = await validator.SlowNexusModStats(new ValidationData(),
new NexusDownloader.State {Game = Game.SkyrimSpecialEdition, ModID = 1137, FileID = 121449});
var result = await validator.FastNexusModStats(new NexusDownloader.State {Game = Game.SkyrimSpecialEdition, ModID = 1137, FileID = 121449});
Assert.Equal(ArchiveStatus.Valid, result);
var gameId = Game.SkyrimSpecialEdition.MetaData().NexusGameId;
@ -119,6 +118,13 @@ namespace Wabbajack.BuildServer.Test
Assert.Equal(b, new Box{Value = b}.ToJson().FromJsonString<Box>().Value);
Assert.NotEqual(a.Hour, b.Hour);
Assert.Equal(b.Hour, new Box{Value = a}.ToJson().FromJsonString<Box>().Value.Hour);
var ts = (long)1589528640;
var ds = DateTime.Parse("2020-05-15 07:44:00.000");
Assert.Equal(ds, ts.AsUnixTime());
Assert.Equal(ts, (long)ds.AsUnixTime());
Assert.Equal(ts, (long)ts.AsUnixTime().AsUnixTime());
}
}

18
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -551,6 +551,24 @@ CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED
) ON [PRIMARY]
GO
/****** Object: Table [dbo].[NexusKeys] Script Date: 5/15/2020 5:20:02 PM ******/
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[NexusKeys](
[ApiKey] [nvarchar](162) NOT NULL,
[DailyRemain] [int] NOT NULL,
[HourlyRemain] [int] NOT NULL,
CONSTRAINT [PK_NexusKeys] PRIMARY KEY CLUSTERED
(
[ApiKey] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY]
GO
/****** Object: StoredProcedure [dbo].[MergeAllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/
SET ANSI_NULLS ON
GO

6
Wabbajack.Server/AppSettings.cs
View File

@ -25,9 +25,15 @@ namespace Wabbajack.BuildServer
public bool RunFrontEndJobs { get; set; }
public bool RunBackEndJobs { get; set; }
public bool RunNexusPolling { get; set; }
public bool RunDownloader { get; set; }
public string BunnyCDN_StorageZone { get; set; }
public string SqlConnection { get; set; }
public int MaxJobs { get; set; } = 2;
public string SpamWebHook { get; set; } = null;
public string HamWebHook { get; set; } = null;
}
}

17
Wabbajack.Server/Controllers/NexusCache.cs
View File

@ -52,7 +52,7 @@ namespace Wabbajack.BuildServer.Controllers
string method = "CACHED";
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var api = await GetClient();
result = await api.GetModInfo(game, ModId, false);
await _sql.AddNexusModInfo(game, ModId, result.updated_time, result);
@ -69,6 +69,21 @@ namespace Wabbajack.BuildServer.Controllers
return result;
}
private async Task<NexusApiClient> GetClient()
{
var key = Request.Headers["apikey"].FirstOrDefault();
if (key == null)
return await NexusApiClient.Get(null);
if (await _sql.HaveKey(key))
return await NexusApiClient.Get(key);
var client = await NexusApiClient.Get(key);
var (daily, hourly) = await client.GetRemainingApiCalls();
await _sql.SetNexusAPIKey(key, daily, hourly);
return client;
}
[HttpGet]
[Route("{GameName}/mods/{ModId}/files.json")]
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, long ModId)

103
Wabbajack.Server/DTOs/DiscordMessage.cs Normal file
View File

@ -0,0 +1,103 @@
using System;
using Newtonsoft.Json;
using Wabbajack.Common.Serialization.Json;
namespace Wabbajack.Server.DTOs
{
[JsonName("DiscordMessage")]
public class DiscordMessage
{
[JsonProperty("username")]
public string UserName { get; set; }
[JsonProperty("avatar_url")]
public Uri AvatarUrl { get; set; }
[JsonProperty("content")]
public string Content { get; set; }
[JsonProperty("embeds")]
public DiscordEmbed[] Embeds { get; set; }
}
[JsonName("DiscordEmbed")]
public class DiscordEmbed
{
[JsonProperty("color")]
public int Color { get; set; }
[JsonProperty("author")]
public DiscordAuthor Author { get; set; }
[JsonProperty("url")]
public Uri Url { get; set; }
[JsonProperty("description")]
public string Description { get; set; }
[JsonProperty("fields")]
public DiscordField Field { get; set; }
[JsonProperty("thumbnail")]
public DiscordNumbnail Thumbnail { get; set; }
[JsonProperty("image")]
public DiscordImage Image { get; set; }
[JsonProperty("footer")]
public DiscordFooter Footer { get; set; }
[JsonProperty("timestamp")]
public DateTime Timestamp { get; set; } = DateTime.UtcNow;
}
[JsonName("DiscordAuthor")]
public class DiscordAuthor
{
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("url")]
public Uri Url { get; set; }
[JsonProperty("icon_url")]
public Uri IconUrl { get; set; }
}
[JsonName("DiscordField")]
public class DiscordField
{
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("value")]
public string Value { get; set; }
[JsonProperty("inline")]
public bool Inline { get; set; }
}
[JsonName("DiscordThumbnail")]
public class DiscordNumbnail
{
[JsonProperty("Url")]
public Uri Url { get; set; }
}
[JsonName("DiscordImage")]
public class DiscordImage
{
[JsonProperty("Url")]
public Uri Url { get; set; }
}
[JsonName("DiscordFooter")]
public class DiscordFooter
{
[JsonProperty("text")]
public string Text { get; set; }
[JsonProperty("icon_url")]
public Uri icon_url { get; set; }
}
}

7
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -30,11 +30,6 @@ namespace Wabbajack.Server.DataLayer
@"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
deleted += await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModFilesSlow WHERE GameId = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
return deleted;
}
@ -95,7 +90,7 @@ namespace Wabbajack.Server.DataLayer
USING (SELECT @GameId GameId, @ModId ModId, @LastChecked LastChecked, @FileId FileId) AS Source
ON Target.GameId = Source.GameId AND Target.ModId = Source.ModId AND Target.FileId = Source.FileId
WHEN MATCHED THEN UPDATE SET Target.LastChecked = @LastChecked
WHEN NOT MATCHED THEN INSERT (GameId, ModId, LastChecked, FileId) VALUES (@GameId, @ModId, @LastChecked, FileId);",
WHEN NOT MATCHED THEN INSERT (GameId, ModId, LastChecked, FileId) VALUES (@GameId, @ModId, @LastChecked, @FileId);",
new
{
GameId = game.MetaData().NexusGameId,

51
Wabbajack.Server/DataLayer/NexusKeys.cs Normal file
View File

@ -0,0 +1,51 @@
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task SetNexusAPIKey(string key, long daily, long hourly)
{
await using var conn = await Open();
await using var trans = await conn.BeginTransactionAsync();
await conn.ExecuteAsync(@"DELETE FROM NexusKeys WHERE ApiKey = @ApiKey", new {ApiKey = key}, trans);
await conn.ExecuteAsync(@"INSERT INTO NexusKeys (ApiKey, DailyRemain, HourlyRemain) VALUES (@ApiKey, @DailyRemain, @HourlyRemain)",
new {ApiKey = key, DailyRemain = daily, HourlyRemain = hourly}, trans);
await trans.CommitAsync();
}
public async Task DeleteNexusAPIKey(string key)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"DELETE FROM NexusKeys WHERE ApiKey = @ApiKey", new {ApiKey = key});
}
public async Task<List<string>> GetNexusApiKeys(int threshold = 1500)
{
await using var conn = await Open();
return (await conn.QueryAsync<string>(@"SELECT ApiKey FROM NexusKeys WHERE DailyRemain >= @Threshold ORDER BY DailyRemain DESC",
new {Threshold = threshold})).ToList();
}
public async Task<List<(string Key, int Daily, int Hourly)>> GetNexusApiKeysWithCounts(int threshold = 1500)
{
await using var conn = await Open();
return (await conn.QueryAsync<(string, int, int)>(@"SELECT ApiKey, DailyRemain, HourlyRemain FROM NexusKeys WHERE DailyRemain >= @Threshold ORDER BY DailyRemain DESC",
new {Threshold = threshold})).ToList();
}
public async Task<bool> HaveKey(string key)
{
await using var conn = await Open();
return (await conn.QueryAsync<string>(@"SELECT ApiKey FROM NexusKeys WHERE ApiKey = @ApiKey",
new {ApiKey = key})).Any();
}
}
}

9
Wabbajack.Server/Services/ArchiveDownloader.cs
View File

@ -30,11 +30,11 @@ namespace Wabbajack.Server.Services
while (true)
{
//var (daily, hourly) = await _nexusClient.GetRemainingApiCalls();
//bool ignoreNexus = hourly < 25;
var ignoreNexus = true;
var (daily, hourly) = await _nexusClient.GetRemainingApiCalls();
bool ignoreNexus = (daily < 100 && hourly < 10);
//var ignoreNexus = true;
if (ignoreNexus)
_logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})");
_logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {daily}, Hourly:{hourly})");
else
_logger.LogInformation($"Looking for any download (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})");
@ -89,6 +89,7 @@ namespace Wabbajack.Server.Services
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, ex.ToString());
}

73
Wabbajack.Server/Services/DiscordWebHook.cs Normal file
View File

@ -0,0 +1,73 @@
using System;
using System.Net.Http;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public enum Channel
{
// High volume messaging, really only useful for internal devs
Spam,
// Low volume messages designed for admins
Ham
}
public class DiscordWebHook : AbstractService<DiscordWebHook, int>
{
private AppSettings _settings;
private ILogger<DiscordWebHook> _logger;
private Random _random = new Random();
public DiscordWebHook(ILogger<DiscordWebHook> logger, AppSettings settings) : base(logger, settings, TimeSpan.FromHours(1))
{
_settings = settings;
_logger = logger;
var message = new DiscordMessage
{
Content = $"\"{GetQuote()}\" - Sheogorath (as he brings the server online)",
};
var a = Send(Channel.Ham, message);
var b = Send(Channel.Spam, message);
}
public async Task Send(Channel channel, DiscordMessage message)
{
try
{
string url = channel switch
{
Channel.Spam => _settings.SpamWebHook,
Channel.Ham => _settings.HamWebHook,
_ => null
};
if (url == null) return;
var client = new Common.Http.Client();
await client.PostAsync(url, new StringContent(message.ToJson(true), Encoding.UTF8, "application/json"));
}
catch (Exception ex)
{
_logger.LogError(ex, ex.ToJson());
}
}
private string GetQuote()
{
var data = Assembly.GetExecutingAssembly()!.GetManifestResourceStream("Wabbajack.Server.sheo_quotes.txt");
var lines = Encoding.UTF8.GetString(data.ReadAll()).Split('\n');
return lines[_random.Next(lines.Length)].Trim();
}
public override async Task<int> Execute()
{
return 0;
}
}
}

161
Wabbajack.Server/Services/ListValidator.cs
View File

@ -22,22 +22,25 @@ namespace Wabbajack.Server.Services
public class ListValidator : AbstractService<ListValidator, int>
{
private SqlService _sql;
private NexusApiClient _nexusClient;
private DiscordWebHook _discord;
private NexusKeyMaintainance _nexus;
public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries { get; private set; } =
new (ModListSummary Summary, DetailedStatus Detailed)[0];
public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql)
public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus)
: base(logger, settings, TimeSpan.FromMinutes(10))
{
_sql = sql;
_discord = discord;
_nexus = nexus;
}
public override async Task<int> Execute()
{
var data = await _sql.GetValidationData();
using var queue = new WorkQueue();
var results = await data.ModLists.PMap(queue, async list =>
@ -94,7 +97,7 @@ namespace Wabbajack.Server.Services
nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
return (archive, ArchiveStatus.Valid);
case NexusDownloader.State ns:
return (archive, await SlowNexusModStats(data, ns));
return (archive, await FastNexusModStats(ns));
case ManualDownloader.State _:
return (archive, ArchiveStatus.Valid);
default:
@ -109,112 +112,88 @@ namespace Wabbajack.Server.Services
}
}
}
private readonly AsyncLock _slowQueryLock = new AsyncLock();
public async Task<ArchiveStatus> SlowNexusModStats(ValidationData data, NexusDownloader.State ns)
private AsyncLock _lock = new AsyncLock();
public async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
{
var gameId = ns.Game.MetaData().NexusGameId;
using var _ = await _slowQueryLock.WaitAsync();
_logger.Log(LogLevel.Warning, $"Slow querying for {ns.Game} {ns.ModID} {ns.FileID}");
if (data.NexusFiles.Contains((gameId, ns.ModID, ns.FileID)))
return ArchiveStatus.Valid;
if (data.SlowQueriedFor.Contains((ns.Game, ns.ModID)))
return ArchiveStatus.InValid;
var queryTime = DateTime.UtcNow;
var regex = new Regex("(?<=[?;&]file_id\\=)\\d+");
var client = new Common.Http.Client();
var result =
await client.GetHtmlAsync(
$"https://www.nexusmods.com/{ns.Game.MetaData().NexusName}/mods/{ns.ModID}/?tab=files");
var fileIds = result.DocumentNode.Descendants()
.Select(f => f.GetAttributeValue("href", ""))
.Select(f =>
{
var match = regex.Match(f);
return !match.Success ? null : match.Value;
})
.Where(m => m != null)
.Select(m => long.Parse(m))
.Distinct()
.ToList();
_logger.Log(LogLevel.Warning, $"Slow queried {fileIds.Count} files");
foreach (var id in fileIds)
{
await _sql.AddNexusModFileSlow(ns.Game, ns.ModID, id, queryTime);
data.NexusFiles.Add((gameId, ns.ModID, id));
}
return fileIds.Contains(ns.FileID) ? ArchiveStatus.Valid : ArchiveStatus.InValid;
}
private async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
{
// Check if some other thread has added them
var mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID);
var files = await _sql.GetModFiles(ns.Game, ns.ModID);
try
if (mod == null || files == null)
{
if (mod == null)
// Aquire the lock
using var lck = await _lock.WaitAsync();
// Check again
mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID);
files = await _sql.GetModFiles(ns.Game, ns.ModID);
if (mod == null || files == null)
{
_nexusClient ??= await NexusApiClient.Get();
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
NexusApiClient nexusClient = await _nexus.GetClient();
var queryTime = DateTime.UtcNow;
try
{
mod = await _nexusClient.GetModInfo(ns.Game, ns.ModID, false);
}
catch
{
mod = new ModInfo
if (mod == null)
{
mod_id = ns.ModID.ToString(), game_id = ns.Game.MetaData().NexusGameId, available = false
};
}
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
try
{
mod = await nexusClient.GetModInfo(ns.Game, ns.ModID, false);
}
catch
{
mod = new ModInfo
{
mod_id = ns.ModID.ToString(),
game_id = ns.Game.MetaData().NexusGameId,
available = false
};
}
try
{
await _sql.AddNexusModInfo(ns.Game, ns.ModID, mod.updated_time, mod);
}
catch (Exception _)
{
// Could be a PK constraint failure
}
try
{
await _sql.AddNexusModInfo(ns.Game, ns.ModID, queryTime, mod);
}
catch (Exception _)
{
// Could be a PK constraint failure
}
}
}
if (files == null)
{
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
try
{
files = await _nexusClient.GetModFiles(ns.Game, ns.ModID, false);
}
catch
{
files = new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>()};
}
if (files == null)
{
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
try
{
files = await nexusClient.GetModFiles(ns.Game, ns.ModID, false);
}
catch
{
files = new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>()};
}
try
{
await _sql.AddNexusModFiles(ns.Game, ns.ModID, mod.updated_time, files);
try
{
await _sql.AddNexusModFiles(ns.Game, ns.ModID, queryTime, files);
}
catch (Exception _)
{
// Could be a PK constraint failure
}
}
}
catch (Exception _)
catch (Exception ex)
{
// Could be a PK constraint failure
return ArchiveStatus.InValid;
}
}
}
catch (Exception ex)
{
return ArchiveStatus.InValid;
}
if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID))
return ArchiveStatus.Valid;

23
Wabbajack.Server/Services/ModListDownloader.cs
View File

@ -11,6 +11,7 @@ using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
@ -20,13 +21,15 @@ namespace Wabbajack.Server.Services
private AppSettings _settings;
private ArchiveMaintainer _maintainer;
private SqlService _sql;
private DiscordWebHook _discord;
public ModListDownloader(ILogger<ModListDownloader> logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql)
public ModListDownloader(ILogger<ModListDownloader> logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql, DiscordWebHook discord)
{
_logger = logger;
_settings = settings;
_maintainer = maintainer;
_sql = sql;
_discord = discord;
}
public void Start()
@ -69,6 +72,8 @@ namespace Wabbajack.Server.Services
if (!_maintainer.HaveArchive(list.DownloadMetadata!.Hash))
{
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"});
var tf = new TempFile();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
if (state == null)
@ -100,7 +105,9 @@ namespace Wabbajack.Server.Services
{
if (entry == null)
{
Utils.Log($"Bad Modlist {list.Links.MachineURL}");
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"});
continue;
}
@ -110,7 +117,9 @@ namespace Wabbajack.Server.Services
}
catch (JsonReaderException ex)
{
Utils.Log($"Bad JSON format for {list.Links.MachineURL}");
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"});
continue;
}
}
@ -120,12 +129,20 @@ namespace Wabbajack.Server.Services
catch (Exception ex)
{
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"});
}
}
_logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
if (downloaded > 0)
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Downloaded {downloaded} new lists"});
var fc = await _sql.EnqueueModListFilesForIndexing();
_logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading");
if (fc > 0)
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Enqueing {fc} files for downloading"});
return downloaded;
}

87
Wabbajack.Server/Services/NexusKeyMaintainance.cs Normal file
View File

@ -0,0 +1,87 @@
using System;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.Services
{
public class NexusKeyMaintainance : AbstractService<NexusKeyMaintainance, int>
{
private SqlService _sql;
public NexusKeyMaintainance(ILogger<NexusKeyMaintainance> logger, AppSettings settings, SqlService sql) : base(logger, settings, TimeSpan.FromHours(1))
{
_sql = sql;
}
public async Task<NexusApiClient> GetClient()
{
var keys = await _sql.GetNexusApiKeysWithCounts(1500);
foreach (var key in keys)
{
return new TrackingClient(_sql, key);
}
return await NexusApiClient.Get();
}
public override async Task<int> Execute()
{
var keys = await _sql.GetNexusApiKeysWithCounts(0);
_logger.Log(LogLevel.Information, $"Verifying {keys.Count} API Keys");
foreach (var key in keys)
{
try
{
var client = new TrackingClient(_sql, key);
var status = await client.GetUserStatus();
if (!status.is_premium)
{
await _sql.DeleteNexusAPIKey(key.Key);
continue;
}
var (daily, hourly) = await client.GetRemainingApiCalls();
await _sql.SetNexusAPIKey(key.Key, daily, hourly);
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, "Update error, purging API key");
await _sql.DeleteNexusAPIKey(key.Key);
}
}
return keys.Count;
}
}
public class TrackingClient : NexusApiClient
{
private SqlService _sql;
public TrackingClient(SqlService sql, (string Key, int Daily, int Hourly) key) : base(key.Key)
{
_sql = sql;
DailyRemaining = key.Daily;
HourlyRemaining = key.Hourly;
}
protected virtual async Task UpdateRemaining(HttpResponseMessage response)
{
await base.UpdateRemaining(response);
try
{
var dailyRemaining = int.Parse(response.Headers.GetValues("x-rl-daily-remaining").First());
var hourlyRemaining = int.Parse(response.Headers.GetValues("x-rl-hourly-remaining").First());
await _sql.SetNexusAPIKey(ApiKey, dailyRemaining, hourlyRemaining);
}
catch (Exception ex)
{
}
}
}
}

13
Wabbajack.Server/Services/NexusPoll.cs
View File

@ -48,7 +48,7 @@ namespace Wabbajack.Server.Services
if (totalPurged > 0)
_logger.Log(LogLevel.Information, $"Purged {totalPurged} cache items {result.Game} {result.ModId} {result.TimeStamp}");
updated++;
updated += totalPurged;
}
catch (Exception ex)
{
@ -58,7 +58,7 @@ namespace Wabbajack.Server.Services
}
if (updated > 0)
_logger.Log(LogLevel.Information, $"Primed {updated} nexus cache entries");
_logger.Log(LogLevel.Information, $"RSS Purged {updated} nexus cache entries");
_globalInformation.LastNexusSyncUTC = DateTime.UtcNow;
}
@ -98,14 +98,15 @@ namespace Wabbajack.Server.Services
// Mod activity could hide files
var b = d.mod.LastestModActivity.AsUnixTime();
return new {Game = d.game.Game, Date = (a > b ? a : b), ModId = d.mod.ModId};
return new {Game = d.game.Game, Date = (a > b) ? a : b, ModId = d.mod.ModId};
});
var purged = await collected.PMap(queue, async t =>
{
var resultA = await _sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date);
var resultB = await _sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date);
return resultA + resultB;
long purgeCount = 0;
purgeCount += await _sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date);
purgeCount += await _sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date);
return purgeCount;
});
_logger.Log(LogLevel.Information, $"Purged {purged.Sum()} cache entries");

4
Wabbajack.Server/Startup.cs
View File

@ -63,6 +63,8 @@ namespace Wabbajack.Server
services.AddSingleton<NonNexusDownloadValidator>();
services.AddSingleton<ListValidator>();
services.AddSingleton<ArchiveDownloader>();
services.AddSingleton<DiscordWebHook>();
services.AddSingleton<NexusKeyMaintainance>();
services.AddMvc();
services.AddControllers()
@ -112,6 +114,8 @@ namespace Wabbajack.Server
app.UseService<NonNexusDownloadValidator>();
app.UseService<ListValidator>();
app.UseService<ArchiveDownloader>();
app.UseService<DiscordWebHook>();
app.UseService<NexusKeyMaintainance>();
app.Use(next =>
{

5
Wabbajack.Server/Wabbajack.Server.csproj
View File

@ -42,5 +42,10 @@
</None>
</ItemGroup>
<ItemGroup>
<None Remove="sheo_quotes.txt" />
<EmbeddedResource Include="sheo_quotes.txt" />
</ItemGroup>
</Project>

55
Wabbajack.Server/sheo_quotes.txt Normal file
View File

@ -0,0 +1,55 @@
I see you have completed my little errand. Well done. Perhaps you’ve gotten a taste of madness aswell? Do not believe madness to be a curse, mortal. For some it is the greatest of blessings. A bitter mercy perhaps, but mercy non the less. Give me the Fork of Horripilation, I believe I have something more suitable for your needs. Go now. Remember what you have seen.
Use the fork wisely, mortal. Few have wield to have not come away changed. Use the fork to strike a deathblow to the giant Bullnetch that resides near the hermit. Do this, return the Fork of Horripilation to me, and Sheogorath will reward you well.
What is it, mortal? Have you come to be of the service to Sheogorath? That in and of itself speaks toward your madness. This pleases me. Fetch the Fork of Horripliation from the mad hermit near Ald Redaynia. Take care with him. He’s not the most... stable man.
Unworthy, unworthy, unworthy! Useless mortal meat. Walking bag of dung!
Bring me a champion! Rend the flesh of my foes! A mortal champion to wade through the entrails of my enemies!
Really, do come in. It’s lovely in the Isles right now. Perfect time for a visit.
Greetings! Salutations! Welcome! Now go away. Leave. Run. Or die.
Isn't that a hoot? I love it, myself. Best part of being a Daedric Prince, really. Go ahead, try it again. He loves it!
Marvellous, marvellous! Self-immolation is a wonderful thing, isn't it? But now that we've had our fun, off to the Sacellum with you.
I suppose an introduction is in order. I'm Sheogorath, Prince of Madness. And other things. I'm not talking about them.
You should be off like the wind, solving problems and doing good deeds!
Time. Time is an artificial construct. An arbitrary system based on the idea that events occur in a linear direction at all times.
Always forward, never back. Is the concept of time correct? Is time relevant? It matters not. One way or another, I fear that our time has run out.
A new Gatekeeper! Excellent. We might be onto something with you, after all. That should keep out the stragglers.
A little busy here! I'm trying to decide what to have for dinner. Oh, how I love eating. One of my favorite things to do.
It's Jyggalag's time, and not a good time at all. You're going to help me stop it. First, though, you need to get your feet wet.
Another Daedric Prince. Not a nice one. I don't think ANY of the other Princes like him, actually. I mean, Malacath is more popular at parties.
The Daedric Prince of Order. Or biscuits... No. Order. And not in a good way. Bleak. Colorless. Dead. Boring, boring, boring.
The Greymarch comes, and Jyggalag walks. Or runs. Never skips, sidles, or struts. Mostly, he just destroys everything around him.
Once you understand what My Realm is, you might understand why it's important to keep it intact.
Two halves, two rulers, two places. Meet and greet. Do what they will, so you know what they're about.
Ask? ASK? I don't ask. I tell. This is My Realm, remember? My creation, My place, My rules.
Wonderful! Time for a celebration... Cheese for everyone!
Makes all of my subjects uneasy. Tense. Homicidal. Some of them, at least. We need to get that Torch relit, before the place falls apart.
You're going to stop the Greymarch by becoming Me. Or a version of Me. You'll be powerful. Powerful enough to stop Jyggalag.
You know what would be a good sign? "Free Sweetrolls!" Who wouldn't like that?
You'll be my champion. You'll grow powerful. You'll grow to be me. Prince of Madness, a new Sheogorath. Or you'll die trying. I love that about you.
Oh, don't forget to make use of dear Haskill. Between you and me, if he's not summoned three or four times a day, I don't think he feels appreciated.
I hate indecision! Or maybe I don't. Make up your mind, or I'll have your skin made into a hat -- one of those arrowcatchers. I love those hats!
So, which is it? What will it be? Mania? Dementia? The suspense is killing me. Or you, if I have to keep waiting.
Except where the backbone is an actual backbone. Ever been to Malacath's realm...? Nasty stuff. But, back to the business at hand.
Happens every time. The Greymarch starts, Order appears, and I become Jyggalag and wipe out My whole Realm.
Flee while you can, mortal. When we next meet I will not know you, and I will slay you like the others.
Ah... New Sheoth. My home away from places that aren't my home. The current location is much better than some of the prior ones. Don't you think?
The Isles, the Isles. A wonderful place! Except when it's horrible. Then it's horribly wonderful. Good for a visit. Or for an eternity.
Time to save the Realm! Rescue the damsel! Slay the beast! Or die trying. Your help is required.
Daedra are the embodiment of change. Change and permanency. I'm no different, except in the ways that I am.
Was it Molag? No, no... Little Tim, the toymaker's son? The ghost of King Lysandus? Or was it... Yes! Stanley, that talking grapefruit from Passwall.
Reeaaaallllyyyy?
Well? Spit it out, mortal. I haven't got an eternity! Oh, wait! I do.
I am a part of you, little mortal. I am a shadow in your subconscious, a blemish on your fragile little psyche. You know me. You just don't know it.
Sheogorath, Daedric Prince of Madness. At your service.
Yaaawwwwnn....
Oh, pardon me. Were you saying something? I do apologize, it's just that I find myself suddenly and irrevocably...
Bored!
I mean, really. Here you stand, before Sheogorath himself, Daedric Prince of Madness, and all you deem fit to do is... deliver a message? How sad.
Now you. You can call me Ann Marie.
Oh... lovely. Now all my dear Pelagius has to worry about are the several hundred legitimate threats...
Ah, wonderful, wonderful! Why waste all that hatred on yourself when it can so easily be directed at others!
Mortal? Insufferable.
Yes, yes, you're entirely brilliant. Conquering madness and all that. Blah blah blah.
Ah, so now my dear Pelagius can hate himself for being legitimately afraid of things that actually threaten his existence...
Conquering paranoia should be a snap after that ordeal, hmm?
Welcome to the deceptively verdant mind of the Emperor Pelagius III. That's right! You're in the head of a dead, homicidally insane monarch.
The Wabbajack! Huh? Huh? Didn't see that coming, did you?

1
Wabbajack/View Models/MainWindowVM.cs
View File

@ -155,6 +155,7 @@ namespace Wabbajack
WorkingDirectory = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location)
};
Process.Start(process);
ShutdownApplication();
}
private static bool IsStartingFromModlist(out AbsolutePath modlistPath)