Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1031 from wabbajack-tools/ll-updates

Browse Source 
Ll updates
This commit is contained in:
Timothy Baldridge 2020-08-13 05:46:54 -06:00 committed by GitHub
commit f0cfbc6b0c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 405 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Wabbajack.CLI/OptionsDefinition.cs
View File

@ -29,7 +29,8 @@ namespace Wabbajack.CLI
typeof(ForceHealing),
typeof(HashVariants),
typeof(ParseMeta),
typeof(NoPatch)
typeof(NoPatch),
typeof(NexusPermissions)
};
}
}

27
Wabbajack.CLI/Verbs/NexusPermissions.cs Normal file
View File

@ -0,0 +1,27 @@
using System;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.CLI.Verbs
{
[Verb("nexus-permissions", HelpText = "Get the nexus permissions for a mod")]
public class NexusPermissions : AVerb
{
[Option('m', "mod-id", Required = true, HelpText = "Mod Id")]
public long ModId { get; set; } = 0;
[Option('g', "game", Required = true, HelpText = "Game Name")]
public string GameName { get; set; } = "";
protected override async Task<ExitCode> Run()
{
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var p = await HTMLInterface.GetUploadPermissions(game, ModId);
Console.WriteLine($"Game: {game}");
Console.WriteLine($"ModId: {ModId}");
Console.WriteLine($"Permissions: {p}");
return ExitCode.Ok;
}
}
}

36
Wabbajack.Common/OctoDiff.cs
View File

@ -7,14 +7,13 @@ namespace Wabbajack.Common
{
public class OctoDiff
{
private static ProgressReporter reporter = new ProgressReporter();
public static void Create(byte[] oldData, byte[] newData, Stream output)
{
using var signature = CreateSignature(oldData);
using var oldStream = new MemoryStream(oldData);
using var newStream = new MemoryStream(newData);
var db = new DeltaBuilder {ProgressReporter = reporter};
db.BuildDelta(newStream, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
var db = new DeltaBuilder {ProgressReporter = new ProgressReporter()};
db.BuildDelta(newStream, new SignatureReader(signature, new ProgressReporter()), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
}
private static Stream CreateSignature(byte[] oldData)
@ -36,24 +35,39 @@ namespace Wabbajack.Common
sigStream.Position = 0;
}
public static void Create(Stream oldData, Stream newData, Stream signature, Stream output)
public static void Create(Stream oldData, Stream newData, Stream signature, Stream output, ProgressReporter? reporter = null)
{
CreateSignature(oldData, signature);
var db = new DeltaBuilder {ProgressReporter = reporter};
db.BuildDelta(newData, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
var db = new DeltaBuilder {ProgressReporter = reporter ?? new ProgressReporter()};
db.BuildDelta(newData, new SignatureReader(signature, reporter ?? new ProgressReporter()), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
}
private class ProgressReporter : IProgressReporter
public class ProgressReporter : IProgressReporter
{
private DateTime _lastUpdate = DateTime.UnixEpoch;
private readonly TimeSpan _updateInterval = TimeSpan.FromMilliseconds(100);
private TimeSpan _updateInterval;
private Action<string, Percent> _report;
public ProgressReporter()
{
_updateInterval = TimeSpan.FromMilliseconds(100);
_report = (s, percent) => Utils.Status(s, percent);
}
public ProgressReporter(TimeSpan updateInterval, Action<string, Percent> report)
{
_updateInterval = updateInterval;
_report = report;
}
public void ReportProgress(string operation, long currentPosition, long total)
{
if (DateTime.Now - _lastUpdate < _updateInterval) return;
_lastUpdate = DateTime.Now;
if (currentPosition >= total || total < 1 || currentPosition < 0)
return;
Utils.Status(operation, new Percent(total, currentPosition));
_report(operation, new Percent(total, currentPosition));
}
}
@ -61,13 +75,13 @@ namespace Wabbajack.Common
{
using var deltaStream = openPatchStream();
var deltaApplier = new DeltaApplier();
deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, reporter), output);
deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, new ProgressReporter()), output);
}
public static void Apply(FileStream input, FileStream patchStream, FileStream output)
{
var deltaApplier = new DeltaApplier();
deltaApplier.Apply(input, new BinaryDeltaReader(patchStream, reporter), output);
deltaApplier.Apply(input, new BinaryDeltaReader(patchStream, new ProgressReporter()), output);
}
}
}

1
Wabbajack.Common/Util/AsyncLock.cs
View File

@ -5,7 +5,6 @@ using System.Reactive.Disposables;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Wabbajack.Common
{
public class AsyncLock

2
Wabbajack.Common/Util/TempFile.cs
View File

@ -37,7 +37,7 @@ namespace Wabbajack.Common
}
public async ValueTask DisposeAsync()
{
if (DeleteAfter)
if (DeleteAfter && Path.Exists)
{
await Path.DeleteAsync();
}

2
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -141,7 +141,7 @@ namespace Wabbajack.Lib.Downloaders
}
public virtual async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
public virtual async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
{
return await ServerFindUpgrade(a);
}

60
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
@ -6,6 +7,8 @@ using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web;
using F23.StringSimilarity;
using HtmlAgilityPack;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.Validation;
@ -25,9 +28,13 @@ namespace Wabbajack.Lib.Downloaders
}
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
{
Uri url = DownloaderUtils.GetDirectURL(archiveINI);
return await GetDownloaderStateFromUrl(url, quickMode);
}
public async Task<AbstractDownloadState?> GetDownloaderStateFromUrl(Uri url, bool quickMode)
{
var absolute = true;
if (url == null || url.Host != SiteURL.Host) return null;
@ -81,7 +88,7 @@ namespace Wabbajack.Lib.Downloaders
FileName = file
};
}
public class State<TStateDownloader> : AbstractDownloadState, IMetaState
where TStateDownloader : IDownloader
{
@ -223,12 +230,59 @@ namespace Wabbajack.Lib.Downloaders
stream.Dispose();
return true;
}
public override IDownloader GetDownloader()
{
return DownloadDispatcher.GetInstance<TDownloader>();
}
public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
{
var files = await GetFilesInGroup();
var nl = new Levenshtein();
var newFile = files.OrderBy(f => nl.Distance(a.Name.ToLowerInvariant(), f.Name.ToLowerInvariant())).FirstOrDefault();
if (newFile == null) return default;
var existing = await downloadResolver(newFile);
if (existing != default) return (newFile, new TempFile());
var tmp = new TempFile();
await DownloadDispatcher.PrepareAll(new []{newFile.State});
if (await newFile.State.Download(newFile, tmp.Path))
{
newFile.Size = tmp.Path.Size;
newFile.Hash = await tmp.Path.FileHashAsync();
return (newFile, tmp);
}
await tmp.DisposeAsync();
return default;
}
public async Task<List<Archive>> GetFilesInGroup()
{
var others = await Downloader.AuthedClient.GetHtmlAsync($"{Site}/files/file/{FileName}?do=download");
var pairs = others.DocumentNode.SelectNodes("//a[@data-action='download']")
.Select(item => (item.GetAttributeValue("href", ""),
item.ParentNode.ParentNode.SelectNodes("//div//h4//span").First().InnerText));
List<Archive> archives = new List<Archive>();
foreach (var (url, name) in pairs)
{
var ini = new[] {"[General]", $"directURL={url}"};
var state = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(
string.Join("\n", ini).LoadIniString(), false));
if (state == null) continue;
archives.Add(new Archive(state) {Name = name});
}
return archives;
}
public override string GetManifestURL(Archive a)
{
return IsAttachment ? FullURL : $"{Site}/files/file/{FileName}/?do=download&r={FileID}";

11
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -112,11 +112,9 @@ namespace Wabbajack.Lib.Downloaders
return false;
}
var upgrade = (IUpgradingState)archive.State;
Utils.Log($"Trying to find solution to broken download for {archive.Name}");
var result = await upgrade.FindUpgrade(archive);
var result = await FindUpgrade(archive);
if (result == default)
{
Utils.Log(
@ -153,7 +151,14 @@ namespace Wabbajack.Lib.Downloaders
return true;
}
public static async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>>? downloadResolver = null)
{
downloadResolver ??= async a => default;
return await a.State.FindUpgrade(a, downloadResolver);
}
private static async Task<bool> DownloadFromMirror(Archive archive, AbsolutePath destination)
{
try

13
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -98,11 +98,13 @@ namespace Wabbajack.Lib.Downloaders
var bufferSize = 1024 * 32;
Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero);
var response = await client.GetAsync(Url);
var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false);
TOP:
if (!response.IsSuccessStatusCode)
if (!response.IsSuccessStatusCode)
{
return false;
}
Stream stream;
try
@ -212,19 +214,22 @@ TOP:
}
public virtual async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
{
var tmpFile = new TempFile();
var newArchive = new Archive(this) {Name = a.Name};
Utils.Log($"Downloading via HTTP to find Upgrade for {Url}");
try
{
if (!await Download(newArchive, tmpFile.Path))
return default;
}
catch (HttpException)
catch (HttpException ex)
{
Utils.Log($"Error finding upgrade via HTTP to find Upgrade for {Url} {ex}");
return default;
}

5
Wabbajack.Lib/Downloaders/IUpgradingState.cs
View File

@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.Lib.Downloaders
@ -11,7 +12,7 @@ namespace Wabbajack.Lib.Downloaders
/// </summary>
/// <param name="a"></param>
/// <returns></returns>
public Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a);
public Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver);
Task<bool> ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState);
}

2
Wabbajack.Lib/Downloaders/MEGADownloader.cs
View File

@ -194,7 +194,7 @@ namespace Wabbajack.Lib.Downloaders
}
}
public override Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
public override Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
{
return ServerFindUpgrade(a);
}

36
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders
private bool _prepared;
private AsyncLock _lock = new AsyncLock();
private UserStatus? _status;
private NexusApiClient? _client;
public INexusApi? Client;
public IObservable<bool> IsLoggedIn => Utils.HaveEncryptedJsonObservable("nexusapikey");
@ -115,9 +115,9 @@ namespace Wabbajack.Lib.Downloaders
await CLIArguments.ApiKey.ToEcryptedJson("nexusapikey");
}
_client = await NexusApiClient.Get();
_status = await _client.GetUserStatus();
if (!_client.IsAuthenticated)
Client = await NexusApiClient.Get();
_status = await Client.GetUserStatus();
if (!Client.IsAuthenticated)
{
Utils.ErrorThrow(new UnconvertedError(
$"Authenticating for the Nexus failed. A nexus account is required to automatically download mods."));
@ -205,20 +205,13 @@ namespace Wabbajack.Lib.Downloaders
try
{
var client = await NexusApiClient.Get();
var modInfo = await client.GetModInfo(Game, ModID);
if (!modInfo.available) return false;
var modFiles = await client.GetModFiles(Game, ModID);
var found = modFiles.files
.FirstOrDefault(file => file.file_id == FileID && file.category_name != null);
if (found != null)
return true;
Utils.Log($"Could not validate {URL} with cache, validating manually");
modFiles = await client.GetModFiles(Game, ModID, false);
found = modFiles.files
.FirstOrDefault(file => file.file_id == FileID && file.category_name != null);
return found != null;
}
catch (Exception ex)
@ -244,11 +237,14 @@ namespace Wabbajack.Lib.Downloaders
return new[] {"[General]", $"gameName={Game.MetaData().MO2ArchiveName}", $"modID={ModID}", $"fileID={FileID}"};
}
public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
{
var client = await NexusApiClient.Get();
var mod = await client.GetModInfo(Game, ModID);
if (!mod.available)
return default;
var files = await client.GetModFiles(Game, ModID);
var oldFile = files.files.FirstOrDefault(f => f.file_id == FileID);
var nl = new Levenshtein();
@ -265,13 +261,21 @@ namespace Wabbajack.Lib.Downloaders
return default;
}
var tempFile = new TempFile();
var newArchive = new Archive(new State {Game = Game, ModID = ModID, FileID = newFile.file_id})
{
Name = newFile.file_name,
};
var fastPath = await downloadResolver(newArchive);
if (fastPath != default)
{
newArchive.Size = fastPath.Size;
newArchive.Hash = await fastPath.FileHashAsync();
return (newArchive, new TempFile());
}
var tempFile = new TempFile();
await newArchive.State.Download(newArchive, tempFile.Path);
newArchive.Size = tempFile.Path.Size;

2
Wabbajack.Lib/ModListRegistry/ModListMetadata.cs
View File

@ -89,7 +89,7 @@ namespace Wabbajack.Lib.ModListRegistry
try
{
var client = new Http.Client();
return (await client.GetStringAsync(Consts.ModlistMetadataURL)).FromJsonString<List<ModlistMetadata>>();
return (await client.GetStringAsync(Consts.UnlistedModlistMetadataURL)).FromJsonString<List<ModlistMetadata>>();
}
catch (Exception ex)
{

17
Wabbajack.Lib/NexusApi/INexusApi.cs Normal file
View File

@ -0,0 +1,17 @@
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.Lib.NexusApi
{
public interface INexusApi
{
public Task<string> GetNexusDownloadLink(NexusDownloader.State archive);
public Task<NexusApiClient.GetModFilesResponse> GetModFiles(Game game, long modid, bool useCache = true);
public Task<ModInfo> GetModInfo(Game game, long modId, bool useCache = true);
public Task<UserStatus> GetUserStatus();
public Task<bool> IsPremium();
public bool IsAuthenticated { get; }
}
}

26
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -16,7 +16,7 @@ using Wabbajack.Lib.WebAutomation;
namespace Wabbajack.Lib.NexusApi
{
public class NexusApiClient : ViewModel
public class NexusApiClient : ViewModel, INexusApi
{
private static readonly string API_KEY_CACHE_FILE = "nexus.key_cache";
@ -24,7 +24,7 @@ namespace Wabbajack.Lib.NexusApi
#region Authentication
public string? ApiKey { get; }
public static string? ApiKey { get; set; }
public bool IsAuthenticated => ApiKey != null;
@ -313,7 +313,11 @@ namespace Wabbajack.Lib.NexusApi
public async Task<string> GetNexusDownloadLink(NexusDownloader.State archive)
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
var info = await GetModInfo(archive.Game, archive.ModID);
if (!info.available)
throw new Exception("Mod unavailable");
var url = $"https://api.nexusmods.com/v1/games/{archive.Game.MetaData().NexusName}/mods/{archive.ModID}/files/{archive.FileID}/download_link.json";
try
{
@ -321,6 +325,8 @@ namespace Wabbajack.Lib.NexusApi
}
catch (HttpException ex)
{
if (ex.Code != 403 || await IsPremium())
{
throw;
@ -383,20 +389,6 @@ namespace Wabbajack.Lib.NexusApi
public string URI { get; set; } = string.Empty;
}
private static string? _localCacheDir;
public static string LocalCacheDir
{
get
{
if (_localCacheDir == null)
_localCacheDir = Environment.GetEnvironmentVariable("NEXUSCACHEDIR");
if (_localCacheDir == null)
throw new ArgumentNullException($"Enviornment variable could not be located: NEXUSCACHEDIR");
return _localCacheDir;
}
set => _localCacheDir = value;
}
public static Uri ManualDownloadUrl(NexusDownloader.State state)
{
return new Uri($"https://www.nexusmods.com/{state.Game.MetaData().NexusName}/mods/{state.ModID}?tab=files");

3
Wabbajack.Server.Test/ABuildServerSystemTest.cs
View File

@ -170,7 +170,8 @@ namespace Wabbajack.BuildServer.Test
Consts.ModlistSummaryURL = MakeURL("lists/status.json");
Consts.ServerWhitelistURL = MakeURL("ServerWhitelist.yaml");
Consts.UnlistedModlistMetadataURL = MakeURL("lists/none.json");
}
public WorkQueue Queue { get; set; }

5
Wabbajack.Server.Test/ModListValidationTests.cs
View File

@ -4,6 +4,7 @@ using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
@ -54,6 +55,10 @@ namespace Wabbajack.BuildServer.Test
[Fact]
public async Task CanValidateModLists()
{
var sql = Fixture.GetService<SqlService>();
await using var conn = await sql.Open();
await conn.ExecuteAsync("DELETE from Patches");
var modlists = await MakeModList("CanValidateModlistsFile.txt");
Consts.ModlistMetadataURL = modlists.ToString();
Utils.Log("Updating modlists");

14
Wabbajack.Server.Test/NexusCacheTests.cs
View File

@ -152,5 +152,19 @@ namespace Wabbajack.BuildServer.Test
Assert.Equal(ts, (long)ts.AsUnixTime().AsUnixTime());
}
[Fact]
public async Task CanGetAndSetPermissions()
{
var game = Game.Oblivion;
var modId = 4424;
var sql = Fixture.GetService<SqlService>();
foreach (HTMLInterface.PermissionValue result in Enum.GetValues(typeof(HTMLInterface.PermissionValue)))
{
await sql.SetNexusPermission(game, modId, result);
Assert.Equal(result, (await sql.GetNexusPermissions())[(game, modId)]);
}
}
}
}

1
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -637,6 +637,7 @@ CREATE TABLE [dbo].[NexusKeys](
[ApiKey] [nvarchar](162) NOT NULL,
[DailyRemain] [int] NOT NULL,
[HourlyRemain] [int] NOT NULL,
[IsPremium] [tinyint] NOT NULL,
CONSTRAINT [PK_NexusKeys] PRIMARY KEY CLUSTERED
(
[ApiKey] ASC

2
Wabbajack.Server/Controllers/Metrics.cs
View File

@ -127,7 +127,7 @@ namespace Wabbajack.BuildServer.Controllers
private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
{
_logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}");
//_logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}");
await _sql.IngestMetric(new Metric
{
Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey

14
Wabbajack.Server/Controllers/ModUpgrade.cs
View File

@ -73,15 +73,15 @@ namespace Wabbajack.BuildServer.Controllers
{
if (await request.OldArchive.State.Verify(request.OldArchive))
{
_logger.LogInformation(
$"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), old archive is valid");
//_logger.LogInformation(
// $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), old archive is valid");
return NotFound("File is Valid");
}
}
catch (Exception ex)
{
_logger.LogInformation(
$"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), due to upgrade failure");
//_logger.LogInformation(
// $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), due to upgrade failure");
return NotFound("File is Valid");
}
@ -99,13 +99,13 @@ namespace Wabbajack.BuildServer.Controllers
{
if (patch.PatchSize != 0)
{
_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found");
//_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found");
await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id);
return
Ok(
$"https://{(await _creds).Username}.b-cdn.net/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
}
_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed");
//_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed");
return NotFound("Patch creation failed");
}
@ -119,7 +119,7 @@ namespace Wabbajack.BuildServer.Controllers
await _quickSync.Notify<PatchBuilder>();
}
_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found is processing");
//_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found is processing");
// Still processing
return Accepted();
}

27
Wabbajack.Server/Controllers/NexusCache.cs
View File

@ -10,9 +10,11 @@ using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Common.Exceptions;
using Wabbajack.Lib;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
@ -27,12 +29,14 @@ namespace Wabbajack.BuildServer.Controllers
private static long ForwardCount = 0;
private SqlService _sql;
private ILogger<NexusCache> _logger;
private NexusKeyMaintainance _keys;
public NexusCache(ILogger<NexusCache> logger, SqlService sql, AppSettings settings)
public NexusCache(ILogger<NexusCache> logger, SqlService sql, AppSettings settings, NexusKeyMaintainance keys)
{
_settings = settings;
_sql = sql;
_logger = logger;
_keys = keys;
}
/// <summary>
@ -74,7 +78,7 @@ namespace Wabbajack.BuildServer.Controllers
{
var key = Request.Headers["apikey"].FirstOrDefault();
if (key == null)
return await NexusApiClient.Get(null);
return await _keys.GetClient();
if (await _sql.HaveKey(key))
return await NexusApiClient.Get(key);
@ -89,18 +93,31 @@ namespace Wabbajack.BuildServer.Controllers
[Route("{GameName}/mods/{ModId}/files.json")]
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, long ModId)
{
_logger.Log(LogLevel.Information, $"{GameName} {ModId}");
//_logger.Log(LogLevel.Information, $"{GameName} {ModId}");
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var result = await _sql.GetModFiles(game, ModId);
string method = "CACHED";
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
result = await api.GetModFiles(game, ModId, false);
var api = await GetClient();
var permission = HTMLInterface.GetUploadPermissions(game, ModId);
try
{
result = await api.GetModFiles(game, ModId, false);
}
catch (HttpException ex)
{
if (ex.Code == 403)
result = new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>()};
else
throw;
}
var date = result.files.Select(f => f.uploaded_time).OrderByDescending(o => o).FirstOrDefault();
date = date == default ? DateTime.UtcNow : date;
await _sql.AddNexusModFiles(game, ModId, date, result);
await _sql.SetNexusPermission(game, ModId, await permission);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);

19
Wabbajack.Server/DataLayer/ArchiveDownloads.cs
View File

@ -83,6 +83,25 @@ namespace Wabbajack.Server.DataLayer
}
public async Task<ArchiveDownload> GetArchiveDownload(string primaryKeyString)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND IsFailed = 0",
new {PrimaryKeyString = primaryKeyString});
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
public async Task<ArchiveDownload> GetArchiveDownload(string primaryKeyString, Hash hash, long size)
{
await using var conn = await Open();

14
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -117,6 +117,7 @@ namespace Wabbajack.Server.DataLayer
await using var conn = await Open();
await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModPermissions WHERE ModId = @ModId", new {ModId = modId});
}
public async Task<Dictionary<(Game, long), HTMLInterface.PermissionValue>> GetNexusPermissions()
@ -128,6 +129,17 @@ namespace Wabbajack.Server.DataLayer
return results.ToDictionary(f => (GameRegistry.ByNexusID[f.Item1], f.Item2),
f => (HTMLInterface.PermissionValue)f.Item3);
}
public async Task<Dictionary<(Game, long), HTMLInterface.PermissionValue>> GetHiddenNexusMods()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<(int, long, int)>("SELECT NexusGameID, ModID, Permissions FROM NexusModPermissions WHERE Permissions = @Permissions",
new {Permissions = (int)HTMLInterface.PermissionValue.Hidden});
return results.ToDictionary(f => (GameRegistry.ByNexusID[f.Item1], f.Item2),
f => (HTMLInterface.PermissionValue)f.Item3);
}
public async Task SetNexusPermissions(IEnumerable<(Game, long, HTMLInterface.PermissionValue)> permissions)
{
@ -166,7 +178,7 @@ namespace Wabbajack.Server.DataLayer
await using var conn = await Open();
var tx = await conn.BeginTransactionAsync();
await conn.ExecuteAsync("DELETE FROM NexusModPermissions WHERE GameID = @GameID AND ModID = @ModID", new
await conn.ExecuteAsync("DELETE FROM NexusModPermissions WHERE NexusGameID = @GameID AND ModID = @ModID", new
{
GameID = game.MetaData().NexusGameId,
ModID = modId

113
Wabbajack.Server/Services/ListValidator.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
@ -46,6 +47,9 @@ namespace Wabbajack.Server.Services
using var queue = new WorkQueue();
var oldSummaries = Summaries;
var stopwatch = new Stopwatch();
stopwatch.Start();
var results = await data.ModLists.PMap(queue, async metadata =>
{
var oldSummary =
@ -54,15 +58,24 @@ namespace Wabbajack.Server.Services
var listArchives = await _sql.ModListArchives(metadata.Links.MachineURL);
var archives = await listArchives.PMap(queue, async archive =>
{
var (_, result) = await ValidateArchive(data, archive);
if (result == ArchiveStatus.InValid)
try
{
if (data.Mirrors.Contains(archive.Hash))
return (archive, ArchiveStatus.Mirrored);
return await TryToHeal(data, archive, metadata);
}
var (_, result) = await ValidateArchive(data, archive);
if (result == ArchiveStatus.InValid)
{
if (data.Mirrors.Contains(archive.Hash))
return (archive, ArchiveStatus.Mirrored);
return await TryToHeal(data, archive, metadata);
}
return (archive, result);
return (archive, result);
}
catch (Exception ex)
{
_logger.LogError(ex, $"During Validation of {archive.Hash} {archive.State.PrimaryKeyString}");
return (archive, ArchiveStatus.InValid);
}
});
var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);
@ -141,20 +154,16 @@ namespace Wabbajack.Server.Services
return (summary, detailed);
});
Summaries = results;
stopwatch.Stop();
_logger.LogInformation($"Finished Validation in {stopwatch.Elapsed}");
return Summaries.Count(s => s.Summary.HasFailures);
}
private AsyncLock _healLock = new AsyncLock();
private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive, ModlistMetadata modList)
{
using var _ = await _healLock.WaitAsync();
if (!(archive.State is IUpgradingState))
{
_logger.Log(LogLevel.Information, $"Cannot heal {archive.State.PrimaryKeyString} because it's not a healable state");
return (archive, ArchiveStatus.InValid);
}
var srcDownload = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size);
if (srcDownload == null || srcDownload.IsFailed == true)
{
@ -163,7 +172,7 @@ namespace Wabbajack.Server.Services
}
var patches = await _sql.PatchesForSource(srcDownload.Id);
var patches = await _sql.PatchesForSource(archive.Hash);
foreach (var patch in patches)
{
if (patch.Finished is null)
@ -176,28 +185,80 @@ namespace Wabbajack.Server.Services
if (status == ArchiveStatus.Valid)
return (archive, ArchiveStatus.Updated);
}
using var _ = await _healLock.WaitAsync();
var upgradeTime = DateTime.UtcNow;
var upgrade = await (archive.State as IUpgradingState)?.FindUpgrade(archive);
_logger.LogInformation($"Validator Finding Upgrade for {archive.Hash} {archive.State.PrimaryKeyString}");
Func<Archive, Task<AbsolutePath>> resolver = async findIt =>
{
_logger.LogInformation($"Quick find for {findIt.State.PrimaryKeyString}");
var foundArchive = await _sql.GetArchiveDownload(findIt.State.PrimaryKeyString);
if (foundArchive == null)
{
_logger.LogInformation($"No Quick find for {findIt.State.PrimaryKeyString}");
return default;
}
return _archives.TryGetPath(foundArchive.Archive.Hash, out var path) ? path : default;
};
var upgrade = await DownloadDispatcher.FindUpgrade(archive, resolver);
if (upgrade == default)
{
_logger.Log(LogLevel.Information, $"Cannot heal {archive.State.PrimaryKeyString} because an alternative wasn't found");
return (archive, ArchiveStatus.InValid);
}
_logger.LogInformation($"Upgrade {upgrade.Archive.State.PrimaryKeyString} found for {archive.State.PrimaryKeyString}");
await _archives.Ingest(upgrade.NewFile.Path);
var id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime);
{
}
var found = await _sql.GetArchiveDownload(upgrade.Archive.State.PrimaryKeyString, upgrade.Archive.Hash,
upgrade.Archive.Size);
Guid id;
if (found == null)
{
if (upgrade.NewFile.Path.Exists)
await _archives.Ingest(upgrade.NewFile.Path);
id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime);
}
else
{
id = found.Id;
}
var destDownload = await _sql.GetArchiveDownload(id);
await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload});
_logger.Log(LogLevel.Information, $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}");
await _discord.Send(Channel.Ham, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`" });
if (destDownload.Archive.Hash == srcDownload.Archive.Hash && destDownload.Archive.State.PrimaryKeyString == srcDownload.Archive.State.PrimaryKeyString)
{
_logger.Log(LogLevel.Information, $"Can't heal because src and dest match");
return (archive, ArchiveStatus.InValid);
}
var existing = await _sql.FindPatch(srcDownload.Id, destDownload.Id);
if (existing == null)
{
await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload});
_logger.Log(LogLevel.Information,
$"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`"
});
}
await upgrade.NewFile.DisposeAsync();
_logger.LogInformation($"Patch in progress {archive.Hash} {archive.State.PrimaryKeyString}");
return (archive, ArchiveStatus.Updating);
}

7
Wabbajack.Server/Services/NexusKeyMaintainance.cs
View File

@ -4,6 +4,7 @@ using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
@ -12,6 +13,7 @@ namespace Wabbajack.Server.Services
public class NexusKeyMaintainance : AbstractService<NexusKeyMaintainance, int>
{
private SqlService _sql;
private string _selfKey;
public NexusKeyMaintainance(ILogger<NexusKeyMaintainance> logger, AppSettings settings, SqlService sql, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(4))
{
@ -21,7 +23,7 @@ namespace Wabbajack.Server.Services
public async Task<NexusApiClient> GetClient()
{
var keys = await _sql.GetNexusApiKeysWithCounts(1500);
foreach (var key in keys)
foreach (var key in keys.Where(k => k.Key != _selfKey))
{
return new TrackingClient(_sql, key);
}
@ -31,6 +33,7 @@ namespace Wabbajack.Server.Services
public override async Task<int> Execute()
{
_selfKey ??= await Utils.FromEncryptedJson<string>("nexusapikey");
var keys = await _sql.GetNexusApiKeysWithCounts(0);
_logger.Log(LogLevel.Information, $"Verifying {keys.Count} API Keys");
foreach (var key in keys)
@ -70,7 +73,7 @@ namespace Wabbajack.Server.Services
HourlyRemaining = key.Hourly;
}
protected virtual async Task UpdateRemaining(HttpResponseMessage response)
protected override async Task UpdateRemaining(HttpResponseMessage response)
{
await base.UpdateRemaining(response);
try

41
Wabbajack.Server/Services/NexusPermissionsUpdater.cs
View File

@ -17,9 +17,7 @@ namespace Wabbajack.Server.Services
private DiscordWebHook _discord;
private SqlService _sql;
public static TimeSpan MaxSync = TimeSpan.FromHours(4);
public NexusPermissionsUpdater(ILogger<NexusKeyMaintainance> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromSeconds(1))
public NexusPermissionsUpdater(ILogger<NexusKeyMaintainance> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromMinutes(5))
{
_discord = discord;
_sql = sql;
@ -32,38 +30,33 @@ namespace Wabbajack.Server.Services
var data = await _sql.ModListArchives();
var nexusArchives = data.Select(a => a.State).OfType<NexusDownloader.State>().Select(d => (d.Game, d.ModID))
.Where(g => g.Game.MetaData().NexusGameId != 0)
.Distinct()
.ToList();
_logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods");
using var queue = new WorkQueue(1);
using var queue = new WorkQueue();
var prev = await _sql.GetNexusPermissions();
var prev = await _sql.GetHiddenNexusMods();
_logger.LogInformation($"Found {prev.Count} hidden nexus mods to check");
var lag = MaxSync / nexusArchives.Count * 2;
await nexusArchives.PMap(queue, async archive =>
await prev.PMap(queue, async archive =>
{
_logger.LogInformation($"Checking permissions for {archive.Game} {archive.ModID}");
var result = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID);
await _sql.SetNexusPermission(archive.Game, archive.ModID, result);
var (game, modID) = archive.Key;
_logger.LogInformation($"Checking permissions for {game} {modID}");
var result = await HTMLInterface.GetUploadPermissions(game, modID);
await _sql.SetNexusPermission(game, modID, result);
if (prev.TryGetValue((archive.Game, archive.ModID), out var oldPermission))
if (archive.Value != result)
{
if (oldPermission != result)
{
await _discord.Send(Channel.Spam,
new DiscordMessage {
Content = $"Permissions status of {archive.Game} {archive.ModID} was {oldPermission} is now {result}"
});
await _sql.PurgeNexusCache(archive.ModID);
await _quickSync.Notify<ListValidator>();
}
await _discord.Send(Channel.Ham,
new DiscordMessage {
Content = $"Permissions status of {game} {modID} was {archive.Value} is now {result}"
});
await _sql.PurgeNexusCache(modID);
await _quickSync.Notify<ListValidator>();
}
await Task.Delay(lag);
});
return 1;

6
Wabbajack.Server/Services/NexusPoll.cs
View File

@ -18,13 +18,15 @@ namespace Wabbajack.Server.Services
private AppSettings _settings;
private GlobalInformation _globalInformation;
private ILogger<NexusPoll> _logger;
private NexusKeyMaintainance _keys;
public NexusPoll(ILogger<NexusPoll> logger, AppSettings settings, SqlService service, GlobalInformation globalInformation)
public NexusPoll(ILogger<NexusPoll> logger, AppSettings settings, SqlService service, GlobalInformation globalInformation, NexusKeyMaintainance keys)
{
_sql = service;
_settings = settings;
_globalInformation = globalInformation;
_logger = logger;
_keys = keys;
}
public async Task UpdateNexusCacheRSS()
@ -67,7 +69,7 @@ namespace Wabbajack.Server.Services
{
using var _ = _logger.BeginScope("Nexus Update via API");
_logger.Log(LogLevel.Information, "Starting Nexus Update via API");
var api = await NexusApiClient.Get();
var api = await _keys.GetClient();
var gameTasks = GameRegistry.Games.Values
.Where(game => game.NexusName != null)

8
Wabbajack.Server/Services/PatchBuilder.cs
View File

@ -55,7 +55,7 @@ namespace Wabbajack.Server.Services
$"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
if (patch.Src.Archive.Hash == patch.Dest.Archive.Hash)
if (patch.Src.Archive.Hash == patch.Dest.Archive.Hash && patch.Src.Archive.State.PrimaryKeyString == patch.Dest.Archive.State.PrimaryKeyString)
{
await patch.Fail(_sql, "Hashes match");
continue;
@ -76,7 +76,7 @@ namespace Wabbajack.Server.Services
await using var destStream = await destPath.OpenShared();
await using var sigStream = await sigFile.Path.Create();
await using var patchOutput = await patchFile.Path.Create();
OctoDiff.Create(destStream, srcStream, sigStream, patchOutput);
OctoDiff.Create(destStream, srcStream, sigStream, patchOutput, new OctoDiff.ProgressReporter(TimeSpan.FromSeconds(1), (s, p) => _logger.LogInformation($"Patch Builder: {p} {s}")));
await patchOutput.DisposeAsync();
var size = patchFile.Path.Size;
@ -170,6 +170,10 @@ namespace Wabbajack.Server.Services
var lst = p.Split("_", StringSplitOptions.RemoveEmptyEntries).Select(Hash.FromHex).ToArray();
return (lst[0], lst[1]);
}
catch (ArgumentException ex)
{
return default;
}
catch (FormatException ex)
{
return default;

14
Wabbajack.Server/Services/QuickSync.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
namespace Wabbajack.Server.Services
@ -11,6 +12,12 @@ namespace Wabbajack.Server.Services
{
private Dictionary<Type, CancellationTokenSource> _syncs = new Dictionary<Type, CancellationTokenSource>();
private AsyncLock _lock = new AsyncLock();
private ILogger<QuickSync> _logger;
public QuickSync(ILogger<QuickSync> logger)
{
_logger = logger;
}
public async Task<CancellationToken> GetToken<T>()
{
@ -36,18 +43,13 @@ namespace Wabbajack.Server.Services
public async Task Notify<T>()
{
_logger.LogInformation($"Quicksync {typeof(T).Name}");
// Needs debugging
/*
using var _ = await _lock.WaitAsync();
if (_syncs.TryGetValue(typeof(T), out var ct))
{
ct.Cancel();
}
*/
}
}
}

13
Wabbajack.Test/DownloaderTests.cs
View File

@ -308,6 +308,18 @@ namespace Wabbajack.Test
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
var files = await ((LoversLabDownloader.State)converted).GetFilesInGroup();
Assert.NotEmpty(files);
Assert.Equal("WABBAJACK_TEST_FILE.zip", files.First().Name);
((LoversLabDownloader.State)converted).FileID = "42";
var upgrade = await DownloadDispatcher.FindUpgrade(new Archive(converted) {Name = "WABBAJACK_TEST_FILE.zip"});
Assert.True(upgrade != default);
}
[Fact]
@ -336,6 +348,7 @@ namespace Wabbajack.Test
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
}
[Fact]