diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs index 8c73a6da..50398100 100644 --- a/Wabbajack.CLI/OptionsDefinition.cs +++ b/Wabbajack.CLI/OptionsDefinition.cs @@ -29,7 +29,8 @@ namespace Wabbajack.CLI typeof(ForceHealing), typeof(HashVariants), typeof(ParseMeta), - typeof(NoPatch) + typeof(NoPatch), + typeof(NexusPermissions) }; } } diff --git a/Wabbajack.CLI/Verbs/NexusPermissions.cs b/Wabbajack.CLI/Verbs/NexusPermissions.cs new file mode 100644 index 00000000..99fdf2bd --- /dev/null +++ b/Wabbajack.CLI/Verbs/NexusPermissions.cs @@ -0,0 +1,27 @@ +using System; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib.NexusApi; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("nexus-permissions", HelpText = "Get the nexus permissions for a mod")] + public class NexusPermissions : AVerb + { + [Option('m', "mod-id", Required = true, HelpText = "Mod Id")] + public long ModId { get; set; } = 0; + + [Option('g', "game", Required = true, HelpText = "Game Name")] + public string GameName { get; set; } = ""; + protected override async Task Run() + { + var game = GameRegistry.GetByFuzzyName(GameName).Game; + var p = await HTMLInterface.GetUploadPermissions(game, ModId); + Console.WriteLine($"Game: {game}"); + Console.WriteLine($"ModId: {ModId}"); + Console.WriteLine($"Permissions: {p}"); + return ExitCode.Ok; + } + } +} diff --git a/Wabbajack.Common/OctoDiff.cs b/Wabbajack.Common/OctoDiff.cs index 271274bb..440fdf84 100644 --- a/Wabbajack.Common/OctoDiff.cs +++ b/Wabbajack.Common/OctoDiff.cs @@ -7,14 +7,13 @@ namespace Wabbajack.Common { public class OctoDiff { - private static ProgressReporter reporter = new ProgressReporter(); public static void Create(byte[] oldData, byte[] newData, Stream output) { using var signature = CreateSignature(oldData); using var oldStream = new MemoryStream(oldData); using var newStream = new MemoryStream(newData); - var db = new DeltaBuilder {ProgressReporter = reporter}; - db.BuildDelta(newStream, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output))); + var db = new DeltaBuilder {ProgressReporter = new ProgressReporter()}; + db.BuildDelta(newStream, new SignatureReader(signature, new ProgressReporter()), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output))); } private static Stream CreateSignature(byte[] oldData) @@ -36,24 +35,39 @@ namespace Wabbajack.Common sigStream.Position = 0; } - public static void Create(Stream oldData, Stream newData, Stream signature, Stream output) + public static void Create(Stream oldData, Stream newData, Stream signature, Stream output, ProgressReporter? reporter = null) { CreateSignature(oldData, signature); - var db = new DeltaBuilder {ProgressReporter = reporter}; - db.BuildDelta(newData, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output))); + var db = new DeltaBuilder {ProgressReporter = reporter ?? new ProgressReporter()}; + db.BuildDelta(newData, new SignatureReader(signature, reporter ?? new ProgressReporter()), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output))); } - private class ProgressReporter : IProgressReporter + public class ProgressReporter : IProgressReporter { private DateTime _lastUpdate = DateTime.UnixEpoch; - private readonly TimeSpan _updateInterval = TimeSpan.FromMilliseconds(100); + private TimeSpan _updateInterval; + private Action _report; + + public ProgressReporter() + { + _updateInterval = TimeSpan.FromMilliseconds(100); + _report = (s, percent) => Utils.Status(s, percent); + } + + public ProgressReporter(TimeSpan updateInterval, Action report) + { + _updateInterval = updateInterval; + _report = report; + } + + public void ReportProgress(string operation, long currentPosition, long total) { if (DateTime.Now - _lastUpdate < _updateInterval) return; _lastUpdate = DateTime.Now; if (currentPosition >= total || total < 1 || currentPosition < 0) return; - Utils.Status(operation, new Percent(total, currentPosition)); + _report(operation, new Percent(total, currentPosition)); } } @@ -61,13 +75,13 @@ namespace Wabbajack.Common { using var deltaStream = openPatchStream(); var deltaApplier = new DeltaApplier(); - deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, reporter), output); + deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, new ProgressReporter()), output); } public static void Apply(FileStream input, FileStream patchStream, FileStream output) { var deltaApplier = new DeltaApplier(); - deltaApplier.Apply(input, new BinaryDeltaReader(patchStream, reporter), output); + deltaApplier.Apply(input, new BinaryDeltaReader(patchStream, new ProgressReporter()), output); } } } diff --git a/Wabbajack.Common/Util/AsyncLock.cs b/Wabbajack.Common/Util/AsyncLock.cs index 9b6a64b2..60f1d7ab 100644 --- a/Wabbajack.Common/Util/AsyncLock.cs +++ b/Wabbajack.Common/Util/AsyncLock.cs @@ -5,7 +5,6 @@ using System.Reactive.Disposables; using System.Text; using System.Threading; using System.Threading.Tasks; - namespace Wabbajack.Common { public class AsyncLock diff --git a/Wabbajack.Common/Util/TempFile.cs b/Wabbajack.Common/Util/TempFile.cs index 37145330..92717eb8 100644 --- a/Wabbajack.Common/Util/TempFile.cs +++ b/Wabbajack.Common/Util/TempFile.cs @@ -37,7 +37,7 @@ namespace Wabbajack.Common } public async ValueTask DisposeAsync() { - if (DeleteAfter) + if (DeleteAfter && Path.Exists) { await Path.DeleteAsync(); } diff --git a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs index 9a7e311c..9250a4a0 100644 --- a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs +++ b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs @@ -141,7 +141,7 @@ namespace Wabbajack.Lib.Downloaders } - public virtual async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a) + public virtual async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func> downloadResolver) { return await ServerFindUpgrade(a); } diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs index a582ef64..47b4a2c3 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; @@ -6,6 +7,8 @@ using System.Net.Http; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Web; +using F23.StringSimilarity; +using HtmlAgilityPack; using Newtonsoft.Json; using Wabbajack.Common; using Wabbajack.Lib.Validation; @@ -25,9 +28,13 @@ namespace Wabbajack.Lib.Downloaders } public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) - { + { Uri url = DownloaderUtils.GetDirectURL(archiveINI); + return await GetDownloaderStateFromUrl(url, quickMode); + } + public async Task GetDownloaderStateFromUrl(Uri url, bool quickMode) + { var absolute = true; if (url == null || url.Host != SiteURL.Host) return null; @@ -81,7 +88,7 @@ namespace Wabbajack.Lib.Downloaders FileName = file }; } - + public class State : AbstractDownloadState, IMetaState where TStateDownloader : IDownloader { @@ -223,12 +230,59 @@ namespace Wabbajack.Lib.Downloaders stream.Dispose(); return true; } - + public override IDownloader GetDownloader() { return DownloadDispatcher.GetInstance(); } + public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func> downloadResolver) + { + var files = await GetFilesInGroup(); + var nl = new Levenshtein(); + var newFile = files.OrderBy(f => nl.Distance(a.Name.ToLowerInvariant(), f.Name.ToLowerInvariant())).FirstOrDefault(); + if (newFile == null) return default; + + var existing = await downloadResolver(newFile); + if (existing != default) return (newFile, new TempFile()); + + var tmp = new TempFile(); + await DownloadDispatcher.PrepareAll(new []{newFile.State}); + if (await newFile.State.Download(newFile, tmp.Path)) + { + newFile.Size = tmp.Path.Size; + newFile.Hash = await tmp.Path.FileHashAsync(); + return (newFile, tmp); + } + + await tmp.DisposeAsync(); + return default; + + } + + public async Task> GetFilesInGroup() + { + var others = await Downloader.AuthedClient.GetHtmlAsync($"{Site}/files/file/{FileName}?do=download"); + + var pairs = others.DocumentNode.SelectNodes("//a[@data-action='download']") + .Select(item => (item.GetAttributeValue("href", ""), + item.ParentNode.ParentNode.SelectNodes("//div//h4//span").First().InnerText)); + + List archives = new List(); + foreach (var (url, name) in pairs) + { + var ini = new[] {"[General]", $"directURL={url}"}; + var state = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive( + string.Join("\n", ini).LoadIniString(), false)); + if (state == null) continue; + + archives.Add(new Archive(state) {Name = name}); + + } + + return archives; + } + public override string GetManifestURL(Archive a) { return IsAttachment ? FullURL : $"{Site}/files/file/{FileName}/?do=download&r={FileID}"; diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index e72ffcca..f5667578 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -112,11 +112,9 @@ namespace Wabbajack.Lib.Downloaders return false; } - var upgrade = (IUpgradingState)archive.State; - Utils.Log($"Trying to find solution to broken download for {archive.Name}"); - var result = await upgrade.FindUpgrade(archive); + var result = await FindUpgrade(archive); if (result == default) { Utils.Log( @@ -153,7 +151,14 @@ namespace Wabbajack.Lib.Downloaders return true; } + + public static async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func>? downloadResolver = null) + { + downloadResolver ??= async a => default; + return await a.State.FindUpgrade(a, downloadResolver); + } + private static async Task DownloadFromMirror(Archive archive, AbsolutePath destination) { try diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index 19e41589..3410030f 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -98,11 +98,13 @@ namespace Wabbajack.Lib.Downloaders var bufferSize = 1024 * 32; Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero); - var response = await client.GetAsync(Url); + var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false); TOP: - if (!response.IsSuccessStatusCode) + if (!response.IsSuccessStatusCode) + { return false; + } Stream stream; try @@ -212,19 +214,22 @@ TOP: } - public virtual async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a) + public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func> downloadResolver) { var tmpFile = new TempFile(); var newArchive = new Archive(this) {Name = a.Name}; + Utils.Log($"Downloading via HTTP to find Upgrade for {Url}"); + try { if (!await Download(newArchive, tmpFile.Path)) return default; } - catch (HttpException) + catch (HttpException ex) { + Utils.Log($"Error finding upgrade via HTTP to find Upgrade for {Url} {ex}"); return default; } diff --git a/Wabbajack.Lib/Downloaders/IUpgradingState.cs b/Wabbajack.Lib/Downloaders/IUpgradingState.cs index dd146d0d..d2b53a37 100644 --- a/Wabbajack.Lib/Downloaders/IUpgradingState.cs +++ b/Wabbajack.Lib/Downloaders/IUpgradingState.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +using System; +using System.Threading.Tasks; using Wabbajack.Common; namespace Wabbajack.Lib.Downloaders @@ -11,7 +12,7 @@ namespace Wabbajack.Lib.Downloaders /// /// /// - public Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a); + public Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func> downloadResolver); Task ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState); } diff --git a/Wabbajack.Lib/Downloaders/MEGADownloader.cs b/Wabbajack.Lib/Downloaders/MEGADownloader.cs index 157184c2..c1404cc0 100644 --- a/Wabbajack.Lib/Downloaders/MEGADownloader.cs +++ b/Wabbajack.Lib/Downloaders/MEGADownloader.cs @@ -194,7 +194,7 @@ namespace Wabbajack.Lib.Downloaders } } - public override Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a) + public override Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func> downloadResolver) { return ServerFindUpgrade(a); } diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 11a6a73f..1d6e7918 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders private bool _prepared; private AsyncLock _lock = new AsyncLock(); private UserStatus? _status; - private NexusApiClient? _client; + public INexusApi? Client; public IObservable IsLoggedIn => Utils.HaveEncryptedJsonObservable("nexusapikey"); @@ -115,9 +115,9 @@ namespace Wabbajack.Lib.Downloaders await CLIArguments.ApiKey.ToEcryptedJson("nexusapikey"); } - _client = await NexusApiClient.Get(); - _status = await _client.GetUserStatus(); - if (!_client.IsAuthenticated) + Client = await NexusApiClient.Get(); + _status = await Client.GetUserStatus(); + if (!Client.IsAuthenticated) { Utils.ErrorThrow(new UnconvertedError( $"Authenticating for the Nexus failed. A nexus account is required to automatically download mods.")); @@ -205,20 +205,13 @@ namespace Wabbajack.Lib.Downloaders try { var client = await NexusApiClient.Get(); + var modInfo = await client.GetModInfo(Game, ModID); + if (!modInfo.available) return false; var modFiles = await client.GetModFiles(Game, ModID); var found = modFiles.files .FirstOrDefault(file => file.file_id == FileID && file.category_name != null); - if (found != null) - return true; - - Utils.Log($"Could not validate {URL} with cache, validating manually"); - modFiles = await client.GetModFiles(Game, ModID, false); - - found = modFiles.files - .FirstOrDefault(file => file.file_id == FileID && file.category_name != null); - return found != null; } catch (Exception ex) @@ -244,11 +237,14 @@ namespace Wabbajack.Lib.Downloaders return new[] {"[General]", $"gameName={Game.MetaData().MO2ArchiveName}", $"modID={ModID}", $"fileID={FileID}"}; } - public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a) + public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func> downloadResolver) { var client = await NexusApiClient.Get(); var mod = await client.GetModInfo(Game, ModID); + if (!mod.available) + return default; + var files = await client.GetModFiles(Game, ModID); var oldFile = files.files.FirstOrDefault(f => f.file_id == FileID); var nl = new Levenshtein(); @@ -265,13 +261,21 @@ namespace Wabbajack.Lib.Downloaders return default; } - var tempFile = new TempFile(); - var newArchive = new Archive(new State {Game = Game, ModID = ModID, FileID = newFile.file_id}) { Name = newFile.file_name, }; + var fastPath = await downloadResolver(newArchive); + if (fastPath != default) + { + newArchive.Size = fastPath.Size; + newArchive.Hash = await fastPath.FileHashAsync(); + return (newArchive, new TempFile()); + } + + var tempFile = new TempFile(); + await newArchive.State.Download(newArchive, tempFile.Path); newArchive.Size = tempFile.Path.Size; diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index 8107a7ee..07619a82 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -89,7 +89,7 @@ namespace Wabbajack.Lib.ModListRegistry try { var client = new Http.Client(); - return (await client.GetStringAsync(Consts.ModlistMetadataURL)).FromJsonString>(); + return (await client.GetStringAsync(Consts.UnlistedModlistMetadataURL)).FromJsonString>(); } catch (Exception ex) { diff --git a/Wabbajack.Lib/NexusApi/INexusApi.cs b/Wabbajack.Lib/NexusApi/INexusApi.cs new file mode 100644 index 00000000..e6bb9efd --- /dev/null +++ b/Wabbajack.Lib/NexusApi/INexusApi.cs @@ -0,0 +1,17 @@ +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Lib.Downloaders; + +namespace Wabbajack.Lib.NexusApi +{ + public interface INexusApi + { + public Task GetNexusDownloadLink(NexusDownloader.State archive); + public Task GetModFiles(Game game, long modid, bool useCache = true); + public Task GetModInfo(Game game, long modId, bool useCache = true); + + public Task GetUserStatus(); + public Task IsPremium(); + public bool IsAuthenticated { get; } + } +} diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index 40e25835..c2be42d9 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -16,7 +16,7 @@ using Wabbajack.Lib.WebAutomation; namespace Wabbajack.Lib.NexusApi { - public class NexusApiClient : ViewModel + public class NexusApiClient : ViewModel, INexusApi { private static readonly string API_KEY_CACHE_FILE = "nexus.key_cache"; @@ -24,7 +24,7 @@ namespace Wabbajack.Lib.NexusApi #region Authentication - public string? ApiKey { get; } + public static string? ApiKey { get; set; } public bool IsAuthenticated => ApiKey != null; @@ -313,7 +313,11 @@ namespace Wabbajack.Lib.NexusApi public async Task GetNexusDownloadLink(NexusDownloader.State archive) { ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; - + + var info = await GetModInfo(archive.Game, archive.ModID); + if (!info.available) + throw new Exception("Mod unavailable"); + var url = $"https://api.nexusmods.com/v1/games/{archive.Game.MetaData().NexusName}/mods/{archive.ModID}/files/{archive.FileID}/download_link.json"; try { @@ -321,6 +325,8 @@ namespace Wabbajack.Lib.NexusApi } catch (HttpException ex) { + + if (ex.Code != 403 || await IsPremium()) { throw; @@ -383,20 +389,6 @@ namespace Wabbajack.Lib.NexusApi public string URI { get; set; } = string.Empty; } - private static string? _localCacheDir; - public static string LocalCacheDir - { - get - { - if (_localCacheDir == null) - _localCacheDir = Environment.GetEnvironmentVariable("NEXUSCACHEDIR"); - if (_localCacheDir == null) - throw new ArgumentNullException($"Enviornment variable could not be located: NEXUSCACHEDIR"); - return _localCacheDir; - } - set => _localCacheDir = value; - } - public static Uri ManualDownloadUrl(NexusDownloader.State state) { return new Uri($"https://www.nexusmods.com/{state.Game.MetaData().NexusName}/mods/{state.ModID}?tab=files"); diff --git a/Wabbajack.Server.Test/ABuildServerSystemTest.cs b/Wabbajack.Server.Test/ABuildServerSystemTest.cs index 7eadaa50..92fbf7ee 100644 --- a/Wabbajack.Server.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.Server.Test/ABuildServerSystemTest.cs @@ -170,7 +170,8 @@ namespace Wabbajack.BuildServer.Test Consts.ModlistSummaryURL = MakeURL("lists/status.json"); Consts.ServerWhitelistURL = MakeURL("ServerWhitelist.yaml"); - + Consts.UnlistedModlistMetadataURL = MakeURL("lists/none.json"); + } public WorkQueue Queue { get; set; } diff --git a/Wabbajack.Server.Test/ModListValidationTests.cs b/Wabbajack.Server.Test/ModListValidationTests.cs index 727aaec7..961e3a01 100644 --- a/Wabbajack.Server.Test/ModListValidationTests.cs +++ b/Wabbajack.Server.Test/ModListValidationTests.cs @@ -4,6 +4,7 @@ using System.IO.Compression; using System.Linq; using System.Text; using System.Threading.Tasks; +using Dapper; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; @@ -54,6 +55,10 @@ namespace Wabbajack.BuildServer.Test [Fact] public async Task CanValidateModLists() { + var sql = Fixture.GetService(); + await using var conn = await sql.Open(); + await conn.ExecuteAsync("DELETE from Patches"); + var modlists = await MakeModList("CanValidateModlistsFile.txt"); Consts.ModlistMetadataURL = modlists.ToString(); Utils.Log("Updating modlists"); diff --git a/Wabbajack.Server.Test/NexusCacheTests.cs b/Wabbajack.Server.Test/NexusCacheTests.cs index 2433a969..0c3ae82c 100644 --- a/Wabbajack.Server.Test/NexusCacheTests.cs +++ b/Wabbajack.Server.Test/NexusCacheTests.cs @@ -152,5 +152,19 @@ namespace Wabbajack.BuildServer.Test Assert.Equal(ts, (long)ts.AsUnixTime().AsUnixTime()); } + + [Fact] + public async Task CanGetAndSetPermissions() + { + var game = Game.Oblivion; + var modId = 4424; + var sql = Fixture.GetService(); + + foreach (HTMLInterface.PermissionValue result in Enum.GetValues(typeof(HTMLInterface.PermissionValue))) + { + await sql.SetNexusPermission(game, modId, result); + Assert.Equal(result, (await sql.GetNexusPermissions())[(game, modId)]); + } + } } } diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index 9ba90c06..d350b2dd 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -637,6 +637,7 @@ CREATE TABLE [dbo].[NexusKeys]( [ApiKey] [nvarchar](162) NOT NULL, [DailyRemain] [int] NOT NULL, [HourlyRemain] [int] NOT NULL, + [IsPremium] [tinyint] NOT NULL, CONSTRAINT [PK_NexusKeys] PRIMARY KEY CLUSTERED ( [ApiKey] ASC diff --git a/Wabbajack.Server/Controllers/Metrics.cs b/Wabbajack.Server/Controllers/Metrics.cs index e371e078..7889573c 100644 --- a/Wabbajack.Server/Controllers/Metrics.cs +++ b/Wabbajack.Server/Controllers/Metrics.cs @@ -127,7 +127,7 @@ namespace Wabbajack.BuildServer.Controllers private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null) { - _logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}"); + //_logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}"); await _sql.IngestMetric(new Metric { Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey diff --git a/Wabbajack.Server/Controllers/ModUpgrade.cs b/Wabbajack.Server/Controllers/ModUpgrade.cs index 00c2f873..e56be45b 100644 --- a/Wabbajack.Server/Controllers/ModUpgrade.cs +++ b/Wabbajack.Server/Controllers/ModUpgrade.cs @@ -73,15 +73,15 @@ namespace Wabbajack.BuildServer.Controllers { if (await request.OldArchive.State.Verify(request.OldArchive)) { - _logger.LogInformation( - $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), old archive is valid"); + //_logger.LogInformation( + // $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), old archive is valid"); return NotFound("File is Valid"); } } catch (Exception ex) { - _logger.LogInformation( - $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), due to upgrade failure"); + //_logger.LogInformation( + // $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), due to upgrade failure"); return NotFound("File is Valid"); } @@ -99,13 +99,13 @@ namespace Wabbajack.BuildServer.Controllers { if (patch.PatchSize != 0) { - _logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found"); + //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found"); await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id); return Ok( $"https://{(await _creds).Username}.b-cdn.net/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); } - _logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed"); + //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed"); return NotFound("Patch creation failed"); } @@ -119,7 +119,7 @@ namespace Wabbajack.BuildServer.Controllers await _quickSync.Notify(); } - _logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found is processing"); + //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found is processing"); // Still processing return Accepted(); } diff --git a/Wabbajack.Server/Controllers/NexusCache.cs b/Wabbajack.Server/Controllers/NexusCache.cs index 03f4e5ff..aa3db466 100644 --- a/Wabbajack.Server/Controllers/NexusCache.cs +++ b/Wabbajack.Server/Controllers/NexusCache.cs @@ -10,9 +10,11 @@ using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Wabbajack.Common; +using Wabbajack.Common.Exceptions; using Wabbajack.Lib; using Wabbajack.Lib.NexusApi; using Wabbajack.Server.DataLayer; +using Wabbajack.Server.Services; namespace Wabbajack.BuildServer.Controllers { @@ -27,12 +29,14 @@ namespace Wabbajack.BuildServer.Controllers private static long ForwardCount = 0; private SqlService _sql; private ILogger _logger; + private NexusKeyMaintainance _keys; - public NexusCache(ILogger logger, SqlService sql, AppSettings settings) + public NexusCache(ILogger logger, SqlService sql, AppSettings settings, NexusKeyMaintainance keys) { _settings = settings; _sql = sql; _logger = logger; + _keys = keys; } /// @@ -74,7 +78,7 @@ namespace Wabbajack.BuildServer.Controllers { var key = Request.Headers["apikey"].FirstOrDefault(); if (key == null) - return await NexusApiClient.Get(null); + return await _keys.GetClient(); if (await _sql.HaveKey(key)) return await NexusApiClient.Get(key); @@ -89,18 +93,31 @@ namespace Wabbajack.BuildServer.Controllers [Route("{GameName}/mods/{ModId}/files.json")] public async Task GetModFiles(string GameName, long ModId) { - _logger.Log(LogLevel.Information, $"{GameName} {ModId}"); + //_logger.Log(LogLevel.Information, $"{GameName} {ModId}"); var game = GameRegistry.GetByFuzzyName(GameName).Game; var result = await _sql.GetModFiles(game, ModId); string method = "CACHED"; if (result == null) { - var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - result = await api.GetModFiles(game, ModId, false); + var api = await GetClient(); + var permission = HTMLInterface.GetUploadPermissions(game, ModId); + try + { + result = await api.GetModFiles(game, ModId, false); + } + catch (HttpException ex) + { + if (ex.Code == 403) + result = new NexusApiClient.GetModFilesResponse {files = new List()}; + else + throw; + } + var date = result.files.Select(f => f.uploaded_time).OrderByDescending(o => o).FirstOrDefault(); date = date == default ? DateTime.UtcNow : date; await _sql.AddNexusModFiles(game, ModId, date, result); + await _sql.SetNexusPermission(game, ModId, await permission); method = "NOT_CACHED"; Interlocked.Increment(ref ForwardCount); diff --git a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs index 508a508f..0af2f679 100644 --- a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs +++ b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs @@ -83,6 +83,25 @@ namespace Wabbajack.Server.DataLayer } + public async Task GetArchiveDownload(string primaryKeyString) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>( + "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND IsFailed = 0", + new {PrimaryKeyString = primaryKeyString}); + if (result == default) + return null; + + return new ArchiveDownload + { + Id = result.Item1, + IsFailed = result.Item4, + DownloadFinished = result.Item6, + Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default} + }; + + } + public async Task GetArchiveDownload(string primaryKeyString, Hash hash, long size) { await using var conn = await Open(); diff --git a/Wabbajack.Server/DataLayer/Nexus.cs b/Wabbajack.Server/DataLayer/Nexus.cs index 487ca3a5..317045da 100644 --- a/Wabbajack.Server/DataLayer/Nexus.cs +++ b/Wabbajack.Server/DataLayer/Nexus.cs @@ -117,6 +117,7 @@ namespace Wabbajack.Server.DataLayer await using var conn = await Open(); await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId}); await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId}); + await conn.ExecuteAsync("DELETE FROM dbo.NexusModPermissions WHERE ModId = @ModId", new {ModId = modId}); } public async Task> GetNexusPermissions() @@ -128,6 +129,17 @@ namespace Wabbajack.Server.DataLayer return results.ToDictionary(f => (GameRegistry.ByNexusID[f.Item1], f.Item2), f => (HTMLInterface.PermissionValue)f.Item3); } + + public async Task> GetHiddenNexusMods() + { + await using var conn = await Open(); + + var results = + await conn.QueryAsync<(int, long, int)>("SELECT NexusGameID, ModID, Permissions FROM NexusModPermissions WHERE Permissions = @Permissions", + new {Permissions = (int)HTMLInterface.PermissionValue.Hidden}); + return results.ToDictionary(f => (GameRegistry.ByNexusID[f.Item1], f.Item2), + f => (HTMLInterface.PermissionValue)f.Item3); + } public async Task SetNexusPermissions(IEnumerable<(Game, long, HTMLInterface.PermissionValue)> permissions) { @@ -166,7 +178,7 @@ namespace Wabbajack.Server.DataLayer await using var conn = await Open(); var tx = await conn.BeginTransactionAsync(); - await conn.ExecuteAsync("DELETE FROM NexusModPermissions WHERE GameID = @GameID AND ModID = @ModID", new + await conn.ExecuteAsync("DELETE FROM NexusModPermissions WHERE NexusGameID = @GameID AND ModID = @ModID", new { GameID = game.MetaData().NexusGameId, ModID = modId diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 500c9006..478b3132 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -46,6 +47,9 @@ namespace Wabbajack.Server.Services using var queue = new WorkQueue(); var oldSummaries = Summaries; + var stopwatch = new Stopwatch(); + stopwatch.Start(); + var results = await data.ModLists.PMap(queue, async metadata => { var oldSummary = @@ -54,15 +58,24 @@ namespace Wabbajack.Server.Services var listArchives = await _sql.ModListArchives(metadata.Links.MachineURL); var archives = await listArchives.PMap(queue, async archive => { - var (_, result) = await ValidateArchive(data, archive); - if (result == ArchiveStatus.InValid) + try { - if (data.Mirrors.Contains(archive.Hash)) - return (archive, ArchiveStatus.Mirrored); - return await TryToHeal(data, archive, metadata); - } + var (_, result) = await ValidateArchive(data, archive); + if (result == ArchiveStatus.InValid) + { + if (data.Mirrors.Contains(archive.Hash)) + return (archive, ArchiveStatus.Mirrored); + return await TryToHeal(data, archive, metadata); + } - return (archive, result); + + return (archive, result); + } + catch (Exception ex) + { + _logger.LogError(ex, $"During Validation of {archive.Hash} {archive.State.PrimaryKeyString}"); + return (archive, ArchiveStatus.InValid); + } }); var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid); @@ -141,20 +154,16 @@ namespace Wabbajack.Server.Services return (summary, detailed); }); Summaries = results; + + stopwatch.Stop(); + _logger.LogInformation($"Finished Validation in {stopwatch.Elapsed}"); + return Summaries.Count(s => s.Summary.HasFailures); } private AsyncLock _healLock = new AsyncLock(); private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive, ModlistMetadata modList) { - using var _ = await _healLock.WaitAsync(); - - if (!(archive.State is IUpgradingState)) - { - _logger.Log(LogLevel.Information, $"Cannot heal {archive.State.PrimaryKeyString} because it's not a healable state"); - return (archive, ArchiveStatus.InValid); - } - var srcDownload = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size); if (srcDownload == null || srcDownload.IsFailed == true) { @@ -163,7 +172,7 @@ namespace Wabbajack.Server.Services } - var patches = await _sql.PatchesForSource(srcDownload.Id); + var patches = await _sql.PatchesForSource(archive.Hash); foreach (var patch in patches) { if (patch.Finished is null) @@ -176,28 +185,80 @@ namespace Wabbajack.Server.Services if (status == ArchiveStatus.Valid) return (archive, ArchiveStatus.Updated); } - - + + using var _ = await _healLock.WaitAsync(); var upgradeTime = DateTime.UtcNow; - var upgrade = await (archive.State as IUpgradingState)?.FindUpgrade(archive); + _logger.LogInformation($"Validator Finding Upgrade for {archive.Hash} {archive.State.PrimaryKeyString}"); + + Func> resolver = async findIt => + { + _logger.LogInformation($"Quick find for {findIt.State.PrimaryKeyString}"); + var foundArchive = await _sql.GetArchiveDownload(findIt.State.PrimaryKeyString); + if (foundArchive == null) + { + _logger.LogInformation($"No Quick find for {findIt.State.PrimaryKeyString}"); + return default; + } + + return _archives.TryGetPath(foundArchive.Archive.Hash, out var path) ? path : default; + }; + + var upgrade = await DownloadDispatcher.FindUpgrade(archive, resolver); + + if (upgrade == default) { _logger.Log(LogLevel.Information, $"Cannot heal {archive.State.PrimaryKeyString} because an alternative wasn't found"); return (archive, ArchiveStatus.InValid); } + + _logger.LogInformation($"Upgrade {upgrade.Archive.State.PrimaryKeyString} found for {archive.State.PrimaryKeyString}"); - await _archives.Ingest(upgrade.NewFile.Path); - var id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime); + { + } + + var found = await _sql.GetArchiveDownload(upgrade.Archive.State.PrimaryKeyString, upgrade.Archive.Hash, + upgrade.Archive.Size); + Guid id; + if (found == null) + { + if (upgrade.NewFile.Path.Exists) + await _archives.Ingest(upgrade.NewFile.Path); + id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime); + } + else + { + id = found.Id; + } + var destDownload = await _sql.GetArchiveDownload(id); - - await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload}); - - _logger.Log(LogLevel.Information, $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}"); - await _discord.Send(Channel.Ham, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`" }); + + if (destDownload.Archive.Hash == srcDownload.Archive.Hash && destDownload.Archive.State.PrimaryKeyString == srcDownload.Archive.State.PrimaryKeyString) + { + _logger.Log(LogLevel.Information, $"Can't heal because src and dest match"); + return (archive, ArchiveStatus.InValid); + } + + + var existing = await _sql.FindPatch(srcDownload.Id, destDownload.Id); + if (existing == null) + { + await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload}); + + _logger.Log(LogLevel.Information, + $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}"); + await _discord.Send(Channel.Ham, + new DiscordMessage + { + Content = + $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`" + }); + } await upgrade.NewFile.DisposeAsync(); + _logger.LogInformation($"Patch in progress {archive.Hash} {archive.State.PrimaryKeyString}"); return (archive, ArchiveStatus.Updating); } diff --git a/Wabbajack.Server/Services/NexusKeyMaintainance.cs b/Wabbajack.Server/Services/NexusKeyMaintainance.cs index ea033203..969248b9 100644 --- a/Wabbajack.Server/Services/NexusKeyMaintainance.cs +++ b/Wabbajack.Server/Services/NexusKeyMaintainance.cs @@ -4,6 +4,7 @@ using System.Net.Http; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Wabbajack.BuildServer; +using Wabbajack.Common; using Wabbajack.Lib.NexusApi; using Wabbajack.Server.DataLayer; @@ -12,6 +13,7 @@ namespace Wabbajack.Server.Services public class NexusKeyMaintainance : AbstractService { private SqlService _sql; + private string _selfKey; public NexusKeyMaintainance(ILogger logger, AppSettings settings, SqlService sql, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(4)) { @@ -21,7 +23,7 @@ namespace Wabbajack.Server.Services public async Task GetClient() { var keys = await _sql.GetNexusApiKeysWithCounts(1500); - foreach (var key in keys) + foreach (var key in keys.Where(k => k.Key != _selfKey)) { return new TrackingClient(_sql, key); } @@ -31,6 +33,7 @@ namespace Wabbajack.Server.Services public override async Task Execute() { + _selfKey ??= await Utils.FromEncryptedJson("nexusapikey"); var keys = await _sql.GetNexusApiKeysWithCounts(0); _logger.Log(LogLevel.Information, $"Verifying {keys.Count} API Keys"); foreach (var key in keys) @@ -70,7 +73,7 @@ namespace Wabbajack.Server.Services HourlyRemaining = key.Hourly; } - protected virtual async Task UpdateRemaining(HttpResponseMessage response) + protected override async Task UpdateRemaining(HttpResponseMessage response) { await base.UpdateRemaining(response); try diff --git a/Wabbajack.Server/Services/NexusPermissionsUpdater.cs b/Wabbajack.Server/Services/NexusPermissionsUpdater.cs index 6190c1ae..5879940a 100644 --- a/Wabbajack.Server/Services/NexusPermissionsUpdater.cs +++ b/Wabbajack.Server/Services/NexusPermissionsUpdater.cs @@ -17,9 +17,7 @@ namespace Wabbajack.Server.Services private DiscordWebHook _discord; private SqlService _sql; - public static TimeSpan MaxSync = TimeSpan.FromHours(4); - - public NexusPermissionsUpdater(ILogger logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromSeconds(1)) + public NexusPermissionsUpdater(ILogger logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromMinutes(5)) { _discord = discord; _sql = sql; @@ -32,38 +30,33 @@ namespace Wabbajack.Server.Services var data = await _sql.ModListArchives(); var nexusArchives = data.Select(a => a.State).OfType().Select(d => (d.Game, d.ModID)) + .Where(g => g.Game.MetaData().NexusGameId != 0) .Distinct() .ToList(); _logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods"); - using var queue = new WorkQueue(1); + using var queue = new WorkQueue(); - var prev = await _sql.GetNexusPermissions(); + var prev = await _sql.GetHiddenNexusMods(); + _logger.LogInformation($"Found {prev.Count} hidden nexus mods to check"); - var lag = MaxSync / nexusArchives.Count * 2; - - - await nexusArchives.PMap(queue, async archive => + await prev.PMap(queue, async archive => { - _logger.LogInformation($"Checking permissions for {archive.Game} {archive.ModID}"); - var result = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID); - await _sql.SetNexusPermission(archive.Game, archive.ModID, result); + var (game, modID) = archive.Key; + _logger.LogInformation($"Checking permissions for {game} {modID}"); + var result = await HTMLInterface.GetUploadPermissions(game, modID); + await _sql.SetNexusPermission(game, modID, result); - if (prev.TryGetValue((archive.Game, archive.ModID), out var oldPermission)) + if (archive.Value != result) { - if (oldPermission != result) - { - await _discord.Send(Channel.Spam, - new DiscordMessage { - Content = $"Permissions status of {archive.Game} {archive.ModID} was {oldPermission} is now {result}" - }); - await _sql.PurgeNexusCache(archive.ModID); - await _quickSync.Notify(); - } + await _discord.Send(Channel.Ham, + new DiscordMessage { + Content = $"Permissions status of {game} {modID} was {archive.Value} is now {result}" + }); + await _sql.PurgeNexusCache(modID); + await _quickSync.Notify(); } - - await Task.Delay(lag); }); return 1; diff --git a/Wabbajack.Server/Services/NexusPoll.cs b/Wabbajack.Server/Services/NexusPoll.cs index b1e77953..d7d90950 100644 --- a/Wabbajack.Server/Services/NexusPoll.cs +++ b/Wabbajack.Server/Services/NexusPoll.cs @@ -18,13 +18,15 @@ namespace Wabbajack.Server.Services private AppSettings _settings; private GlobalInformation _globalInformation; private ILogger _logger; + private NexusKeyMaintainance _keys; - public NexusPoll(ILogger logger, AppSettings settings, SqlService service, GlobalInformation globalInformation) + public NexusPoll(ILogger logger, AppSettings settings, SqlService service, GlobalInformation globalInformation, NexusKeyMaintainance keys) { _sql = service; _settings = settings; _globalInformation = globalInformation; _logger = logger; + _keys = keys; } public async Task UpdateNexusCacheRSS() @@ -67,7 +69,7 @@ namespace Wabbajack.Server.Services { using var _ = _logger.BeginScope("Nexus Update via API"); _logger.Log(LogLevel.Information, "Starting Nexus Update via API"); - var api = await NexusApiClient.Get(); + var api = await _keys.GetClient(); var gameTasks = GameRegistry.Games.Values .Where(game => game.NexusName != null) diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index 6cb4546c..47878007 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -55,7 +55,7 @@ namespace Wabbajack.Server.Services $"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}" }); - if (patch.Src.Archive.Hash == patch.Dest.Archive.Hash) + if (patch.Src.Archive.Hash == patch.Dest.Archive.Hash && patch.Src.Archive.State.PrimaryKeyString == patch.Dest.Archive.State.PrimaryKeyString) { await patch.Fail(_sql, "Hashes match"); continue; @@ -76,7 +76,7 @@ namespace Wabbajack.Server.Services await using var destStream = await destPath.OpenShared(); await using var sigStream = await sigFile.Path.Create(); await using var patchOutput = await patchFile.Path.Create(); - OctoDiff.Create(destStream, srcStream, sigStream, patchOutput); + OctoDiff.Create(destStream, srcStream, sigStream, patchOutput, new OctoDiff.ProgressReporter(TimeSpan.FromSeconds(1), (s, p) => _logger.LogInformation($"Patch Builder: {p} {s}"))); await patchOutput.DisposeAsync(); var size = patchFile.Path.Size; @@ -170,6 +170,10 @@ namespace Wabbajack.Server.Services var lst = p.Split("_", StringSplitOptions.RemoveEmptyEntries).Select(Hash.FromHex).ToArray(); return (lst[0], lst[1]); } + catch (ArgumentException ex) + { + return default; + } catch (FormatException ex) { return default; diff --git a/Wabbajack.Server/Services/QuickSync.cs b/Wabbajack.Server/Services/QuickSync.cs index 127644b7..3a772c21 100644 --- a/Wabbajack.Server/Services/QuickSync.cs +++ b/Wabbajack.Server/Services/QuickSync.cs @@ -3,6 +3,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Wabbajack.Common; namespace Wabbajack.Server.Services @@ -11,6 +12,12 @@ namespace Wabbajack.Server.Services { private Dictionary _syncs = new Dictionary(); private AsyncLock _lock = new AsyncLock(); + private ILogger _logger; + + public QuickSync(ILogger logger) + { + _logger = logger; + } public async Task GetToken() { @@ -36,18 +43,13 @@ namespace Wabbajack.Server.Services public async Task Notify() { + _logger.LogInformation($"Quicksync {typeof(T).Name}"); // Needs debugging - /* using var _ = await _lock.WaitAsync(); if (_syncs.TryGetValue(typeof(T), out var ct)) { ct.Cancel(); } - */ - } - - - } } diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 88b34e4f..d7a9f3a7 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -308,6 +308,18 @@ namespace Wabbajack.Test Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); + + var files = await ((LoversLabDownloader.State)converted).GetFilesInGroup(); + + Assert.NotEmpty(files); + Assert.Equal("WABBAJACK_TEST_FILE.zip", files.First().Name); + + ((LoversLabDownloader.State)converted).FileID = "42"; + + var upgrade = await DownloadDispatcher.FindUpgrade(new Archive(converted) {Name = "WABBAJACK_TEST_FILE.zip"}); + Assert.True(upgrade != default); + + } [Fact] @@ -336,6 +348,7 @@ namespace Wabbajack.Test Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); + } [Fact]