From f036420b42af38e855dc9595f389a40cfe83925d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 13 Feb 2020 05:29:59 -0700 Subject: [PATCH] * Disable server-side indexing of all mods from the Nexus * Accept download states from clients and index the mods we haven't seen * Fixes for Skyrin VR USSEP patch * Remember the download states that we index on the server * Only print remaining nexus quotas when they change --- CHANGELOG.md | 12 ++++- .../Controllers/IndexedFiles.cs | 51 +++++++++++++++++++ Wabbajack.BuildServer/JobManager.cs | 1 - Wabbajack.BuildServer/Models/Jobs/IndexJob.cs | 10 +++- .../Wabbajack.BuildServer.csproj | 7 +-- Wabbajack.Common/GameMetaData.cs | 3 +- Wabbajack.Common/Utils.cs | 7 +++ .../Downloaders/GameFileSourceDownloader.cs | 6 +-- Wabbajack.Lib/FileUploader/AuthorAPI.cs | 31 +++++++++++ Wabbajack.Lib/MO2Compiler.cs | 4 ++ Wabbajack.Lib/NexusApi/NexusApi.cs | 7 ++- 11 files changed, 127 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ea1ba99..5b8bc63d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ### Changelog +#### Version - 0.9.19.0 +* Disable server-side indexing of all mods from the Nexus +* Accept download states from clients and index the mods we haven't seen +* Fixes for Skyrin VR USSEP patch +* Remember the download states that we index on the server +* Only print remaining nexus quotas when they change + +#### Version - 0.9.18.0 - 2/11/2020 * Auto update functionality added client-side. * Slideshow now moves to next slide when users clicks, even if paused * Installer now prints to log what modlist it is installing @@ -7,8 +15,8 @@ that mod (issue #465) * Added support for non-premium Nexus downloads via manual downloading through the in-app browser. * Downloads from Bethesda.NET are now supported. Can login via SkyrimSE or Fallout 4. - -======= +* Manual URL downloads are streamlined +* AFKMods.com download support is improved #### Version - 1.0 beta 17 - 1/22/2020 * Build server now indexes CDN files after they are uploaded diff --git a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs index a82c050e..2a3e8865 100644 --- a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs @@ -1,6 +1,9 @@ using System; using System.Collections.Generic; +using System.IO; +using System.IO.Compression; using System.Linq; +using System.Text; using System.Threading.Tasks; using DynamicData; using Microsoft.AspNetCore.Authorization; @@ -11,7 +14,10 @@ using MongoDB.Driver; using MongoDB.Driver.Linq; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.Common; +using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.VirtualFileSystem; using IndexedFile = Wabbajack.BuildServer.Models.IndexedFile; @@ -87,6 +93,51 @@ namespace Wabbajack.BuildServer.Controllers return Ok(acc.ToList()); } + [HttpPost] + [Route("notify")] + public async Task Notify() + { + Utils.Log("Starting ingestion of uploaded INIs"); + var body = await Request.Body.ReadAllAsync(); + await using var ms = new MemoryStream(body); + using var za = new ZipArchive(ms, ZipArchiveMode.Read); + int enqueued = 0; + foreach (var entry in za.Entries) + { + await using var ins = entry.Open(); + var iniString = Encoding.UTF8.GetString(await ins.ReadAllAsync()); + var data = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(iniString.LoadIniString())); + if (data == null) + { + Utils.Log("No valid INI parser for: \n" + iniString); + continue; + } + + var key = data.PrimaryKeyString; + var found = await Db.DownloadStates.AsQueryable().Where(f => f.Key == key).Take(1).ToListAsync(); + if (found.Count > 0) + continue; + + await Db.Jobs.InsertOneAsync(new Job + { + Priority = Job.JobPriority.Low, + Payload = new IndexJob() + { + Archive = new Archive + { + Name = entry.Name, + State = data + } + } + }); + enqueued += 1; + } + + Utils.Log($"Enqueued {enqueued} out of {za.Entries.Count} entries from uploaded ini package"); + + return Ok(enqueued.ToString()); + } + [HttpGet] [Route("{xxHashAsBase64}")] public async Task GetFile(string xxHashAsBase64) diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index f4351078..aa1687d2 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -82,7 +82,6 @@ namespace Wabbajack.BuildServer await ScheduledJob(TimeSpan.FromMinutes(30), Job.JobPriority.High); await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.Low); await ScheduledJob(TimeSpan.FromHours(24), Job.JobPriority.High); - await ScheduledJob(TimeSpan.FromHours(6), Job.JobPriority.Low); await ScheduledJob(TimeSpan.FromHours(1), Job.JobPriority.Normal); await Task.Delay(10000); } diff --git a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs index 905c1a18..dd7c48e3 100644 --- a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs @@ -32,7 +32,7 @@ namespace Wabbajack.BuildServer.Models.Jobs var found = await db.DownloadStates.AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync(); if (found.Count > 0) return JobResult.Success(); - + string fileName = Archive.Name; string folder = Guid.NewGuid().ToString(); Utils.Log($"Indexer is downloading {fileName}"); @@ -46,6 +46,11 @@ namespace Wabbajack.BuildServer.Models.Jobs var archive = vfs.Index.ByRootPath.First().Value; await sql.MergeVirtualFile(archive); + + await db.DownloadStates.InsertOneAsync(new DownloadState + { + Key = pk_str, Hash = archive.Hash, State = Archive.State, IsValid = true + }); var to_path = Path.Combine(settings.ArchiveDir, $"{Path.GetFileName(fileName)}_{archive.Hash.FromBase64().ToHex()}_{Path.GetExtension(fileName)}"); @@ -54,7 +59,10 @@ namespace Wabbajack.BuildServer.Models.Jobs else File.Move(downloadDest, to_path); Utils.DeleteDirectory(Path.Combine(settings.DownloadDir, folder)); + } + + return JobResult.Success(); } diff --git a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj index 34e25873..f862ce31 100644 --- a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj +++ b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj @@ -4,10 +4,11 @@ netcoreapp3.1 aspnet-Wabbajack.BuildServer-6E798B30-DB04-4436-BE65-F043AF37B314 0 + true + true win10-x64 - Debug;Release - x64 - win10-x64 + Wabbajack + Wabbajack diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index e98bd6f8..62ce40c7 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -295,7 +295,8 @@ namespace Wabbajack.Common RequiredFiles = new List { "SkyrimVR.exe" - } + }, + MainExecutable = "SkyrimVR.exe" } }, { diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index a11db29f..45dd77ad 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -601,6 +601,13 @@ namespace Wabbajack.Common } } + public static async Task ReadAllAsync(this Stream ins) + { + await using var ms = new MemoryStream(); + await ins.CopyToAsync(ms); + return ms.ToArray(); + } + public static async Task PMap(this IEnumerable coll, WorkQueue queue, StatusUpdateTracker updateTracker, Func f) { diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs index 00da800e..fb412251 100644 --- a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs @@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders if (gameFile == null || gameFile == null) return null; - var game = GameRegistry.GetByMO2ArchiveName(gameName); + var game = GameRegistry.GetByFuzzyName(gameName); if (game == null) return null; var path = game.GameLocation(); @@ -34,10 +34,10 @@ namespace Wabbajack.Lib.Downloaders return new State { - Game = GameRegistry.GetByMO2ArchiveName(gameName).Game, + Game = game.Game, GameFile = gameFile, Hash = hash, - GameVersion = GameRegistry.GetByMO2ArchiveName(gameName).InstalledVersion + GameVersion = game.InstalledVersion }; } diff --git a/Wabbajack.Lib/FileUploader/AuthorAPI.cs b/Wabbajack.Lib/FileUploader/AuthorAPI.cs index b1c24bfb..ad2098cc 100644 --- a/Wabbajack.Lib/FileUploader/AuthorAPI.cs +++ b/Wabbajack.Lib/FileUploader/AuthorAPI.cs @@ -2,13 +2,16 @@ using System.Collections; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Net; using System.Net.Http; using System.Reactive.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using Wabbajack.Common; +using Wabbajack.Lib.Downloaders; using File = Alphaleonis.Win32.Filesystem.File; using Path = Alphaleonis.Win32.Filesystem.Path; @@ -140,5 +143,33 @@ namespace Wabbajack.Lib.FileUploader { return await RunJob("UpdateModLists"); } + + public static async Task UploadPackagedInis(IEnumerable archives) + { + archives = archives.ToArray(); // defensive copy + Utils.Log($"Packaging {archives.Count()} inis"); + try + { + await using var ms = new MemoryStream(); + using (var z = new ZipArchive(ms, ZipArchiveMode.Create, true)) + { + foreach (var archive in archives) + { + var state = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(archive.IniData)); + var entry = z.CreateEntry(Path.GetFileName(archive.Name)); + await using var os = entry.Open(); + await os.WriteAsync(Encoding.UTF8.GetBytes(string.Join("\n", state.GetMetaIni()))); + } + } + + var webClient = new WebClient(); + await webClient.UploadDataTaskAsync($"https://{Consts.WabbajackCacheHostname}/indexed_files/notify", + "POST", ms.ToArray()); + } + catch (Exception ex) + { + Utils.Log(ex.ToString()); + } + } } } diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 4fce0949..f6725caa 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -12,6 +12,7 @@ using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.FileUploader; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; using Directory = Alphaleonis.Win32.Filesystem.Directory; @@ -150,6 +151,9 @@ namespace Wabbajack.Lib Meta = File.ReadAllText(f + Consts.MetaFileExtension) }) .ToList(); + + // Don't await this because we don't care if it fails. + var _ = AuthorAPI.UploadPackagedInis(IndexedArchives); await CleanInvalidArchives(); diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index 36ea2f08..ca95f072 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -173,15 +173,20 @@ namespace Wabbajack.Lib.NexusApi { try { + var oldDaily = _dailyRemaining; + var oldHourly = _hourlyRemaining; var dailyRemaining = int.Parse(response.Headers.GetValues("x-rl-daily-remaining").First()); var hourlyRemaining = int.Parse(response.Headers.GetValues("x-rl-hourly-remaining").First()); - Utils.Log($"Nexus requests remaining: {dailyRemaining} daily - {hourlyRemaining} hourly"); lock (RemainingLock) { _dailyRemaining = Math.Min(dailyRemaining, hourlyRemaining); _hourlyRemaining = Math.Min(dailyRemaining, hourlyRemaining); } + + if (oldDaily != _dailyRemaining || oldHourly != _hourlyRemaining) + Utils.Log($"Nexus requests remaining: {dailyRemaining} daily - {hourlyRemaining} hourly"); + this.RaisePropertyChanged(nameof(DailyRemaining)); this.RaisePropertyChanged(nameof(HourlyRemaining)); }