From df68a5e2a4c089c661e4549719b4324a645b4b3c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 6 Feb 2021 09:43:11 -0700 Subject: [PATCH] 2.4.2.2 --- CHANGELOG.md | 4 + Wabbajack.CLI/Wabbajack.CLI.csproj | 4 +- Wabbajack.Common/Consts.cs | 1 + Wabbajack.Launcher/Wabbajack.Launcher.csproj | 4 +- Wabbajack.Lib/AInstaller.cs | 14 +- Wabbajack.Lib/MO2Installer.cs | 3 +- Wabbajack.Lib/Validation/ValidateModlist.cs | 2 +- .../ABuildServerSystemTest.cs | 6 +- Wabbajack.Server.Test/AuthoredFilesTests.cs | 42 +++++- Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs | 9 ++ Wabbajack.Server/DataLayer/AuthoredFiles.cs | 28 ++-- .../Services/AuthoredFilesCleanup.cs | 139 ++++++++++++++++++ Wabbajack.Server/Startup.cs | 2 + Wabbajack.Server/Wabbajack.Server.csproj | 4 +- Wabbajack.Server/public/metrics.html | 86 ++++++----- Wabbajack/Wabbajack.csproj | 4 +- 16 files changed, 288 insertions(+), 64 deletions(-) create mode 100644 Wabbajack.Server/Services/AuthoredFilesCleanup.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 75ae5a33..85acfcf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ### Changelog +#### Version - 2.4.2.2 - 2/6/2020 +* Better Origin game detection +* Don't check the download whitelist for files that are already downloaded + #### Version - 2.4.2.1 - 2/4/2020 * HOTFIX - fix for the download path sometimes being empty * HOTFIX - fix for some drive types not being detected (e.g. RAID drives) diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 884b9f22..a477c70f 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -6,8 +6,8 @@ wabbajack-cli Wabbajack x64 - 2.4.2.1 - 2.4.2.1 + 2.4.2.2 + 2.4.2.2 Copyright © 2019-2020 An automated ModList installer true diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 803cbe6a..659df9d4 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -132,6 +132,7 @@ namespace Wabbajack.Common public static RelativePath SettingsIni = (RelativePath)"settings.ini"; public static byte SettingsVersion => 2; public static TimeSpan MaxVerifyTime => TimeSpan.FromMinutes(10); + public static readonly string WabbajackAuthoredFilesPrefix = "https://wabbajack.b-cdn.net/"; public static RelativePath NativeSettingsJson = (RelativePath)"native_compiler_settings.json"; diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj index f2fa3380..f975a804 100644 --- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj +++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj @@ -4,8 +4,8 @@ Exe net5.0-windows true - 2.4.2.1 - 2.4.2.1 + 2.4.2.2 + 2.4.2.2 Copyright © 2019-2020 Wabbajack Application Launcher true diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 8930f44b..5df31d7d 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.Validation; using Wabbajack.VirtualFileSystem; using Path = Alphaleonis.Win32.Filesystem.Path; @@ -194,7 +195,9 @@ namespace Wabbajack.Lib Info("Getting Nexus API Key, if a browser appears, please accept"); - var dispatchers = missing.Select(m => m.State.GetDownloader()).Distinct(); + var dispatchers = missing.Select(m => m.State.GetDownloader()) + .Distinct() + .ToList(); await Task.WhenAll(dispatchers.Select(d => d.Prepare())); @@ -204,6 +207,14 @@ namespace Wabbajack.Lib throw new Exception($"Not enough Nexus API calls to download this list, please try again after midnight GMT when your API limits reset"); } + var validationData = new ValidateModlist(); + await validationData.LoadListsFromGithub(); + + foreach (var archive in missing.Where(archive => !archive.State.IsWhitelisted(validationData.ServerWhitelist))) + { + throw new Exception($"File {archive.State.PrimaryKeyString} failed validation"); + } + await DownloadMissingArchives(missing); } @@ -264,6 +275,7 @@ namespace Wabbajack.Lib } catch (Exception ex) { + var tsk = Metrics.Send("failed_download", archive.State.PrimaryKeyString); Utils.Log($"Download error for file {archive.Name}"); Utils.Log(ex.ToString()); return false; diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index f82a9cd9..33f93b59 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -103,8 +103,7 @@ namespace Wabbajack.Lib await ValidateGameESMs(); if (cancel.IsCancellationRequested) return false; - UpdateTracker.NextStep("Validating Modlist"); - await ValidateModlist.RunValidation(ModList); + UpdateTracker.NextStep("Creating Output Folders"); OutputFolder.CreateDirectory(); DownloadFolder.CreateDirectory(); diff --git a/Wabbajack.Lib/Validation/ValidateModlist.cs b/Wabbajack.Lib/Validation/ValidateModlist.cs index 43ac949b..360f2389 100644 --- a/Wabbajack.Lib/Validation/ValidateModlist.cs +++ b/Wabbajack.Lib/Validation/ValidateModlist.cs @@ -57,7 +57,7 @@ namespace Wabbajack.Lib.Validation public async Task> Validate(ModList modlist) { - ConcurrentStack ValidationErrors = new ConcurrentStack(); + ConcurrentStack ValidationErrors = new(); modlist.Archives .Where(m => !m.State.IsWhitelisted(ServerWhitelist)) .Do(m => diff --git a/Wabbajack.Server.Test/ABuildServerSystemTest.cs b/Wabbajack.Server.Test/ABuildServerSystemTest.cs index c3ae1621..62bfb879 100644 --- a/Wabbajack.Server.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.Server.Test/ABuildServerSystemTest.cs @@ -84,7 +84,11 @@ namespace Wabbajack.BuildServer.Test public T GetService() { - return (T)_host.Services.GetService(typeof(T)); + var result = (T)_host.Services.GetService(typeof(T)); + + if (result == null) + throw new Exception($"Service {typeof(T)} not found in configuration"); + return result; } diff --git a/Wabbajack.Server.Test/AuthoredFilesTests.cs b/Wabbajack.Server.Test/AuthoredFilesTests.cs index 7ddce64e..3f4eb15f 100644 --- a/Wabbajack.Server.Test/AuthoredFilesTests.cs +++ b/Wabbajack.Server.Test/AuthoredFilesTests.cs @@ -22,6 +22,11 @@ namespace Wabbajack.BuildServer.Test [Fact] public async Task CanUploadDownloadAndDeleteAuthoredFiles() { + var cleanup = Fixture.GetService(); + var sql = Fixture.GetService(); + + var toDelete = await cleanup.FindFilesToDelete(); + await using var file = new TempFile(); await file.Path.WriteAllBytesAsync(RandomData(Consts.UPLOADED_FILE_BLOCK_SIZE * 4 + Consts.UPLOADED_FILE_BLOCK_SIZE / 3)); var originalHash = await file.Path.FileHashAsync(); @@ -30,9 +35,23 @@ namespace Wabbajack.BuildServer.Test using var queue = new WorkQueue(2); var uri = await client.UploadFile(queue, file.Path, (s, percent) => Utils.Log($"({percent}) {s}")); - var data = await Fixture.GetService().AllAuthoredFiles(); + var data = (await Fixture.GetService().AllAuthoredFiles()).ToArray(); Assert.Contains((string)file.Path.FileName, data.Select(f => f.OriginalFileName)); + var listing = await cleanup.GetCDNMungedNames(); + foreach (var d in data) + { + Assert.Contains(d.MungedName, listing); + } + + // Just uploaded it, so it shouldn't be marked for deletion + toDelete = await cleanup.FindFilesToDelete(); + foreach (var d in data) + { + Assert.DoesNotContain(d.MungedName, toDelete.CDNDelete); + Assert.DoesNotContain(d.ServerAssignedUniqueId, toDelete.SQLDelete); + } + var result = await _client.GetStringAsync(MakeURL("authored_files")); Assert.Contains((string)file.Path.FileName, result); @@ -41,6 +60,27 @@ namespace Wabbajack.BuildServer.Test await state.Download(new Archive(state) {Name = (string)file.Path.FileName}, file.Path); Assert.Equal(originalHash, await file.Path.FileHashAsync()); + + // Mark it as old + foreach (var d in data) + { + await sql.TouchAuthoredFile(await sql.GetCDNFileDefinition(d.ServerAssignedUniqueId), DateTime.Now - TimeSpan.FromDays(8)); + } + + // Now it should be marked for deletion + toDelete = await cleanup.FindFilesToDelete(); + foreach (var d in data) + { + Assert.Contains(d.MungedName, toDelete.CDNDelete); + Assert.Contains(d.ServerAssignedUniqueId, toDelete.SQLDelete); + } + + await cleanup.Execute(); + + toDelete = await cleanup.FindFilesToDelete(); + + Assert.Empty(toDelete.CDNDelete); + Assert.Empty(toDelete.SQLDelete); } diff --git a/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs b/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs index 652e9612..34470cd9 100644 --- a/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs +++ b/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs @@ -1,5 +1,7 @@ using System.Collections.Generic; +using System.Net; using System.Threading.Tasks; +using FluentFTP; using Wabbajack.Common; namespace Wabbajack.Server.DTOs @@ -21,5 +23,12 @@ namespace Wabbajack.Server.DTOs { return (await Utils.FromEncryptedJson>("bunnycdn"))[space.ToString()]; } + + public async Task GetClient() + { + var ftpClient = new FtpClient(Hostname, new NetworkCredential(Username, Password)); + await ftpClient.ConnectAsync(); + return ftpClient; + } } } diff --git a/Wabbajack.Server/DataLayer/AuthoredFiles.cs b/Wabbajack.Server/DataLayer/AuthoredFiles.cs index 0dda7647..87f3b0c6 100644 --- a/Wabbajack.Server/DataLayer/AuthoredFiles.cs +++ b/Wabbajack.Server/DataLayer/AuthoredFiles.cs @@ -15,13 +15,21 @@ namespace Wabbajack.Server.DataLayer { public partial class SqlService { - public async Task TouchAuthoredFile(CDNFileDefinition definition) + public async Task TouchAuthoredFile(CDNFileDefinition definition, DateTime? date = null) { await using var conn = await Open(); - await conn.ExecuteAsync("UPDATE AuthoredFiles SET LastTouched = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid", - new { - Uid = definition.ServerAssignedUniqueId - }); + if (date == null) + { + await conn.ExecuteAsync( + "UPDATE AuthoredFiles SET LastTouched = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid", + new {Uid = definition.ServerAssignedUniqueId}); + } + else + { + await conn.ExecuteAsync( + "UPDATE AuthoredFiles SET LastTouched = @Date WHERE ServerAssignedUniqueId = @Uid", + new {Uid = definition.ServerAssignedUniqueId, Date = date}); + } } public async Task CreateAuthoredFile(CDNFileDefinition definition, string login) @@ -55,12 +63,13 @@ namespace Wabbajack.Server.DataLayer new {Uid = serverAssignedUniqueId})).First(); } - public async Task DeleteFileDefinition(CDNFileDefinition definition) + public async Task DeleteFileDefinition(CDNFileDefinition definition) { await using var conn = await Open(); - return (await conn.QueryAsync( + await conn.ExecuteAsync( "DELETE FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid", - new {Uid = definition.ServerAssignedUniqueId})).First(); + new {Uid = definition.ServerAssignedUniqueId}); + return; } public async Task> AllAuthoredFiles() @@ -68,8 +77,7 @@ namespace Wabbajack.Server.DataLayer await using var conn = await Open(); var results = await conn.QueryAsync("SELECT CONVERT(NVARCHAR(50), ServerAssignedUniqueId) as ServerAssignedUniqueId, Size, OriginalFileName, Author, LastTouched, Finalized, MungedName from dbo.AuthoredFilesSummaries ORDER BY LastTouched DESC"); return results; - } - + } } diff --git a/Wabbajack.Server/Services/AuthoredFilesCleanup.cs b/Wabbajack.Server/Services/AuthoredFilesCleanup.cs new file mode 100644 index 00000000..bfa992ed --- /dev/null +++ b/Wabbajack.Server/Services/AuthoredFilesCleanup.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Web; +using FluentFTP; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer; +using Wabbajack.Common; +using Wabbajack.Lib.AuthorApi; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.ModListRegistry; +using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; +using WebSocketSharp; + +namespace Wabbajack.Server.Services +{ + public class AuthoredFilesCleanup : AbstractService + { + private SqlService _sql; + private DiscordWebHook _discord; + + public AuthoredFilesCleanup(ILogger logger, AppSettings settings, QuickSync quickSync, SqlService sql, DiscordWebHook discord) : base(logger, settings, quickSync, TimeSpan.FromHours(6)) + { + _sql = sql; + _discord = discord; + } + + public override async Task Execute() + { + + var toDelete = await FindFilesToDelete(); + + var log = new[] {$"CDNDelete ({toDelete.CDNDelete.Length}):\n\n"} + .Concat(toDelete.CDNDelete) + .Concat(new[] {$"SQLDelete ({toDelete.SQLDelete.Length}"}) + .Concat(toDelete.SQLDelete) + .Concat(new[] {$"CDNRemain ({toDelete.CDNNotDeleted.Length}"}) + .Concat(toDelete.CDNNotDeleted) + .Concat(new[] {$"SQLRemain ({toDelete.SQLNotDeleted.Length}"}) + .Concat(toDelete.SQLNotDeleted) + .ToArray(); + + //await AbsolutePath.EntryPoint.Combine("cdn_delete_log.txt").WriteAllLinesAsync(log); + + + foreach (var sqlFile in toDelete.SQLDelete) + { + Utils.Log($"Deleting {sqlFile} from SQL"); + await _sql.DeleteFileDefinition(await _sql.GetCDNFileDefinition(sqlFile)); + } + + + using var queue = new WorkQueue(6); + await toDelete.CDNDelete.Select((d, idx) => (d, idx)).PMap(queue, async cdnFile => + { + using var conn = await (await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles)).GetClient(); + Utils.Log($"Deleting {cdnFile} from CDN"); + await _discord.Send(Channel.Ham, + new DiscordMessage + { + Content = + $"({cdnFile.idx}/{toDelete.CDNDelete.Length}) {cdnFile.d} is no longer referenced by any modlist and will be removed from the CDN" + }); + if (await conn.DirectoryExistsAsync(cdnFile.d)) + await conn.DeleteDirectoryAsync(cdnFile.d); + + if (await conn.FileExistsAsync(cdnFile.d)) + await conn.DeleteFileAsync(cdnFile.d); + }); + return toDelete.CDNDelete.Length + toDelete.SQLDelete.Length; + + } + + public async Task<(string[] CDNDelete, string[] SQLDelete, string[] CDNNotDeleted, string[] SQLNotDeleted)> FindFilesToDelete() + { + var cdnNames = (await GetCDNMungedNames()).ToHashSet(); + var usedNames = (await GetUsedCDNFiles()).ToHashSet(); + var sqlFiles = (await _sql.AllAuthoredFiles()).ToDictionary(f => f.MungedName); + var keep = GetKeepList(cdnNames, usedNames, sqlFiles).ToHashSet(); + + var cdnDelete = cdnNames.Where(h => !keep.Contains(h)).ToArray(); + var sqlDelete = sqlFiles.Where(s => !keep.Contains(s.Value.MungedName)) + .Select(s => s.Value.ServerAssignedUniqueId) + .ToArray(); + + var cdnhs = cdnDelete.ToHashSet(); + var notDeletedCDN = cdnNames.Where(f => !cdnhs.Contains(f)).ToArray(); + var sqlhs = sqlDelete.ToHashSet(); + var sqlNotDeleted = sqlFiles.Where(f => !sqlDelete.Contains(f.Value.ServerAssignedUniqueId)) + .Select(f => f.Value.MungedName) + .ToArray(); + return (cdnDelete, sqlDelete, notDeletedCDN, sqlNotDeleted); + } + + private IEnumerable GetKeepList(HashSet cdnNames, HashSet usedNames, Dictionary sqlFiles) + { + var cutOff = DateTime.UtcNow - TimeSpan.FromDays(7); + foreach (var file in sqlFiles.Where(f => f.Value.LastTouched > cutOff)) + yield return file.Value.MungedName; + + foreach (var file in usedNames) + yield return file; + } + + public async Task GetCDNMungedNames() + { + using var client = await (await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles)).GetClient(); + var lst = await client.GetListingAsync(@"\"); + return lst.Select(l => l.Name).ToArray(); + } + + public async Task GetUsedCDNFiles() + { + var modlists = (await ModlistMetadata.LoadFromGithub()) + .Concat((await ModlistMetadata.LoadUnlistedFromGithub())) + .Select(f => f.Links.Download) + .Where(f => f.StartsWith(Consts.WabbajackAuthoredFilesPrefix)) + .Select(f => f.Substring(Consts.WabbajackAuthoredFilesPrefix.Length)); + + var files = (await _sql.ModListArchives()) + .Select(a => a.State) + .OfType() + .Select(s => s.Url.ToString().Substring(Consts.WabbajackAuthoredFilesPrefix.Length)); + + + + var names = modlists.Concat(files).Distinct().ToArray(); + var namesBoth = names.Concat(names.Select(HttpUtility.UrlDecode)) + .Concat(names.Select(HttpUtility.UrlEncode)) + .Distinct() + .ToArray(); + return namesBoth; + } + } +} diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 6dfd3cf7..2d1c4e2c 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -75,6 +75,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddMvc(); services.AddControllers() @@ -135,6 +136,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); + app.UseService(); app.Use(next => { diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index 2b1e8f44..d6a3e28e 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -3,8 +3,8 @@ Exe net5.0-windows - 2.4.2.1 - 2.4.2.1 + 2.4.2.2 + 2.4.2.2 Copyright © 2019-2020 Wabbajack Server win-x64 diff --git a/Wabbajack.Server/public/metrics.html b/Wabbajack.Server/public/metrics.html index d81e4c0b..e9a22c0a 100644 --- a/Wabbajack.Server/public/metrics.html +++ b/Wabbajack.Server/public/metrics.html @@ -33,40 +33,45 @@
+

Exceptions

+ +
+ + diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index bd605dff..5e1e49fa 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -6,8 +6,8 @@ true x64 win10-x64 - 2.4.2.1 - 2.4.2.1 + 2.4.2.2 + 2.4.2.2 Copyright © 2019-2020 An automated ModList installer true