From 0c39698225c04754e156a60f200d59239c1d6d66 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 15 Jul 2020 16:29:43 -0600 Subject: [PATCH] Speed up list validation on the srver --- Wabbajack.Server/DTOs/ValidationData.cs | 2 +- Wabbajack.Server/DataLayer/Mappers.cs | 3 ++ Wabbajack.Server/DataLayer/ModLists.cs | 19 ++++++++-- Wabbajack.Server/DataLayer/ValidationData.cs | 6 ++-- Wabbajack.Server/Services/ListValidator.cs | 8 ++--- Wabbajack.Server/Services/PatchBuilder.cs | 38 +++++++++++++++++--- 6 files changed, 61 insertions(+), 15 deletions(-) diff --git a/Wabbajack.Server/DTOs/ValidationData.cs b/Wabbajack.Server/DTOs/ValidationData.cs index 4276dc88..f8b4e260 100644 --- a/Wabbajack.Server/DTOs/ValidationData.cs +++ b/Wabbajack.Server/DTOs/ValidationData.cs @@ -9,7 +9,7 @@ namespace Wabbajack.Server.DTOs { public ConcurrentHashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } = new ConcurrentHashSet<(long Game, long ModId, long FileId)>(); public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; } - public List<(ModlistMetadata Metadata, ModList ModList)> ModLists { get; set; } + public List ModLists { get; set; } public ConcurrentHashSet<(Game Game, long ModId)> SlowQueriedFor { get; set; } = new ConcurrentHashSet<(Game Game, long ModId)>(); } diff --git a/Wabbajack.Server/DataLayer/Mappers.cs b/Wabbajack.Server/DataLayer/Mappers.cs index 95312b18..f145ef07 100644 --- a/Wabbajack.Server/DataLayer/Mappers.cs +++ b/Wabbajack.Server/DataLayer/Mappers.cs @@ -2,8 +2,10 @@ using System.Data; using Dapper; using Wabbajack.Common; +using Wabbajack.Lib; using Wabbajack.Lib.AuthorApi; using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.ModListRegistry; namespace Wabbajack.Server.DataLayer { @@ -15,6 +17,7 @@ namespace Wabbajack.Server.DataLayer SqlMapper.AddTypeHandler(new RelativePathMapper()); SqlMapper.AddTypeHandler(new JsonMapper()); SqlMapper.AddTypeHandler(new JsonMapper()); + SqlMapper.AddTypeHandler(new JsonMapper()); SqlMapper.AddTypeHandler(new VersionMapper()); SqlMapper.AddTypeHandler(new GameMapper()); SqlMapper.AddTypeHandler(new DateTimeHandler()); diff --git a/Wabbajack.Server/DataLayer/ModLists.cs b/Wabbajack.Server/DataLayer/ModLists.cs index 1f62dc9f..4d83c129 100644 --- a/Wabbajack.Server/DataLayer/ModLists.cs +++ b/Wabbajack.Server/DataLayer/ModLists.cs @@ -1,9 +1,11 @@ -using System.Linq; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Dapper; using Wabbajack.Lib; using Wabbajack.Lib.ModListRegistry; using Wabbajack.Common; +using Wabbajack.Lib.Downloaders; namespace Wabbajack.Server.DataLayer { @@ -17,6 +19,11 @@ namespace Wabbajack.Server.DataLayer await conn.ExecuteAsync(@"DELETE FROM dbo.ModLists Where MachineUrl = @MachineUrl", new {MachineUrl = metadata.Links.MachineURL}, tran); + var archives = modlist.Archives; + var directives = modlist.Directives; + modlist.Archives = new List(); + modlist.Directives = new List(); + await conn.ExecuteAsync( @"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList, BrokenDownload) VALUES (@MachineUrl, @Hash, @Metadata, @ModList, @BrokenDownload)", new @@ -28,7 +35,7 @@ namespace Wabbajack.Server.DataLayer BrokenDownload = brokenDownload }, tran); - var entries = modlist.Archives.Select(a => + var entries = archives.Select(a => new { MachineUrl = metadata.Links.MachineURL, @@ -67,5 +74,13 @@ namespace Wabbajack.Server.DataLayer new {Hash = hash}); return result; } + + public async Task> ModListArchives(string machineURL) + { + await using var conn = await Open(); + var archives = await conn.QueryAsync<(Hash, long, AbstractDownloadState)>("SELECT Hash, Size, State FROM dbo.ModListArchives WHERE MachineUrl = @MachineUrl", + new {MachineUrl = machineURL}); + return archives.Select(t => new Archive(t.Item3) {Size = t.Item2, Hash = t.Item1}).ToList(); + } } } diff --git a/Wabbajack.Server/DataLayer/ValidationData.cs b/Wabbajack.Server/DataLayer/ValidationData.cs index 5f1cb027..b7f9a889 100644 --- a/Wabbajack.Server/DataLayer/ValidationData.cs +++ b/Wabbajack.Server/DataLayer/ValidationData.cs @@ -48,11 +48,11 @@ namespace Wabbajack.Server.DataLayer return results.ToHashSet(); } - public async Task> AllModLists() + public async Task> AllModLists() { await using var conn = await Open(); - var results = await conn.QueryAsync<(string, string)>(@"SELECT Metadata, ModList FROM dbo.ModLists"); - return results.Select(m => (m.Item1.FromJsonString(), m.Item2.FromJsonString())).ToList(); + var results = await conn.QueryAsync(@"SELECT Metadata FROM dbo.ModLists"); + return results.ToList(); } } } diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 3294fa9b..67dabd3f 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -46,13 +46,13 @@ namespace Wabbajack.Server.Services using var queue = new WorkQueue(); var oldSummaries = Summaries; - var results = await data.ModLists.PMap(queue, async list => + var results = await data.ModLists.PMap(queue, async metadata => { var oldSummary = - oldSummaries.FirstOrDefault(s => s.Summary.MachineURL == list.Metadata.Links.MachineURL); + oldSummaries.FirstOrDefault(s => s.Summary.MachineURL == metadata.Links.MachineURL); - var (metadata, modList) = list; - var archives = await modList.Archives.PMap(queue, async archive => + var listArchives = await _sql.ModListArchives(metadata.Links.MachineURL); + var archives = await listArchives.PMap(queue, async archive => { var (_, result) = await ValidateArchive(data, archive); if (result == ArchiveStatus.InValid) diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index 6e97130d..af2d4cd7 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; @@ -118,7 +119,13 @@ namespace Wabbajack.Server.Services private static string PatchName(Patch patch) { - return $"{Consts.ArchiveUpdatesCDNFolder}\\{patch.Src.Archive.Hash.ToHex()}_{patch.Dest.Archive.Hash.ToHex()}"; + return PatchName(patch.Src.Archive.Hash, patch.Dest.Archive.Hash); + + } + + private static string PatchName(Hash oldHash, Hash newHash) + { + return $"{Consts.ArchiveUpdatesCDNFolder}\\{oldHash.ToHex()}_{newHash.ToHex()}"; } private async Task CleanupOldPatches() @@ -154,18 +161,39 @@ namespace Wabbajack.Server.Services var sqlFiles = await _sql.AllPatchHashes(); _logger.LogInformation($"Found {sqlFiles.Count} in SQL"); - var hashPairs = files.Select(f => f.Name).Where(f => f.Contains("_")).Select(p => + HashSet<(Hash, Hash)> NamesToPairs(IEnumerable ftpFiles) { - var lst = p.Split("_", StringSplitOptions.RemoveEmptyEntries).Select(Hash.FromHex).ToArray(); - return (lst[0], lst[1]); - }).ToHashSet(); + return ftpFiles.Select(f => f.Name).Where(f => f.Contains("_")).Select(p => + { + try + { + var lst = p.Split("_", StringSplitOptions.RemoveEmptyEntries).Select(Hash.FromHex).ToArray(); + return (lst[0], lst[1]); + } + catch (FormatException ex) + { + return default; + } + }).Where(f => f != default).ToHashSet(); + } + var oldHashPairs = NamesToPairs(files.Where(f => DateTime.UtcNow - f.Modified > TimeSpan.FromDays(2))); + foreach (var (oldHash, newHash) in oldHashPairs.Where(o => !sqlFiles.Contains(o))) + { + _logger.LogInformation($"Removing CDN File entry for {oldHash} -> {newHash} it's not SQL"); + await client.DeleteFileAsync(PatchName(oldHash, newHash)); + } + + var hashPairs = NamesToPairs(files); foreach (var sqlFile in sqlFiles.Where(s => !hashPairs.Contains(s))) { _logger.LogInformation($"Removing SQL File entry for {sqlFile.Item1} -> {sqlFile.Item2} it's not on the CDN"); await _sql.DeletePatchesForHashPair(sqlFile); } + + + } private async Task UploadToCDN(AbsolutePath patchFile, string patchName)