diff --git a/CHANGELOG.md b/CHANGELOG.md index dc7e2f6e..f8def467 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Make the CDN downloads multi-threaded * Optimize installation of included files * Reinstate a broken feature with disabled mods +* Fix how JSON serializers handle dates (UTC all the things!) * Fix for Absolute Paths in Steam files #### Version - 2.0.4.4 - 5/11/2020 diff --git a/Wabbajack.Common/ConcurrentHashSet.cs b/Wabbajack.Common/ConcurrentHashSet.cs new file mode 100644 index 00000000..19d08122 --- /dev/null +++ b/Wabbajack.Common/ConcurrentHashSet.cs @@ -0,0 +1,30 @@ +using System.Collections.Generic; + +namespace Wabbajack.Common +{ + public class ConcurrentHashSet where T : notnull + { + private Dictionary _inner; + + public ConcurrentHashSet() + { + _inner = new Dictionary(); + } + public ConcurrentHashSet(IEnumerable input) + { + _inner = new Dictionary(); + foreach (var itm in input) + Add(itm); + } + + public bool Contains(T key) + { + return _inner.ContainsKey(key); + } + + public void Add(T key) + { + _inner[key] = true; + } + } +} diff --git a/Wabbajack.Common/Json.cs b/Wabbajack.Common/Json.cs index afa08f38..7bc93385 100644 --- a/Wabbajack.Common/Json.cs +++ b/Wabbajack.Common/Json.cs @@ -31,10 +31,15 @@ namespace Wabbajack.Common new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Objects, SerializationBinder = new JsonNameSerializationBinder(), - Converters = Converters}; + Converters = Converters, + DateTimeZoneHandling = DateTimeZoneHandling.Utc + }; public static JsonSerializerSettings GenericJsonSettings => - new JsonSerializerSettings(); + new JsonSerializerSettings + { + DateTimeZoneHandling = DateTimeZoneHandling.Utc, + }; public static void ToJson(this T obj, string filename) diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index 297dd029..adbe3dea 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -77,7 +77,7 @@ namespace Wabbajack.Common return; } - throw new InvalidDataException("Absolute path must be absolute"); + throw new InvalidDataException($"Absolute path must be absolute, got {_path}"); } public string Normalize() @@ -487,7 +487,7 @@ namespace Wabbajack.Common { if (Path.IsPathRooted(_path)) { - throw new InvalidDataException("Cannot create relative path from absolute path string"); + throw new InvalidDataException($"Cannot create relative path from absolute path string, got {_path}"); } } diff --git a/Wabbajack.Server.Test/NexusCacheTests.cs b/Wabbajack.Server.Test/NexusCacheTests.cs index b4f6e03d..97ffcf73 100644 --- a/Wabbajack.Server.Test/NexusCacheTests.cs +++ b/Wabbajack.Server.Test/NexusCacheTests.cs @@ -3,8 +3,12 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Wabbajack.Common; +using Wabbajack.Common.Serialization.Json; +using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.NexusApi; using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; using Xunit; using Xunit.Abstractions; @@ -54,5 +58,68 @@ namespace Wabbajack.BuildServer.Test Assert.Single(modInfoResponse.files); Assert.Equal("blerg", modInfoResponse.files.First().file_name); } + + [Fact] + public async Task CanQueryAndFindNexusModfilesSlow() + { + var startTime = DateTime.UtcNow; + var sql = Fixture.GetService(); + var validator = Fixture.GetService(); + await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow); + await sql.DeleteNexusModInfosUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow); + + var result = await validator.SlowNexusModStats(new ValidationData(), + new NexusDownloader.State {Game = Game.SkyrimSpecialEdition, ModID = 1137, FileID = 121449}); + Assert.Equal(ArchiveStatus.Valid, result); + + var gameId = Game.SkyrimSpecialEdition.MetaData().NexusGameId; + var hs = await sql.AllNexusFiles(); + + var found = hs.FirstOrDefault(h => + h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449); + Assert.True(found != default); + + Assert.True(found.LastChecked > startTime && found.LastChecked < DateTime.UtcNow); + + // Delete with exactly the same date, shouldn't clear out the record + await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, found.LastChecked); + var hs2 = await sql.AllNexusFiles(); + + var found2 = hs2.FirstOrDefault(h => + h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449); + Assert.True(found != default); + + Assert.True(found2.LastChecked == found.LastChecked); + + // Delete all the records, it should now be gone + await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow); + var hs3 = await sql.AllNexusFiles(); + Assert.DoesNotContain(hs3, f => f.NexusGameId == gameId && f.ModId == 1137); + + } + + + [JsonName("DateBox")] + class Box + { + public DateTime Value + { + get; + set; + } + } + [Fact] + public async Task DatesConvertProperly() + { + + var a = DateTime.Now; + var b = DateTime.UtcNow; + + Assert.NotEqual(a, new Box{Value = a}.ToJson().FromJsonString().Value); + Assert.Equal(b, new Box{Value = b}.ToJson().FromJsonString().Value); + Assert.NotEqual(a.Hour, b.Hour); + Assert.Equal(b.Hour, new Box{Value = a}.ToJson().FromJsonString().Value.Hour); + + } } } diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index 20afb36a..56c61b95 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -535,6 +535,22 @@ CREATE UNIQUE NONCLUSTERED INDEX [IX_IndexedFile_By_SHA256] ON [dbo].[IndexedFil [Sha256] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] GO + +/****** Object: Table [dbo].[NexusModFilesSlow] Script Date: 5/14/2020 2:23:15 PM ******/ +CREATE TABLE [dbo].[NexusModFilesSlow]( +[GameId] [bigint] NOT NULL, +[FileId] [bigint] NOT NULL, +[ModId] [bigint] NOT NULL, +[LastChecked] [datetime] NOT NULL, +CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED + ( + [GameId] ASC, + [FileId] ASC, + [ModId] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO + /****** Object: StoredProcedure [dbo].[MergeAllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/ SET ANSI_NULLS ON GO diff --git a/Wabbajack.Server/DTOs/ArchiveStatus.cs b/Wabbajack.Server/DTOs/ArchiveStatus.cs index 0e44971e..98c9c9bf 100644 --- a/Wabbajack.Server/DTOs/ArchiveStatus.cs +++ b/Wabbajack.Server/DTOs/ArchiveStatus.cs @@ -1,6 +1,6 @@ namespace Wabbajack.Server.DTOs { - enum ArchiveStatus + public enum ArchiveStatus { Valid, InValid, diff --git a/Wabbajack.Server/DTOs/ValidationData.cs b/Wabbajack.Server/DTOs/ValidationData.cs index 2db82ff7..4276dc88 100644 --- a/Wabbajack.Server/DTOs/ValidationData.cs +++ b/Wabbajack.Server/DTOs/ValidationData.cs @@ -7,8 +7,10 @@ namespace Wabbajack.Server.DTOs { public class ValidationData { - public HashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } + public ConcurrentHashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } = new ConcurrentHashSet<(long Game, long ModId, long FileId)>(); public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; } public List<(ModlistMetadata Metadata, ModList ModList)> ModLists { get; set; } + + public ConcurrentHashSet<(Game Game, long ModId)> SlowQueriedFor { get; set; } = new ConcurrentHashSet<(Game Game, long ModId)>(); } } diff --git a/Wabbajack.Server/DataLayer/Nexus.cs b/Wabbajack.Server/DataLayer/Nexus.cs index 6ff8bf75..a5ecb0a3 100644 --- a/Wabbajack.Server/DataLayer/Nexus.cs +++ b/Wabbajack.Server/DataLayer/Nexus.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; using Dapper; using Newtonsoft.Json; using Wabbajack.Common; @@ -29,6 +30,11 @@ namespace Wabbajack.Server.DataLayer @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date SELECT @@ROWCOUNT AS Deleted", new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date}); + + deleted += await conn.ExecuteScalarAsync( + @"DELETE FROM dbo.NexusModFilesSlow WHERE GameId = @Game AND ModID = @ModId AND LastChecked < @Date + SELECT @@ROWCOUNT AS Deleted", + new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date}); return deleted; } @@ -78,7 +84,25 @@ namespace Wabbajack.Server.DataLayer LastChecked = lastCheckedUtc, Data = JsonConvert.SerializeObject(data) }); - + } + + public async Task AddNexusModFileSlow(Game game, long modId, long fileId, DateTime lastCheckedUtc) + { + await using var conn = await Open(); + + await conn.ExecuteAsync( + @"MERGE dbo.NexusModFilesSlow AS Target + USING (SELECT @GameId GameId, @ModId ModId, @LastChecked LastChecked, @FileId FileId) AS Source + ON Target.GameId = Source.GameId AND Target.ModId = Source.ModId AND Target.FileId = Source.FileId + WHEN MATCHED THEN UPDATE SET Target.LastChecked = @LastChecked + WHEN NOT MATCHED THEN INSERT (GameId, ModId, LastChecked, FileId) VALUES (@GameId, @ModId, @LastChecked, FileId);", + new + { + GameId = game.MetaData().NexusGameId, + ModId = modId, + FileId = fileId, + LastChecked = lastCheckedUtc, + }); } public async Task GetModFiles(Game game, long modId) diff --git a/Wabbajack.Server/DataLayer/ValidationData.cs b/Wabbajack.Server/DataLayer/ValidationData.cs index f826104d..5f1cb027 100644 --- a/Wabbajack.Server/DataLayer/ValidationData.cs +++ b/Wabbajack.Server/DataLayer/ValidationData.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Dapper; @@ -18,7 +19,7 @@ namespace Wabbajack.Server.DataLayer var modLists = AllModLists(); return new ValidationData { - NexusFiles = await nexusFiles, + NexusFiles = new ConcurrentHashSet<(long Game, long ModId, long FileId)>((await nexusFiles).Select(f => (f.NexusGameId, f.ModId, f.FileId))), ArchiveStatus = await archiveStatus, ModLists = await modLists, }; @@ -33,14 +34,17 @@ namespace Wabbajack.Server.DataLayer return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3); } - public async Task> AllNexusFiles() + public async Task> AllNexusFiles() { await using var conn = await Open(); - var results = await conn.QueryAsync<(long, long, long)>(@"SELECT Game, ModId, p.file_id - FROM [NexusModFiles] files - CROSS APPLY - OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p - WHERE p.category is not null"); + var results = await conn.QueryAsync<(long, long, long, DateTime)>(@"SELECT Game, ModId, p.file_id, LastChecked + FROM [NexusModFiles] files + CROSS APPLY + OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p + WHERE p.category is not null + UNION + SELECT GameId, ModId, FileId, LastChecked FROM dbo.NexusModFilesSlow + "); return results.ToHashSet(); } diff --git a/Wabbajack.Server/GlobalInformation.cs b/Wabbajack.Server/GlobalInformation.cs index d1eb3705..2b51f673 100644 --- a/Wabbajack.Server/GlobalInformation.cs +++ b/Wabbajack.Server/GlobalInformation.cs @@ -5,7 +5,7 @@ namespace Wabbajack.Server public class GlobalInformation { public TimeSpan NexusRSSPollRate = TimeSpan.FromMinutes(1); - public TimeSpan NexusAPIPollRate = TimeSpan.FromHours(2); + public TimeSpan NexusAPIPollRate = TimeSpan.FromHours(24); public DateTime LastNexusSyncUTC { get; set; } public TimeSpan TimeSinceLastNexusSync => DateTime.UtcNow - LastNexusSyncUTC; } diff --git a/Wabbajack.Server/Services/ArchiveDownloader.cs b/Wabbajack.Server/Services/ArchiveDownloader.cs index 68069b53..7335bfeb 100644 --- a/Wabbajack.Server/Services/ArchiveDownloader.cs +++ b/Wabbajack.Server/Services/ArchiveDownloader.cs @@ -30,8 +30,9 @@ namespace Wabbajack.Server.Services while (true) { - var (daily, hourly) = await _nexusClient.GetRemainingApiCalls(); - bool ignoreNexus = hourly < 25; + //var (daily, hourly) = await _nexusClient.GetRemainingApiCalls(); + //bool ignoreNexus = hourly < 25; + var ignoreNexus = true; if (ignoreNexus) _logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})"); else diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 1216e0e9..6063931a 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -1,9 +1,12 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc.Filters; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; +using Org.BouncyCastle.Crypto.Digests; using RocksDbSharp; using Wabbajack.BuildServer; using Wabbajack.Common; @@ -91,7 +94,7 @@ namespace Wabbajack.Server.Services nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)): return (archive, ArchiveStatus.Valid); case NexusDownloader.State ns: - return (archive, await FastNexusModStats(ns)); + return (archive, await SlowNexusModStats(data, ns)); case ManualDownloader.State _: return (archive, ArchiveStatus.Valid); default: @@ -106,6 +109,50 @@ namespace Wabbajack.Server.Services } } } + + + private readonly AsyncLock _slowQueryLock = new AsyncLock(); + public async Task SlowNexusModStats(ValidationData data, NexusDownloader.State ns) + { + var gameId = ns.Game.MetaData().NexusGameId; + using var _ = await _slowQueryLock.WaitAsync(); + _logger.Log(LogLevel.Warning, $"Slow querying for {ns.Game} {ns.ModID} {ns.FileID}"); + + + if (data.NexusFiles.Contains((gameId, ns.ModID, ns.FileID))) + return ArchiveStatus.Valid; + + if (data.SlowQueriedFor.Contains((ns.Game, ns.ModID))) + return ArchiveStatus.InValid; + + var queryTime = DateTime.UtcNow; + var regex = new Regex("(?<=[?;&]file_id\\=)\\d+"); + var client = new Common.Http.Client(); + var result = + await client.GetHtmlAsync( + $"https://www.nexusmods.com/{ns.Game.MetaData().NexusName}/mods/{ns.ModID}/?tab=files"); + + var fileIds = result.DocumentNode.Descendants() + .Select(f => f.GetAttributeValue("href", "")) + .Select(f => + { + var match = regex.Match(f); + return !match.Success ? null : match.Value; + }) + .Where(m => m != null) + .Select(m => long.Parse(m)) + .Distinct() + .ToList(); + + _logger.Log(LogLevel.Warning, $"Slow queried {fileIds.Count} files"); + foreach (var id in fileIds) + { + await _sql.AddNexusModFileSlow(ns.Game, ns.ModID, id, queryTime); + data.NexusFiles.Add((gameId, ns.ModID, id)); + } + + return fileIds.Contains(ns.FileID) ? ArchiveStatus.Valid : ArchiveStatus.InValid; + } private async Task FastNexusModStats(NexusDownloader.State ns) { diff --git a/Wabbajack.Server/Services/NexusPoll.cs b/Wabbajack.Server/Services/NexusPoll.cs index d3691adc..bd2902cc 100644 --- a/Wabbajack.Server/Services/NexusPoll.cs +++ b/Wabbajack.Server/Services/NexusPoll.cs @@ -48,17 +48,6 @@ namespace Wabbajack.Server.Services if (totalPurged > 0) _logger.Log(LogLevel.Information, $"Purged {totalPurged} cache items {result.Game} {result.ModId} {result.TimeStamp}"); - if (await _sql.GetNexusModInfoString(result.Game, result.ModId) != null) continue; - - // Lazily create the client - client ??= await NexusApiClient.Get(); - - // Cache the info - var files = await client.GetModFiles(result.Game, result.ModId, false); - await _sql.AddNexusModFiles(result.Game, result.ModId, result.TimeStamp, files); - - var modInfo = await client.GetModInfo(result.Game, result.ModId, useCache: false); - await _sql.AddNexusModInfo(result.Game, result.ModId, result.TimeStamp, modInfo); updated++; } catch (Exception ex)