From 895bdb15a6c1710655d16806cf7789a928b50f7a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 30 Apr 2020 15:43:54 -0600 Subject: [PATCH 1/2] Print the archive being extracted when analysis fails. --- .../UploadedFilesTest.cs | 22 ++++++ .../Controllers/ListValidation.cs | 76 +++++++++++++------ .../Controllers/UploadedFiles.cs | 23 +++--- Wabbajack.Lib/ACompiler.cs | 4 +- Wabbajack.Lib/FileUploader/AuthorAPI.cs | 6 +- 5 files changed, 92 insertions(+), 39 deletions(-) diff --git a/Wabbajack.BuildServer.Test/UploadedFilesTest.cs b/Wabbajack.BuildServer.Test/UploadedFilesTest.cs index 6ba59e62..7b9d6b4e 100644 --- a/Wabbajack.BuildServer.Test/UploadedFilesTest.cs +++ b/Wabbajack.BuildServer.Test/UploadedFilesTest.cs @@ -64,6 +64,28 @@ namespace Wabbajack.BuildServer.Test } } + + [Fact] + public async Task CanDeleteFilesUsingClientApi() + { + using (var file = new TempFile()) + { + var data = new byte[1024]; + await using (var fs = file.Path.Create()) + { + await fs.WriteAsync(data); + } + + Utils.Log($"Uploading {file.Path.Size.ToFileSizeString()} file"); + var result = await AuthorAPI.UploadFile(file.Path, + progress => Utils.Log($"Uploading : {progress * 100}%"), Fixture.APIKey); + + Utils.Log($"Delete {result}"); + await AuthorAPI.DeleteFile((string)((RelativePath)new Uri(result).AbsolutePath).FileName); + + } + + } public UploadedFilesTest(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, fixture) { diff --git a/Wabbajack.BuildServer/Controllers/ListValidation.cs b/Wabbajack.BuildServer/Controllers/ListValidation.cs index 8722d6fa..4f02cd17 100644 --- a/Wabbajack.BuildServer/Controllers/ListValidation.cs +++ b/Wabbajack.BuildServer/Controllers/ListValidation.cs @@ -114,10 +114,14 @@ namespace Wabbajack.BuildServer.Controllers DownloadMetaData = metadata.DownloadMetadata, HasFailures = failedCount > 0, MachineName = metadata.Links.MachineURL, - Archives = archives.Select(a => new DetailedStatusItem + Archives = archives.Select(a => { - Archive = a.Item1, - IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating + a.Item1.Meta = ""; + return new DetailedStatusItem + { + Archive = a.Item1, + IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating + }; }).ToList() }; @@ -172,33 +176,59 @@ namespace Wabbajack.BuildServer.Controllers var mod = await SQL.GetNexusModInfoString(ns.Game, ns.ModID); var files = await SQL.GetModFiles(ns.Game, ns.ModID); - if (mod == null) + try { - Utils.Log($"Found missing Nexus mod info {ns.Game} {ns.ModID}"); - mod = await (await _nexusClient).GetModInfo(ns.Game, ns.ModID, false); - try + if (mod == null) { - await SQL.AddNexusModInfo(ns.Game, ns.ModID, mod.updated_time, mod); + Utils.Log($"Found missing Nexus mod info {ns.Game} {ns.ModID}"); + try + { + mod = await (await _nexusClient).GetModInfo(ns.Game, ns.ModID, false); + } + catch + { + mod = new ModInfo + { + mod_id = ns.ModID.ToString(), game_id = ns.Game.MetaData().NexusGameId, available = false + }; + } + + try + { + await SQL.AddNexusModInfo(ns.Game, ns.ModID, mod.updated_time, mod); + } + catch (Exception _) + { + // Could be a PK constraint failure + } + } - catch (Exception _) + + if (files == null) { - // Could be a PK constraint failure + Utils.Log($"Found missing Nexus mod file infos {ns.Game} {ns.ModID}"); + try + { + files = await (await _nexusClient).GetModFiles(ns.Game, ns.ModID, false); + } + catch + { + files = new NexusApiClient.GetModFilesResponse {files = new List()}; + } + + try + { + await SQL.AddNexusModFiles(ns.Game, ns.ModID, mod.updated_time, files); + } + catch (Exception _) + { + // Could be a PK constraint failure + } } } - - if (files == null) + catch (Exception ex) { - Utils.Log($"Found missing Nexus mod file infos {ns.Game} {ns.ModID}"); - files = await (await _nexusClient).GetModFiles(ns.Game, ns.ModID, false); - - try - { - await SQL.AddNexusModFiles(ns.Game, ns.ModID, mod.updated_time, files); - } - catch (Exception _) - { - // Could be a PK constraint failure - } + return ArchiveStatus.InValid; } if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID)) diff --git a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs index e18379fa..32754b07 100644 --- a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs @@ -214,20 +214,24 @@ namespace Wabbajack.BuildServer.Controllers { var user = User.FindFirstValue(ClaimTypes.Name); Utils.Log($"Delete Uploaded File {user} {name}"); - var files = await SQL.AllUploadedFilesForUser(name); + var files = await SQL.AllUploadedFilesForUser(user); var to_delete = files.First(f => f.MungedName == name); if (AlphaFile.Exists(Path.Combine("public", "files", to_delete.MungedName))) AlphaFile.Delete(Path.Combine("public", "files", to_delete.MungedName)); - using (var client = new FtpClient("storage.bunnycdn.com")) - { - client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password); - await client.ConnectAsync(); - if (await client.FileExistsAsync(to_delete.MungedName)) - await client.DeleteFileAsync(to_delete.MungedName); + if (_settings.BunnyCDN_User != "TEST" || _settings.BunnyCDN_Password != "TEST") + { + using (var client = new FtpClient("storage.bunnycdn.com")) + { + client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password); + await client.ConnectAsync(); + if (await client.FileExistsAsync(to_delete.MungedName)) + await client.DeleteFileAsync(to_delete.MungedName); + + } } await SQL.DeleteUploadedFile(to_delete.Id); @@ -263,12 +267,7 @@ namespace Wabbajack.BuildServer.Controllers files.Add(uf); await SQL.AddUploadedFile(uf); } - - return Ok(files.Count); } - - - } } diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index a018e359..9d0c694b 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -229,7 +229,6 @@ namespace Wabbajack.Lib result.Name = archive.Name; result.Hash = archive.File.Hash; - result.Meta = archive.Meta; result.Size = archive.File.Size; await result.State!.GetDownloader().Prepare(); @@ -238,6 +237,9 @@ namespace Wabbajack.Lib Error( $"Unable to resolve link for {archive.Name}. If this is hosted on the Nexus the file may have been removed."); + result.Meta = string.Join("\n", result.State!.GetMetaIni()); + + return result; } diff --git a/Wabbajack.Lib/FileUploader/AuthorAPI.cs b/Wabbajack.Lib/FileUploader/AuthorAPI.cs index 638dc927..25fd0796 100644 --- a/Wabbajack.Lib/FileUploader/AuthorAPI.cs +++ b/Wabbajack.Lib/FileUploader/AuthorAPI.cs @@ -184,18 +184,18 @@ namespace Wabbajack.Lib.FileUploader public static async Task GetServerLog() { - return await (await GetAuthorizedClient()).GetStringAsync($"https://{Consts.WabbajackCacheHostname}/heartbeat/logs"); + return await (await GetAuthorizedClient()).GetStringAsync($"{Consts.WabbajackBuildServerUri}heartbeat/logs"); } public static async Task> GetMyFiles() { - return (await (await GetAuthorizedClient()).GetStringAsync($"https://{Consts.WabbajackCacheHostname}/uploaded_files/list")).FromJsonString(); + return (await (await GetAuthorizedClient()).GetStringAsync($"{Consts.WabbajackBuildServerUri}uploaded_files/list")).FromJsonString(); } public static async Task DeleteFile(string name) { var result = await (await GetAuthorizedClient()) - .DeleteStringAsync($"https://{Consts.WabbajackCacheHostname}/uploaded_files/{name}"); + .DeleteStringAsync($"{Consts.WabbajackBuildServerUri}uploaded_files/{name}"); return result; } } From a1c5cfad8220883afd627679791c2cea30e8caa6 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 30 Apr 2020 16:35:16 -0600 Subject: [PATCH 2/2] Can get the names/hashes for stock game files --- .../IndexedFilesTests.cs | 46 +++++++++++++++++++ .../sql/wabbajack_db.sql | 27 +++++++++++ .../Controllers/IndexedFiles.cs | 8 ++++ Wabbajack.BuildServer/JobManager.cs | 2 +- .../Models/Sql/SqlService.cs | 39 ++++++++++++++++ Wabbajack.Lib/ClientAPI.cs | 10 +++- 6 files changed, 130 insertions(+), 2 deletions(-) diff --git a/Wabbajack.BuildServer.Test/IndexedFilesTests.cs b/Wabbajack.BuildServer.Test/IndexedFilesTests.cs index c7af0997..727ab95e 100644 --- a/Wabbajack.BuildServer.Test/IndexedFilesTests.cs +++ b/Wabbajack.BuildServer.Test/IndexedFilesTests.cs @@ -86,6 +86,52 @@ namespace Wabbajack.BuildServer.Test Assert.Null(await SQL.GetJob()); } + [Fact] + public async Task CanGetGameFiles() + { + var sql = Fixture.GetService(); + await sql.AddDownloadState(Hash.FromLong(1), + new GameFileSourceDownloader.State("1.2.3.4") + { + Game = Game.SkyrimSpecialEdition, + Hash = Hash.FromLong(1), + GameFile = (RelativePath)@"Data\foo.bsa", + }); + await sql.AddDownloadState(Hash.FromLong(2), + new GameFileSourceDownloader.State("1.2.3.4") + { + Game = Game.SkyrimSpecialEdition, + Hash = Hash.FromLong(2), + GameFile = (RelativePath)@"Data\foo - Textures.bsa", + }); + + + await sql.AddDownloadState(Hash.FromLong(3), + new GameFileSourceDownloader.State("1.2.3.4") + { + Game = Game.Skyrim, + Hash = Hash.FromLong(3), + GameFile = (RelativePath)@"Data\foo - Textures.bsa", + }); + + await sql.AddDownloadState(Hash.FromLong(4), + new GameFileSourceDownloader.State("1.9.3.4") + { + Game = Game.SkyrimSpecialEdition, + Hash = Hash.FromLong(4), + GameFile = (RelativePath)@"Data\foo - Textures.bsa", + }); + + var results = await ClientAPI.GetGameFiles(Game.SkyrimSpecialEdition, Version.Parse("1.2.3.4")); + + Assert.Equal(new Dictionary + { + {(RelativePath)@"Data\foo.bsa", Hash.FromLong(1)}, + {(RelativePath)@"Data\foo - Textures.bsa", Hash.FromLong(2)}, + }, results); + + } + public IndexedFilesTests(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, fixture) { diff --git a/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql b/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql index fbfcd194..1f6315b3 100644 --- a/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql +++ b/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql @@ -460,6 +460,33 @@ CREATE NONCLUSTERED INDEX [ByHash] ON [dbo].[DownloadStates] INCLUDE([IniState]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] GO +/****** Object: View [dbo].[GameFiles] Script Date: 4/30/2020 4:23:25 PM ******/ + +CREATE VIEW [dbo].[GameFiles] + WITH SCHEMABINDING +AS + +Select + Id, + CONVERT(NVARCHAR(20), JSON_VALUE(JsonState,'$.GameVersion')) as GameVersion, + CONVERT(NVARCHAR(32),JSON_VALUE(JsonState,'$.Game')) as Game, + JSON_VALUE(JsonState,'$.GameFile') as Path, + Hash as Hash +FROM dbo.DownloadStates +WHERE PrimaryKey like 'GameFileSourceDownloader+State|%' + AND JSON_VALUE(JsonState,'$.GameFile') NOT LIKE '%.xxhash' +GO + +CREATE UNIQUE CLUSTERED INDEX [ByGameAndVersion] ON [dbo].[GameFiles] + ( + [Game] ASC, + [GameVersion] ASC, + [Id] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +GO + + + /****** Object: Index [IX_Child] Script Date: 3/28/2020 4:58:59 PM ******/ CREATE NONCLUSTERED INDEX [IX_Child] ON [dbo].[AllFilesInArchive] ( diff --git a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs index 574c9b3f..b3dca847 100644 --- a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs @@ -131,6 +131,14 @@ namespace Wabbajack.BuildServer.Controllers return Ok(result); } + [HttpGet] + [Route("/game_files/{game}/{version}")] + public async Task GetGameFiles(string game, string version) + { + var result = await _sql.GameFiles(GameRegistry.GetByFuzzyName(game).Game, Version.Parse(version)); + return Ok(result.ToDictionary(k => k.Item1, k => k.Item2)); + } + public class TreeResult : IndexedFile { public List ChildFiles { get; set; } diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index c3166352..d89e6580 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -86,7 +86,7 @@ namespace Wabbajack.BuildServer while (true) { await KillOrphanedJobs(); - //await ScheduledJob(TimeSpan.FromHours(1), Job.JobPriority.High); + await ScheduledJob(TimeSpan.FromHours(1), Job.JobPriority.High); //await ScheduledJob(TimeSpan.FromMinutes(30), Job.JobPriority.High); //await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.Low); //await ScheduledJob(TimeSpan.FromHours(24), Job.JobPriority.High); diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index 948d5611..f3451128 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -138,6 +138,17 @@ namespace Wabbajack.BuildServer.Model.Models return Build(0).FirstOrDefault(); } + public async Task> GameFiles(Game game, Version version) + { + await using var conn = await Open(); + var files = await conn.QueryAsync<(RelativePath, Hash)>( + @"SELECT Path, Hash FROM dbo.GameFiles where Game = @Game AND GameVersion = @GameVersion", + new {Game = game.ToString(), GameVersion = version}); + + return files; + + } + public async Task IngestAllMetrics(IEnumerable allMetrics) { await using var conn = await Open(); @@ -287,6 +298,8 @@ namespace Wabbajack.BuildServer.Model.Models SqlMapper.AddTypeHandler(new JsonMapper()); SqlMapper.AddTypeHandler(new JsonMapper()); SqlMapper.AddTypeHandler(new JsonMapper()); + SqlMapper.AddTypeHandler(new VersionMapper()); + SqlMapper.AddTypeHandler(new GameMapper()); } public class JsonMapper : SqlMapper.TypeHandler @@ -328,6 +341,32 @@ namespace Wabbajack.BuildServer.Model.Models } } + class VersionMapper : SqlMapper.TypeHandler + { + public override void SetValue(IDbDataParameter parameter, Version value) + { + parameter.Value = value.ToString(); + } + + public override Version Parse(object value) + { + return Version.Parse((string)value); + } + } + + class GameMapper : SqlMapper.TypeHandler + { + public override void SetValue(IDbDataParameter parameter, Game value) + { + parameter.Value = value.ToString(); + } + + public override Game Parse(object value) + { + return GameRegistry.GetByFuzzyName((string)value).Game; + } + } + #endregion diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index 959fbbf7..f0a4e839 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -1,4 +1,6 @@ -using System.Threading.Tasks; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Lib.Exceptions; @@ -59,5 +61,11 @@ namespace Wabbajack.Lib return await GetClient() .GetJsonAsync($"{Consts.WabbajackBuildServerUri}nexus_cache/stats"); } + + public static async Task> GetGameFiles(Game game, Version version) + { + return await GetClient() + .GetJsonAsync>($"{Consts.WabbajackBuildServerUri}game_files/{game}/{version}"); + } } }