From 7f603e9c855c2c7b9739ad682328f3ca85f8043b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 2 Nov 2020 18:55:54 -0700 Subject: [PATCH] Store game file hashes on GitHub --- Wabbajack.CLI/OptionsDefinition.cs | 4 +- Wabbajack.CLI/Verbs/ExportServerGameFiles.cs | 32 +++++++++++ Wabbajack.CLI/Verbs/HashGameFiles.cs | 54 +++++++++++++++++++ Wabbajack.Common/Json.cs | 26 +++++++-- Wabbajack.Lib/ClientAPI.cs | 29 ++++++++-- Wabbajack.Server/Controllers/GameFiles.cs | 8 +++ .../DataLayer/ArchiveDownloads.cs | 11 ++++ 7 files changed, 156 insertions(+), 8 deletions(-) create mode 100644 Wabbajack.CLI/Verbs/ExportServerGameFiles.cs create mode 100644 Wabbajack.CLI/Verbs/HashGameFiles.cs diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs index 50398100..e0193872 100644 --- a/Wabbajack.CLI/OptionsDefinition.cs +++ b/Wabbajack.CLI/OptionsDefinition.cs @@ -30,7 +30,9 @@ namespace Wabbajack.CLI typeof(HashVariants), typeof(ParseMeta), typeof(NoPatch), - typeof(NexusPermissions) + typeof(NexusPermissions), + typeof(ExportServerGameFiles), + typeof(HashGamefiles) }; } } diff --git a/Wabbajack.CLI/Verbs/ExportServerGameFiles.cs b/Wabbajack.CLI/Verbs/ExportServerGameFiles.cs new file mode 100644 index 00000000..9c1887b1 --- /dev/null +++ b/Wabbajack.CLI/Verbs/ExportServerGameFiles.cs @@ -0,0 +1,32 @@ +using System.Linq; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("export-server-game-files", HelpText = "Exports all the game file data from the server to the output folder")] + public class ExportServerGameFiles : AVerb + { + [Option('o', "output", Required = true, HelpText = @"Output folder in which the files will be placed")] + public string OutputFolder { get; set; } = ""; + + private AbsolutePath _outputFolder => (AbsolutePath)OutputFolder; + + protected override async Task Run() + { + var games = await ClientAPI.GetServerGamesAndVersions(); + foreach (var (game, version) in games) + { + Utils.Log($"Exporting {game} {version}"); + var file = _outputFolder.Combine(game.ToString(), version).WithExtension(new Extension(".json")); + file.Parent.CreateDirectory(); + var files = await ClientAPI.GetGameFilesFromServer(game, version); + await files.ToJsonAsync(file, prettyPrint:true); + } + + return ExitCode.Ok; + } + } +} diff --git a/Wabbajack.CLI/Verbs/HashGameFiles.cs b/Wabbajack.CLI/Verbs/HashGameFiles.cs new file mode 100644 index 00000000..e632a488 --- /dev/null +++ b/Wabbajack.CLI/Verbs/HashGameFiles.cs @@ -0,0 +1,54 @@ +using System.Linq; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("hash-game-files", HelpText = "Hashes a game's files for inclusion in the public github repo")] + public class HashGamefiles : AVerb + { + [Option('o', "output", Required = true, HelpText = @"Output folder in which the file will be placed")] + public string OutputFolder { get; set; } = ""; + + private AbsolutePath _outputFolder => (AbsolutePath)OutputFolder; + + [Option('g', "game", Required = true, HelpText = @"WJ Game to index")] + public string Game { get; set; } = ""; + + private Game _game => GameRegistry.GetByFuzzyName(Game).Game; + + protected override async Task Run() + { + var version = _game.MetaData().InstalledVersion; + var file = _outputFolder.Combine(_game.ToString(), version).WithExtension(new Extension(".json")); + file.Parent.CreateDirectory(); + + using var queue = new WorkQueue(); + var gameLocation = _game.MetaData().GameLocation(); + + Utils.Log($"Hashing files for {_game} {version}"); + + var indexed = await gameLocation + .EnumerateFiles() + .PMap(queue, async f => + { + var hash = await f.FileHashCachedAsync(); + return new GameFileSourceDownloader.State + { + Game = _game, + GameFile = f.RelativeTo(gameLocation), + Hash = hash, + GameVersion = version + }; + + }); + + Utils.Log($"Found and hashed {indexed.Length} files"); + await indexed.ToJsonAsync(file, prettyPrint: true); + return ExitCode.Ok; + } + } +} diff --git a/Wabbajack.Common/Json.cs b/Wabbajack.Common/Json.cs index 006421e9..0de935bb 100644 --- a/Wabbajack.Common/Json.cs +++ b/Wabbajack.Common/Json.cs @@ -36,6 +36,15 @@ namespace Wabbajack.Common Converters = Converters, DateTimeZoneHandling = DateTimeZoneHandling.Utc }; + + public static JsonSerializerSettings JsonSettingsPretty => + new JsonSerializerSettings { + TypeNameHandling = TypeNameHandling.Objects, + SerializationBinder = new JsonNameSerializationBinder(), + Converters = Converters, + DateTimeZoneHandling = DateTimeZoneHandling.Utc, + Formatting = Formatting.Indented + }; public static JsonSerializerSettings GenericJsonSettings => new JsonSerializerSettings @@ -51,18 +60,27 @@ namespace Wabbajack.Common File.WriteAllText(filename, JsonConvert.SerializeObject(obj, Formatting.Indented, JsonSettings)); } - public static void ToJson(this T obj, Stream stream, bool useGenericSettings = false) + public static void ToJson(this T obj, Stream stream, bool useGenericSettings = false, bool prettyPrint = false) { using var tw = new StreamWriter(stream, Encoding.UTF8, bufferSize: 1024, leaveOpen: true); using var writer = new JsonTextWriter(tw); - var ser = JsonSerializer.Create(useGenericSettings ? GenericJsonSettings : JsonSettings); + + JsonSerializerSettings settings = (useGenericSettings, prettyPrint) switch + { + (true, true) => GenericJsonSettings, + (false, true) => JsonSettingsPretty, + (false, false) => JsonSettings, + (true, false) => GenericJsonSettings + }; + + var ser = JsonSerializer.Create(settings); ser.Serialize(writer, obj); } - public static async ValueTask ToJsonAsync(this T obj, AbsolutePath path, bool useGenericSettings = false) + public static async ValueTask ToJsonAsync(this T obj, AbsolutePath path, bool useGenericSettings = false, bool prettyPrint = false) { await using var fs = await path.Create(); - obj.ToJson(fs, useGenericSettings); + obj.ToJson(fs, useGenericSettings, prettyPrint: prettyPrint); } public static string ToJson(this T obj, bool useGenericSettings = false) diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index 8ec1aa54..fea3de11 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -157,9 +157,7 @@ using Wabbajack.Lib.Downloaders; return new Archive[0]; var client = await GetClient(); var metaData = game.MetaData(); - var results = - await client.GetJsonAsync( - $"{Consts.WabbajackBuildServerUri}game_files/{game}/{metaData.InstalledVersion}"); + var results = await GetGameFilesFromGithub(game, metaData.InstalledVersion); return (await results.PMap(queue, async file => (await file.State.Verify(file), file))).Where(f => f.Item1) .Select(f => @@ -169,6 +167,22 @@ using Wabbajack.Lib.Downloaders; }) .ToArray(); } + + public static async Task GetGameFilesFromGithub(Game game, string version) + { + var url = + $"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}.json"; + Utils.Log($"Loading game file definition from {url}"); + var client = await GetClient(); + return await client.GetJsonAsync(url); + } + + public static async Task GetGameFilesFromServer(Game game, string version) + { + var client = await GetClient(); + return await client.GetJsonAsync( + $"{Consts.WabbajackBuildServerUri}game_files/{game}/{version}"); + } public static async Task InferDownloadState(Hash hash) { @@ -262,5 +276,14 @@ using Wabbajack.Lib.Downloaders; return await client.GetJsonAsync( $"{Consts.WabbajackBuildServerUri}site-integration/auth-info/{key}"); } + + public static async Task> GetServerGamesAndVersions() + { + var client = await GetClient(); + var results = + await client.GetJsonAsync<(Game, string)[]>( + $"{Consts.WabbajackBuildServerUri}game_files"); + return results; + } } } diff --git a/Wabbajack.Server/Controllers/GameFiles.cs b/Wabbajack.Server/Controllers/GameFiles.cs index a4c695be..27bb7174 100644 --- a/Wabbajack.Server/Controllers/GameFiles.cs +++ b/Wabbajack.Server/Controllers/GameFiles.cs @@ -67,6 +67,14 @@ namespace Wabbajack.BuildServer.Controllers return Ok(files.ToJson()); } + [Authorize(Roles = "User")] + [HttpGet] + public async Task GetAllGames() + { + var registeredGames = await _sql.GetAllRegisteredGames(); + return Ok(registeredGames.ToArray().ToJson()); + } + } diff --git a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs index f988d750..ee328419 100644 --- a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs +++ b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs @@ -255,5 +255,16 @@ namespace Wabbajack.Server.DataLayer return files.ToArray(); } + + public async Task> GetAllRegisteredGames() + { + await using var conn = await Open(); + var pks = (await conn.QueryAsync( + @"SELECT PrimaryKeyString FROM dbo.ArchiveDownloads WHERE PrimaryKeyString like 'GameFileSourceDownloader+State|%'") + ); + return pks.Select(p => p.Split("|")) + .Select(t => (GameRegistry.GetByFuzzyName(t[1]).Game, t[2])) + .Distinct(); + } } }