Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Store game file hashes on GitHub

Browse Source
This commit is contained in:
Timothy Baldridge 2020-11-02 18:55:54 -07:00
parent c668f67a75
commit 7f603e9c85
7 changed files with 156 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Wabbajack.CLI/OptionsDefinition.cs
View File

@ -30,7 +30,9 @@ namespace Wabbajack.CLI
typeof(HashVariants),
typeof(ParseMeta),
typeof(NoPatch),
typeof(NexusPermissions)
typeof(NexusPermissions),
typeof(ExportServerGameFiles),
typeof(HashGamefiles)
};
}
}

32
Wabbajack.CLI/Verbs/ExportServerGameFiles.cs Normal file
View File

@ -0,0 +1,32 @@
using System.Linq;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Common;
using Wabbajack.Lib;
namespace Wabbajack.CLI.Verbs
{
[Verb("export-server-game-files", HelpText = "Exports all the game file data from the server to the output folder")]
public class ExportServerGameFiles : AVerb
{
[Option('o', "output", Required = true, HelpText = @"Output folder in which the files will be placed")]
public string OutputFolder { get; set; } = "";
private AbsolutePath _outputFolder => (AbsolutePath)OutputFolder;
protected override async Task<ExitCode> Run()
{
var games = await ClientAPI.GetServerGamesAndVersions();
foreach (var (game, version) in games)
{
Utils.Log($"Exporting {game} {version}");
var file = _outputFolder.Combine(game.ToString(), version).WithExtension(new Extension(".json"));
file.Parent.CreateDirectory();
var files = await ClientAPI.GetGameFilesFromServer(game, version);
await files.ToJsonAsync(file, prettyPrint:true);
}
return ExitCode.Ok;
}
}
}

54
Wabbajack.CLI/Verbs/HashGameFiles.cs Normal file
View File

@ -0,0 +1,54 @@
using System.Linq;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.CLI.Verbs
{
[Verb("hash-game-files", HelpText = "Hashes a game's files for inclusion in the public github repo")]
public class HashGamefiles : AVerb
{
[Option('o', "output", Required = true, HelpText = @"Output folder in which the file will be placed")]
public string OutputFolder { get; set; } = "";
private AbsolutePath _outputFolder => (AbsolutePath)OutputFolder;
[Option('g', "game", Required = true, HelpText = @"WJ Game to index")]
public string Game { get; set; } = "";
private Game _game => GameRegistry.GetByFuzzyName(Game).Game;
protected override async Task<ExitCode> Run()
{
var version = _game.MetaData().InstalledVersion;
var file = _outputFolder.Combine(_game.ToString(), version).WithExtension(new Extension(".json"));
file.Parent.CreateDirectory();
using var queue = new WorkQueue();
var gameLocation = _game.MetaData().GameLocation();
Utils.Log($"Hashing files for {_game} {version}");
var indexed = await gameLocation
.EnumerateFiles()
.PMap(queue, async f =>
{
var hash = await f.FileHashCachedAsync();
return new GameFileSourceDownloader.State
{
Game = _game,
GameFile = f.RelativeTo(gameLocation),
Hash = hash,
GameVersion = version
};
});
Utils.Log($"Found and hashed {indexed.Length} files");
await indexed.ToJsonAsync(file, prettyPrint: true);
return ExitCode.Ok;
}
}
}

26
Wabbajack.Common/Json.cs
View File

@ -36,6 +36,15 @@ namespace Wabbajack.Common
Converters = Converters,
DateTimeZoneHandling = DateTimeZoneHandling.Utc
};
public static JsonSerializerSettings JsonSettingsPretty =>
new JsonSerializerSettings {
TypeNameHandling = TypeNameHandling.Objects,
SerializationBinder = new JsonNameSerializationBinder(),
Converters = Converters,
DateTimeZoneHandling = DateTimeZoneHandling.Utc,
Formatting = Formatting.Indented
};
public static JsonSerializerSettings GenericJsonSettings =>
new JsonSerializerSettings
@ -51,18 +60,27 @@ namespace Wabbajack.Common
File.WriteAllText(filename, JsonConvert.SerializeObject(obj, Formatting.Indented, JsonSettings));
}
public static void ToJson<T>(this T obj, Stream stream, bool useGenericSettings = false)
public static void ToJson<T>(this T obj, Stream stream, bool useGenericSettings = false, bool prettyPrint = false)
{
using var tw = new StreamWriter(stream, Encoding.UTF8, bufferSize: 1024, leaveOpen: true);
using var writer = new JsonTextWriter(tw);
var ser = JsonSerializer.Create(useGenericSettings ? GenericJsonSettings : JsonSettings);
JsonSerializerSettings settings = (useGenericSettings, prettyPrint) switch
{
(true, true) => GenericJsonSettings,
(false, true) => JsonSettingsPretty,
(false, false) => JsonSettings,
(true, false) => GenericJsonSettings
};
var ser = JsonSerializer.Create(settings);
ser.Serialize(writer, obj);
}
public static async ValueTask ToJsonAsync<T>(this T obj, AbsolutePath path, bool useGenericSettings = false)
public static async ValueTask ToJsonAsync<T>(this T obj, AbsolutePath path, bool useGenericSettings = false, bool prettyPrint = false)
{
await using var fs = await path.Create();
obj.ToJson(fs, useGenericSettings);
obj.ToJson(fs, useGenericSettings, prettyPrint: prettyPrint);
}
public static string ToJson<T>(this T obj, bool useGenericSettings = false)

29
Wabbajack.Lib/ClientAPI.cs
View File

@ -157,9 +157,7 @@ using Wabbajack.Lib.Downloaders;
return new Archive[0];
var client = await GetClient();
var metaData = game.MetaData();
var results =
await client.GetJsonAsync<Archive[]>(
$"{Consts.WabbajackBuildServerUri}game_files/{game}/{metaData.InstalledVersion}");
var results = await GetGameFilesFromGithub(game, metaData.InstalledVersion);
return (await results.PMap(queue, async file => (await file.State.Verify(file), file))).Where(f => f.Item1)
.Select(f =>
@ -169,6 +167,22 @@ using Wabbajack.Lib.Downloaders;
})
.ToArray();
}
public static async Task<Archive[]> GetGameFilesFromGithub(Game game, string version)
{
var url =
$"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}.json";
Utils.Log($"Loading game file definition from {url}");
var client = await GetClient();
return await client.GetJsonAsync<Archive[]>(url);
}
public static async Task<Archive[]> GetGameFilesFromServer(Game game, string version)
{
var client = await GetClient();
return await client.GetJsonAsync<Archive[]>(
$"{Consts.WabbajackBuildServerUri}game_files/{game}/{version}");
}
public static async Task<AbstractDownloadState?> InferDownloadState(Hash hash)
{
@ -262,5 +276,14 @@ using Wabbajack.Lib.Downloaders;
return await client.GetJsonAsync<Helpers.Cookie[]>(
$"{Consts.WabbajackBuildServerUri}site-integration/auth-info/{key}");
}
public static async Task<IEnumerable<(Game, string)>> GetServerGamesAndVersions()
{
var client = await GetClient();
var results =
await client.GetJsonAsync<(Game, string)[]>(
$"{Consts.WabbajackBuildServerUri}game_files");
return results;
}
}
}

8
Wabbajack.Server/Controllers/GameFiles.cs
View File

@ -67,6 +67,14 @@ namespace Wabbajack.BuildServer.Controllers
return Ok(files.ToJson());
}
[Authorize(Roles = "User")]
[HttpGet]
public async Task<IActionResult> GetAllGames()
{
var registeredGames = await _sql.GetAllRegisteredGames();
return Ok(registeredGames.ToArray().ToJson());
}
}

11
Wabbajack.Server/DataLayer/ArchiveDownloads.cs
View File

@ -255,5 +255,16 @@ namespace Wabbajack.Server.DataLayer
return files.ToArray();
}
public async Task<IEnumerable<(Game, string)>> GetAllRegisteredGames()
{
await using var conn = await Open();
var pks = (await conn.QueryAsync<string>(
@"SELECT PrimaryKeyString FROM dbo.ArchiveDownloads WHERE PrimaryKeyString like 'GameFileSourceDownloader+State|%'")
);
return pks.Select(p => p.Split("|"))
.Select(t => (GameRegistry.GetByFuzzyName(t[1]).Game, t[2]))
.Distinct();
}
}
}