Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #929 from wabbajack-tools/game-hosted-files

Browse Source 
Game hosted files
This commit is contained in:
Timothy Baldridge 2020-06-21 17:01:52 -07:00 committed by GitHub
commit b62e89f9fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 406 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -1,5 +1,11 @@
### Changelog
#### Version - 2.1.0.0 - ???
* Game files are available as downloads automatically during compilation/installation
* Game files are patched/copied/used in BSA creation automatically
* CleanedESM support removed from the compiler stack (still usable during installation for backwards compatibility)
* VR games automatically pull from base games if they are required and are installed during compilation
#### Version - 2.0.9.4 - 6/16/2020
* Improve interactions between server and client code

3
Wabbajack.CLI/OptionsDefinition.cs
View File

@ -20,7 +20,8 @@ namespace Wabbajack.CLI
typeof(Changelog),
typeof(FindSimilar),
typeof(BSADump),
typeof(MigrateGameFolderFiles)
typeof(MigrateGameFolderFiles),
typeof(HashFile)
};
}
}

1
Wabbajack.CLI/Program.cs
View File

@ -28,6 +28,7 @@ namespace Wabbajack.CLI
(FindSimilar opts) => opts.Execute(),
(BSADump opts) => opts.Execute(),
(MigrateGameFolderFiles opts) => opts.Execute(),
(HashFile opts) => opts.Execute(),
errs => 1);
}
}

22
Wabbajack.CLI/Verbs/HashFile.cs Normal file
View File

@ -0,0 +1,22 @@
using System;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Common;
namespace Wabbajack.CLI.Verbs
{
[Verb("hash-file", HelpText = "Hash a file and print the result")]
public class HashFile : AVerb
{
[Option('i', "input", Required = true, HelpText = "Input file name")]
public string Input { get; set; } = "";
protected override async Task<ExitCode> Run()
{
var abs = (AbsolutePath)Input;
Console.WriteLine($"{abs} hash: {await abs.FileHashAsync()}");
return ExitCode.Ok;
}
}
}

24
Wabbajack.Common/GameMetaData.cs
View File

@ -73,8 +73,15 @@ namespace Wabbajack.Common
// Games that this game are commonly confused with, for example Skyrim SE vs Skyrim LE
public Game[] CommonlyConfusedWith { get; set; } = Array.Empty<Game>();
/// <summary>
/// Other games this game can pull source files from (if the game is installed on the user's machine)
/// </summary>
public Game[] CanSourceFrom { get; set; } = Array.Empty<Game>();
public string HumanFriendlyGameName => Game.GetDescription();
private AbsolutePath _cachedPath = default;
public string InstalledVersion
{
get
@ -97,9 +104,16 @@ namespace Wabbajack.Common
public bool TryGetGameLocation(out AbsolutePath path)
{
if (_cachedPath != default)
{
path = _cachedPath;
return true;
}
var ret = TryGetGameLocation();
if (ret != null)
{
_cachedPath = ret.Value;
path = ret.Value;
return true;
}
@ -329,7 +343,7 @@ namespace Wabbajack.Common
"SkyrimSE.exe"
},
MainExecutable = "SkyrimSE.exe",
CommonlyConfusedWith = new []{Game.Skyrim, Game.SkyrimVR}
CommonlyConfusedWith = new []{Game.Skyrim, Game.SkyrimVR},
}
},
{
@ -347,7 +361,7 @@ namespace Wabbajack.Common
"Fallout4.exe"
},
MainExecutable = "Fallout4.exe",
CommonlyConfusedWith = new [] {Game.Fallout4VR}
CommonlyConfusedWith = new [] {Game.Fallout4VR},
}
},
{
@ -365,7 +379,8 @@ namespace Wabbajack.Common
"SkyrimVR.exe"
},
MainExecutable = "SkyrimVR.exe",
CommonlyConfusedWith = new []{Game.Skyrim, Game.SkyrimSpecialEdition}
CommonlyConfusedWith = new []{Game.Skyrim, Game.SkyrimSpecialEdition},
CanSourceFrom = new [] {Game.SkyrimSpecialEdition}
}
},
{
@ -398,7 +413,8 @@ namespace Wabbajack.Common
"Fallout4VR.exe"
},
MainExecutable = "Fallout4VR.exe",
CommonlyConfusedWith = new [] {Game.Fallout4}
CommonlyConfusedWith = new [] {Game.Fallout4},
CanSourceFrom = new [] {Game.Fallout4}
}
},
{

4
Wabbajack.Common/Http/Client.cs
View File

@ -96,11 +96,11 @@ namespace Wabbajack.Common.Http
{
if (ex is HttpException http)
{
if (http.Code != 503) throw;
if (http.Code != 503 && http.Code != 521) throw;
retries++;
var ms = Utils.NextRandom(100, 1000);
Utils.Log($"Got a 503 from {msg.RequestUri} retrying in {ms}ms");
Utils.Log($"Got a {http.Code} from {msg.RequestUri} retrying in {ms}ms");
await Task.Delay(ms);
msg = CloneMessage(msg);

7
Wabbajack.Common/Paths.cs
View File

@ -198,6 +198,7 @@ namespace Wabbajack.Common
return new RelativePath(relPath);
}
public async Task<string> ReadAllTextAsync()
{
await using var fs = File.OpenRead(_path);
@ -460,6 +461,12 @@ namespace Wabbajack.Common
{
return (RelativePath)Guid.NewGuid().ToString();
}
public RelativePath Munge()
{
return (RelativePath)_path.Replace('\\', '_').Replace('/', '_').Replace(':', '_');
}
private void Validate()
{

14
Wabbajack.Common/Utils.cs
View File

@ -942,8 +942,16 @@ namespace Wabbajack.Common
/// <param name="data"></param>
public static async ValueTask ToEcryptedJson<T>(this T data, string key)
{
var bytes = Encoding.UTF8.GetBytes(data.ToJson());
await bytes.ToEcryptedData(key);
try
{
var bytes = Encoding.UTF8.GetBytes(data.ToJson());
await bytes.ToEcryptedData(key);
}
catch (Exception ex)
{
Log($"Error encrypting data {key} {ex}");
throw;
}
}
public static async Task<T> FromEncryptedJson<T>(string key)
@ -1125,7 +1133,7 @@ namespace Wabbajack.Common
if (lst.Count > 0 && lst.Count != size)
yield return lst;
}
private static Random _random = new Random();
public static int NextRandom(int min, int max)
{

10
Wabbajack.Lib/ACompiler.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.IO.Compression;
using System.Linq;
@ -46,6 +47,8 @@ namespace Wabbajack.Lib
public ModList ModList = new ModList();
public List<IndexedArchive> IndexedArchives = new List<IndexedArchive>();
public Dictionary<AbsolutePath, IndexedArchive> ArchivesByFullPath { get; set; } = new Dictionary<AbsolutePath, IndexedArchive>();
public Dictionary<Hash, IEnumerable<VirtualFile>> IndexedFiles = new Dictionary<Hash, IEnumerable<VirtualFile>>();
public ACompiler(int steps)
@ -221,9 +224,10 @@ namespace Wabbajack.Lib
throw new ArgumentException($"No match found for Archive sha: {hash.ToBase64()} this shouldn't happen");
}
public async Task<Archive> ResolveArchive(IndexedArchive archive)
public async Task<Archive> ResolveArchive([NotNull] IndexedArchive archive)
{
Utils.Status($"Checking link for {archive.Name}", alsoLog: true);
if (!string.IsNullOrWhiteSpace(archive.Name))
Utils.Status($"Checking link for {archive.Name}", alsoLog: true);
if (archive.IniData == null)
Error(
@ -234,7 +238,7 @@ namespace Wabbajack.Lib
if (result.State == null)
Error($"{archive.Name} could not be handled by any of the downloaders");
result.Name = archive.Name;
result.Name = archive.Name ?? "";
result.Hash = archive.File.Hash;
result.Size = archive.File.Size;

64
Wabbajack.Lib/ClientAPI.cs
View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
@ -8,8 +9,9 @@ using Wabbajack.Common;
using Wabbajack.Common.Exceptions;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.Lib
namespace Wabbajack.Lib
{
[JsonName("ModUpgradeRequest")]
public class ModUpgradeRequest
@ -83,25 +85,6 @@ namespace Wabbajack.Lib
throw ex;
}
/// <summary>
/// Given an archive hash, search the Wabbajack server for a matching .ini file
/// </summary>
/// <param name="hash"></param>
/// <returns></returns>
public static async Task<string?> GetModIni(Hash hash)
{
var client = new Common.Http.Client();
try
{
return await client.GetStringAsync(
$"{Consts.WabbajackBuildServerUri}indexed_files/{hash.ToHex()}/meta.ini");
}
catch (HttpException)
{
return null;
}
}
public class NexusCacheStats
{
public long CachedCount { get; set; }
@ -115,20 +98,39 @@ namespace Wabbajack.Lib
.GetJsonAsync<NexusCacheStats>($"{Consts.WabbajackBuildServerUri}nexus_cache/stats");
}
public static async Task<Dictionary<RelativePath, Hash>> GetGameFiles(Game game, Version version)
{
// TODO: Disabled for now
return new Dictionary<RelativePath, Hash>();
/*
return await GetClient()
.GetJsonAsync<Dictionary<RelativePath, Hash>>($"{Consts.WabbajackBuildServerUri}game_files/{game}/{version}");
*/
}
public static async Task SendModListDefinition(ModList modList)
{
var client = await GetClient();
await client.PostAsync($"{Consts.WabbajackBuildServerUri}list_definitions/ingest", new StringContent(modList.ToJson(), Encoding.UTF8, "application/json"));
}
public static async Task<Archive[]> GetExistingGameFiles(WorkQueue queue, Game game)
{
var client = await GetClient();
var metaData = game.MetaData();
var results =
await client.GetJsonAsync<Archive[]>(
$"{Consts.WabbajackBuildServerUri}game_files/{game}/{metaData.InstalledVersion}");
return (await results.PMap(queue, async file => (await file.State.Verify(file), file))).Where(f => f.Item1)
.Select(f =>
{
f.file.Name = ((GameFileSourceDownloader.State)f.file.State).GameFile.Munge().ToString();
return f.file;
})
.ToArray();
}
public static async Task<AbstractDownloadState?> InferDownloadState(Hash hash)
{
var client = await GetClient();
var results = await client.GetJsonAsync<Archive[]>($"{Consts.WabbajackBuildServerUri}mod_files/by_hash/{hash.ToHex()}");
foreach (var result in results)
{
if (await result.State.Verify(result)) return result.State;
}
return null;
}
}
}

17
Wabbajack.Lib/CompilationSteps/DirectMatch.cs
View File

@ -2,6 +2,8 @@
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.Lib.CompilationSteps
{
@ -11,13 +13,26 @@ namespace Wabbajack.Lib.CompilationSteps
{
}
public static int GetFilePriority(MO2Compiler compiler, VirtualFile file)
{
var archive = file.TopParent;
var adata = compiler.ArchivesByFullPath[archive.AbsoluteName];
if (adata.State is GameFileSourceDownloader.State gs)
{
return gs.Game == compiler.CompilingGame.Game ? 1 : 3;
}
return 1;
}
public override async ValueTask<Directive?> Run(RawSourceFile source)
{
var mo2Compiler = (MO2Compiler)_compiler;
if (!_compiler.IndexedFiles.TryGetValue(source.Hash, out var found)) return null;
var result = source.EvolveTo<FromArchive>();
var match = found.Where(f => f.Name.FileName == source.Path.FileName)
.OrderBy(f => f.NestingFactor)
.OrderBy(f => GetFilePriority(mo2Compiler, f))
.ThenBy(f => f.NestingFactor)
.FirstOrDefault()
?? found.OrderBy(f => f.NestingFactor).FirstOrDefault();

10
Wabbajack.Lib/CompilationSteps/IncludePatches.cs
View File

@ -13,7 +13,7 @@ namespace Wabbajack.Lib.CompilationSteps
{
private readonly Dictionary<RelativePath, IGrouping<RelativePath, VirtualFile>> _indexed;
private VirtualFile? _bsa;
private Dictionary<RelativePath, VirtualFile> _indexedByName;
private Dictionary<RelativePath, IEnumerable<VirtualFile>> _indexedByName;
private MO2Compiler _mo2Compiler;
public IncludePatches(ACompiler compiler, VirtualFile? constructingFromBSA = null) : base(compiler)
@ -27,7 +27,8 @@ namespace Wabbajack.Lib.CompilationSteps
_indexedByName = _indexed.Values
.SelectMany(s => s)
.Where(f => f.IsNative)
.ToDictionary(f => f.FullPath.FileName);
.GroupBy(f => f.FullPath.FileName)
.ToDictionary(f => f.Key, f => (IEnumerable<VirtualFile>)f);
}
public override async ValueTask<Directive?> Run(RawSourceFile source)
@ -78,8 +79,9 @@ namespace Wabbajack.Lib.CompilationSteps
if (_indexedByName.TryGetValue(relName, out var arch))
{
// Just match some file in the archive based on the smallest delta difference
found = arch.ThisAndAllChildren
.OrderBy(o => Math.Abs(o.Size - source.File.Size))
found = arch.SelectMany(a => a.ThisAndAllChildren)
.OrderBy(o => DirectMatch.GetFilePriority(_mo2Compiler, o))
.ThenBy(o => Math.Abs(o.Size - source.File.Size))
.First();
}
}

1
Wabbajack.Lib/Data.cs
View File

@ -290,6 +290,7 @@ namespace Wabbajack.Lib
public string Meta = string.Empty;
public string Name = string.Empty;
public VirtualFile File { get; }
public AbstractDownloadState? State { get; set; }
public IndexedArchive(VirtualFile file)
{

35
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -53,41 +53,6 @@ namespace Wabbajack.Lib.FileUploader
return await RunJob("UpdateModLists");
}
public static async Task<bool> UploadPackagedInis(IEnumerable<Archive> archives)
{
archives = archives.ToArray(); // defensive copy
Utils.Log($"Packaging {archives.Count()} archive states");
try
{
await using var ms = new MemoryStream();
using (var z = new ZipArchive(ms, ZipArchiveMode.Create, true))
{
foreach (var e in archives)
{
if (e.State == null) continue;
var entry = z.CreateEntry(Path.GetFileName(e.Name));
await using var os = entry.Open();
e.ToJson(os);
}
}
var client = new Common.Http.Client();
var response = await client.PostAsync($"{Consts.WabbajackBuildServerUri}indexed_files/notify", new ByteArrayContent(ms.ToArray()));
if (response.IsSuccessStatusCode) return true;
Utils.Log("Error sending Inis");
Utils.Log(await response.Content.ReadAsStringAsync());
return false;
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
return false;
}
}
public static async Task<string> GetServerLog()
{
return await (await GetAuthorizedClient()).GetStringAsync($"{Consts.WabbajackBuildServerUri}heartbeat/logs");

85
Wabbajack.Lib/MO2Compiler.cs
View File

@ -4,6 +4,7 @@ using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Common;
@ -31,6 +32,11 @@ namespace Wabbajack.Lib
public override AbsolutePath GamePath { get; }
public GameMetaData CompilingGame { get; }
/// <summary>
/// All games available for sourcing during compilation (including the Compiling Game)
/// </summary>
public List<Game> AvailableGames { get; }
public override AbsolutePath ModListOutputFolder => ((RelativePath)"output_folder").RelativeToEntryPoint();
@ -61,6 +67,8 @@ namespace Wabbajack.Lib
CompilingGame = GameRegistry.Games.First(g => g.Value.MO2Name == mo2game).Value;
GamePath = new AbsolutePath((string)MO2Ini.General.gamePath.Replace("\\\\", "\\"));
ModListOutputFile = outputFile;
AvailableGames = CompilingGame.CanSourceFrom.Cons(CompilingGame.Game).Where(g => g.MetaData().IsInstalled).ToList();
}
public AbsolutePath MO2DownloadsFolder
@ -104,8 +112,9 @@ namespace Wabbajack.Lib
{
roots = new List<AbsolutePath>
{
MO2Folder, GamePath, MO2DownloadsFolder, CompilingGame.GameLocation()
MO2Folder, GamePath, MO2DownloadsFolder
};
roots.AddRange(AvailableGames.Select(g => g.MetaData().GameLocation()));
}
else
{
@ -191,40 +200,30 @@ namespace Wabbajack.Lib
})).ToList();
var stockGameFolder = CompilingGame.GameLocation();
var installedVersion = CompilingGame.InstalledVersion;
if (installedVersion != null)
if (UseGamePaths)
{
foreach (var (relativePath, hash) in await ClientAPI.GetGameFiles(CompilingGame.Game, Version.Parse(installedVersion)))
foreach (var ag in AvailableGames)
{
if (!VFS.Index.ByRootPath.TryGetValue(relativePath.RelativeTo(stockGameFolder), out var virtualFile))
continue;
if (virtualFile.Hash != hash)
{
Utils.Log(
$"File {relativePath} int the game folder appears to be modified, it will not be used during compilation");
continue;
}
var files = await ClientAPI.GetExistingGameFiles(Queue, ag);
Utils.Log($"Including {files.Length} stock game files from {ag} as download sources");
var state = new GameFileSourceDownloader.State
IndexedArchives.AddRange(files.Select(f =>
{
Game = CompilingGame.Game,
GameVersion = CompilingGame.InstalledVersion,
GameFile = relativePath
};
Utils.Log($"Adding Game file: {relativePath}");
IndexedArchives.Add(new IndexedArchive(virtualFile)
{
Name = (string)relativePath.FileName,
IniData = state.GetMetaIniString().LoadIniString(),
Meta = state.GetMetaIniString()
});
var meta = f.State.GetMetaIniString();
var ini = meta.LoadIniString();
var state = (GameFileSourceDownloader.State)f.State;
return new IndexedArchive(
VFS.Index.ByRootPath[ag.MetaData().GameLocation().Combine(state.GameFile)])
{
IniData = ini, Meta = meta,
};
}));
}
}
await CleanInvalidArchives();
IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();
await CleanInvalidArchivesAndFillState();
UpdateTracker.NextStep("Finding Install Files");
ModListOutputFolder.CreateDirectory();
@ -296,6 +295,8 @@ namespace Wabbajack.Lib
.Where(f => f.Item1 != default)
.Select(f => new KeyValuePair<AbsolutePath, dynamic>(f.Item1, f.Item2)));
ArchivesByFullPath = IndexedArchives.ToDictionary(a => a.File.AbsoluteName);
if (cancel.IsCancellationRequested) return false;
var stack = MakeStack();
UpdateTracker.NextStep("Running Compilation Stack");
@ -320,10 +321,6 @@ namespace Wabbajack.Lib
UpdateTracker.NextStep("Gathering Archives");
await GatherArchives();
// Don't await this because we don't care if it fails.
Utils.Log("Finding States to package");
await AuthorAPI.UploadPackagedInis(SelectedArchives.ToArray());
UpdateTracker.NextStep("Including Archive Metadata");
await IncludeArchiveMetadata();
UpdateTracker.NextStep("Building Patches");
@ -366,15 +363,16 @@ namespace Wabbajack.Lib
return true;
}
public bool UseGamePaths { get; set; } = true;
private async Task CleanInvalidArchives()
private async Task CleanInvalidArchivesAndFillState()
{
var remove = (await IndexedArchives.PMap(Queue, async a =>
{
try
{
await ResolveArchive(a);
a.State = (await ResolveArchive(a)).State;
return null;
}
catch
@ -414,12 +412,12 @@ namespace Wabbajack.Lib
await to_find.PMap(Queue, async f =>
{
var vf = VFS.Index.ByRootPath[f];
var client = new Common.Http.Client();
using var response =
await client.GetAsync(
$"http://build.wabbajack.org/indexed_files/{vf.Hash.ToHex()}/meta.ini", errorsAsExceptions: false);
if (!response.IsSuccessStatusCode)
var meta = await ClientAPI.InferDownloadState(vf.Hash);
if (meta == null)
{
await vf.AbsoluteName.WithExtension(Consts.MetaFileExtension).WriteAllLinesAsync(
"[General]",
@ -427,9 +425,8 @@ namespace Wabbajack.Lib
return;
}
var iniData = await response.Content.ReadAsStringAsync();
Utils.Log($"Inferred .meta for {vf.FullPath.FileName}, writing to disk");
await vf.AbsoluteName.WithExtension(Consts.MetaFileExtension).WriteAllTextAsync(iniData);
await vf.AbsoluteName.WithExtension(Consts.MetaFileExtension).WriteAllTextAsync(meta.GetMetaIniString());
});
}
@ -438,13 +435,15 @@ namespace Wabbajack.Lib
Utils.Log($"Including {SelectedArchives.Count} .meta files for downloads");
await SelectedArchives.PMap(Queue, async a =>
{
if (a.State is GameFileSourceDownloader.State) return;
var source = MO2DownloadsFolder.Combine(a.Name + Consts.MetaFileExtension);
var ini = a.State.GetMetaIniString();
var (id, fullPath) = await IncludeString(ini);
InstallDirectives.Add(new ArchiveMeta
{
SourceDataID = id,
Size = source.Size,
Size = fullPath.Size,
Hash = await fullPath.FileHashAsync(),
To = source.FileName
});
@ -602,7 +601,7 @@ namespace Wabbajack.Lib
new IgnoreWabbajackInstallCruft(this),
new PatchStockESMs(this),
//new PatchStockESMs(this),
new IncludeAllConfigs(this),
new zEditIntegration.IncludeZEditPatches(this),

1
Wabbajack.Server.Test/ArchiveDownloadsTests.cs
View File

@ -37,7 +37,6 @@ namespace Wabbajack.BuildServer.Test
var allStates = await service.GetAllArchiveDownloads();
Assert.Contains(state.PrimaryKeyString, allStates.Select(s => s.PrimaryKeyString));
}
private async Task ClearDownloaderQueue()

44
Wabbajack.Server.Test/ModFileTests.cs Normal file
View File

@ -0,0 +1,44 @@
using System;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DataLayer;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Server.Test
{
public class ModFileTests : ABuildServerSystemTest
{
public ModFileTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanGetDownloadStates()
{
var sql = Fixture.GetService<SqlService>();
var archive =
new Archive(new WabbajackCDNDownloader.State(new Uri(
"https://wabbajack.b-cdn.net/WABBAJACK_TEST_FILE.zip_a1a3e961-5c0b-4ccf-84b4-7aa437d9640d")))
{
Size = 20, Hash = Hash.FromBase64("eSIyd+KOG3s=")
};
await sql.EnqueueDownload(archive);
var dld = await sql.GetNextPendingDownload();
await dld.Finish(sql);
var state = await ClientAPI.InferDownloadState(archive.Hash);
Assert.Equal(archive.State.GetMetaIniString(), state!.GetMetaIniString());
}
}
}

73
Wabbajack.Server/Controllers/GameFiles.cs Normal file
View File

@ -0,0 +1,73 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
[Route("/game_files")]
public class EnqueueGameFiles : ControllerBase
{
private readonly ILogger<EnqueueGameFiles> _logger;
private readonly SqlService _sql;
private readonly QuickSync _quickSync;
public EnqueueGameFiles(ILogger<EnqueueGameFiles> logger, SqlService sql, QuickSync quickSync)
{
_logger = logger;
_sql = sql;
_quickSync = quickSync;
}
[Authorize(Roles = "Author")]
[HttpGet("enqueue")]
public async Task<IActionResult> Enqueue()
{
var games = GameRegistry.Games.Where(g => g.Value.IsInstalled).Select(g => g.Value).ToList();
_logger.Log(LogLevel.Information, $"Found {games.Count} installed games");
var files = games.SelectMany(game =>
game.GameLocation().EnumerateFiles(true).Select(file => new {File = file, Game = game})).ToList();
_logger.Log(LogLevel.Information, $"Found {files.Count} game files");
using var queue = new WorkQueue();
var hashed = await files.PMap(queue, async pair =>
{
var hash = await pair.File.FileHashCachedAsync();
return await _sql.GetOrEnqueueArchive(new Archive(new GameFileSourceDownloader.State
{
Game = pair.Game.Game,
GameFile = pair.File.RelativeTo(pair.Game.GameLocation()),
GameVersion = pair.Game.InstalledVersion,
Hash = hash
}) {Name = pair.File.FileName.ToString(), Size = pair.File.Size, Hash = hash});
});
await _quickSync.Notify<ArchiveDownloader>();
return Ok(hashed);
}
[Authorize(Roles = "User")]
[HttpGet("{game}/{version}")]
public async Task<IActionResult> GetFiles(string game, string version)
{
if (!GameRegistry.TryGetByFuzzyName(game, out var meta))
return NotFound($"Game {game} not found");
var files = await _sql.GetGameFiles(meta.Game, version);
return Ok(files.ToJson());
}
}
}

31
Wabbajack.Server/Controllers/ModFiles.cs Normal file
View File

@ -0,0 +1,31 @@
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.BuildServer.Controllers
{
[Authorize(Roles = "User")]
[ApiController]
[Route("/mod_files")]
public class ModFiles : ControllerBase
{
private SqlService _sql;
private ILogger<ModFiles> _logger;
public ModFiles(ILogger<ModFiles> logger, SqlService sql)
{
_logger = logger;
_sql = sql;
}
[HttpGet("by_hash/{hashAsHex}")]
public async Task<IActionResult> GetByHash(string hashAsHex)
{
var files = await _sql.ResolveDownloadStatesByHash(Hash.FromHex(hashAsHex));
return Ok(files.ToJson());
}
}
}

3
Wabbajack.Server/DTOs/DiscordMessage.cs
View File

@ -23,6 +23,9 @@ namespace Wabbajack.Server.DTOs
[JsonName("DiscordEmbed")]
public class DiscordEmbed
{
[JsonProperty("title")]
public string Title { get; set; }
[JsonProperty("color")]
public int Color { get; set; }

32
Wabbajack.Server/DataLayer/ArchiveDownloads.cs
View File

@ -105,7 +105,7 @@ namespace Wabbajack.Server.DataLayer
public async Task<ArchiveDownload> GetOrEnqueueArchive(Archive a)
{
await using var conn = await Open();
using var trans = await conn.BeginTransactionAsync();
await using var trans = await conn.BeginTransactionAsync();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size",
new
@ -114,7 +114,7 @@ namespace Wabbajack.Server.DataLayer
Hash = a.Hash,
Size = a.Size
}, trans);
if (result != default)
if (result.Item1 != default)
{
return new ArchiveDownload
{
@ -152,12 +152,12 @@ namespace Wabbajack.Server.DataLayer
if (ignoreNexus)
{
result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, AbstractDownloadState)>(
"SELECT Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL AND Downloader != 'NexusDownloader+State'");
"SELECT TOP(1) Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL AND Downloader != 'NexusDownloader+State'");
}
else
{
result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, AbstractDownloadState)>(
"SELECT Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL");
"SELECT TOP(1) Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL");
}
if (result == default)
@ -197,5 +197,29 @@ namespace Wabbajack.Server.DataLayer
WHERE ad.PrimaryKeyString is null");
}
public async Task<List<Archive>> GetGameFiles(Game game, string version)
{
await using var conn = await Open();
var files = (await conn.QueryAsync<(Hash, long, AbstractDownloadState)>(
$"SELECT Hash, Size, DownloadState FROM dbo.ArchiveDownloads WHERE PrimaryKeyString like 'GameFileSourceDownloader+State|{game}|{version}|%'"))
.Select(f => new Archive(f.Item3)
{
Hash = f.Item1,
Size = f.Item2
}).ToList();
return files;
}
public async Task<Archive[]> ResolveDownloadStatesByHash(Hash hash)
{
await using var conn = await Open();
var files = (await conn.QueryAsync<(long, Hash, AbstractDownloadState)>(
@"SELECT Size, Hash, DownloadState from dbo.ArchiveDownloads WHERE Hash = @Hash AND IsFailed = 0 AND DownloadFinished IS NOT NULL ORDER BY DownloadFinished DESC",
new {Hash = hash})
);
return files.Select(e =>
new Archive(e.Item3) {Size = e.Item1, Hash = e.Item2}
).ToArray();
}
}
}

9
Wabbajack.Server/Services/ArchiveDownloader.cs
View File

@ -46,6 +46,8 @@ namespace Wabbajack.Server.Services
if (nextDownload == null)
break;
_logger.LogInformation($"Checking for previously archived {nextDownload.Archive.Hash}");
if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash))
{
await nextDownload.Finish(_sql);
@ -61,7 +63,8 @@ namespace Wabbajack.Server.Services
try
{
_logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"});
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"});
await DownloadDispatcher.PrepareAll(new[] {nextDownload.Archive.State});
await using var tempPath = new TempFile();
@ -90,7 +93,9 @@ namespace Wabbajack.Server.Services
_logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Finish(_sql);
await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}"});
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}"});
}

7
Wabbajack.Server/Services/ListValidator.cs
View File

@ -102,7 +102,7 @@ namespace Wabbajack.Server.Services
{
new DiscordEmbed
{
Description =
Title =
$"Number of failures in {summary.Name} (`{summary.MachineURL}`) was {oldSummary.Summary.Failed} is now {summary.Failed}",
Url = new Uri(
$"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html")
@ -120,7 +120,10 @@ namespace Wabbajack.Server.Services
{
new DiscordEmbed
{
Description = $"{summary.Name} (`{summary.MachineURL}`) is now passing.",
Title = $"{summary.Name} (`{summary.MachineURL}`) is now passing.",
Url = new Uri(
$"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html")
}
}
});

36
Wabbajack.Test/SanityTests.cs
View File

@ -473,6 +473,42 @@ namespace Wabbajack.Test
await utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex");
}
[Fact]
public async Task CanSourceFilesFromTheGameFiles()
{
var profile = utils.AddProfile();
var mod = await utils.AddMod();
Game.SkyrimSpecialEdition.MetaData().CanSourceFrom = new[] {Game.Morrowind, Game.Skyrim};
// Morrowind file with different name
var mwFile = Game.Morrowind.MetaData().GameLocation().Combine("Data Files", "Bloodmoon.esm");
var testMW = await utils.AddModFile(mod, @"Data\MW\Bm.esm");
await mwFile.CopyToAsync(testMW);
// SkyrimSE file with same name
var skyrimFile = Game.SkyrimSpecialEdition.MetaData().GameLocation().Combine("Data", "Update.esm");
var testSky = await utils.AddModFile(mod, @"Data\SkyrimSE\Update.esm.old");
await skyrimFile.CopyToAsync(testSky);
// Same game, but patched ata
var pdata = utils.RandomData(1024);
var testSkySE = await utils.AddModFile(mod, @"Data\SkyrimSE\Update.esm");
await testSkySE.WriteAllBytesAsync(pdata);
await utils.Configure();
await CompileAndInstall(profile, useGameFiles: true);
await utils.VerifyInstalledFile(mod, @"Data\MW\Bm.esm");
await utils.VerifyInstalledFile(mod, @"Data\SkyrimSE\Update.esm.old");
await utils.VerifyInstalledFile(mod, @"Data\SkyrimSE\Update.esm");
}
/// <summary>
/// Issue #861 : https://github.com/wabbajack-tools/wabbajack/issues/861

3
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -125,6 +125,9 @@ namespace Wabbajack.VirtualFileSystem
}
public VirtualFile TopParent => IsNative ? this : Parent.TopParent;
public T ThisAndAllChildrenReduced<T>(T acc, Func<T, VirtualFile, T> fn)
{
acc = fn(acc, this);