diff --git a/CHANGELOG.md b/CHANGELOG.md index bf266c28..ef95a148 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ #### Version - 2.3.0.0 - ??? * Rewrote the file extraction routines. New code uses less memory, less disk space, and performs less thrashing on HDDs * Reworked IPS4 integration to reduce download failures - +* Profiles can now contain an (optional) file `compiler_settings.json` that includes options for other games to be used during install. +This is now the only way to include extra games in the install process, implicit inclusion has been removed. #### Version - 2.2.2.0 - 8/31/2020 * Route CDN requests through a reverse proxy to improve reliability diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index 9ec216a9..aa9279ff 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -4,10 +4,13 @@ using System.ComponentModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; using Wabbajack.Common.StoreHandlers; namespace Wabbajack.Common { + [JsonConverter(typeof(StringEnumConverter))] public enum Game { //MO2 GAMES diff --git a/Wabbajack.Common/Json.cs b/Wabbajack.Common/Json.cs index 78c6b611..006421e9 100644 --- a/Wabbajack.Common/Json.cs +++ b/Wabbajack.Common/Json.cs @@ -51,18 +51,18 @@ namespace Wabbajack.Common File.WriteAllText(filename, JsonConvert.SerializeObject(obj, Formatting.Indented, JsonSettings)); } - public static void ToJson(this T obj, Stream stream) + public static void ToJson(this T obj, Stream stream, bool useGenericSettings = false) { using var tw = new StreamWriter(stream, Encoding.UTF8, bufferSize: 1024, leaveOpen: true); using var writer = new JsonTextWriter(tw); - var ser = JsonSerializer.Create(JsonSettings); + var ser = JsonSerializer.Create(useGenericSettings ? GenericJsonSettings : JsonSettings); ser.Serialize(writer, obj); } - public static async ValueTask ToJsonAsync(this T obj, AbsolutePath path) + public static async ValueTask ToJsonAsync(this T obj, AbsolutePath path, bool useGenericSettings = false) { await using var fs = await path.Create(); - obj.ToJson(fs); + obj.ToJson(fs, useGenericSettings); } public static string ToJson(this T obj, bool useGenericSettings = false) diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index a9df9d68..833bcfaa 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -783,6 +783,14 @@ namespace Wabbajack.Common .Build(); return d.Deserialize(new StringReader(s)); } + + public static T FromYaml(this AbsolutePath s) + { + var d = new DeserializerBuilder() + .WithNamingConvention(PascalCaseNamingConvention.Instance) + .Build(); + return d.Deserialize(new StringReader((string)s)); + } public static void LogStatus(string s) { Status(s); diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index ad291fed..6924cefd 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -334,7 +334,6 @@ namespace Wabbajack.Lib .PMap(Queue, UpdateTracker, async f => { var relativeTo = f.RelativeTo(OutputFolder); - Utils.Status($"Checking if ModList file {relativeTo}"); if (indexed.ContainsKey(relativeTo) || f.InFolder(DownloadFolder)) return; @@ -378,17 +377,19 @@ namespace Wabbajack.Lib Utils.Log("Error when trying to clean empty folders. This doesn't really matter."); } + var existingfiles = OutputFolder.EnumerateFiles().ToHashSet(); + UpdateTracker.NextStep("Looking for unmodified files"); (await indexed.Values.PMap(Queue, UpdateTracker, async d => { // Bit backwards, but we want to return null for // all files we *want* installed. We return the files // to remove from the install list. - Status($"Optimizing {d.To}"); var path = OutputFolder.Combine(d.To); - if (!path.Exists) return null; + if (!existingfiles.Contains(path)) return null; if (path.Size != d.Size) return null; + Status($"Optimizing {d.To}"); return await path.FileHashCachedAsync() == d.Hash ? d : null; })) diff --git a/Wabbajack.Lib/CompilerSettings.cs b/Wabbajack.Lib/CompilerSettings.cs new file mode 100644 index 00000000..4de29596 --- /dev/null +++ b/Wabbajack.Lib/CompilerSettings.cs @@ -0,0 +1,19 @@ +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.Lib +{ + public class CompilerSettings + { + public const string FileName = "compiler_settings.json"; + + public static async Task Load(AbsolutePath folder) + { + var path = folder.Combine(FileName); + return !path.IsFile ? new CompilerSettings() : path.FromJson(); + } + + public Game[] IncludedGames { get; set; } = new Game[0]; + public string[] OtherProfiles { get; set; } = new string[0]; + } +} diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 7b935812..c4fa92ca 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -31,11 +31,8 @@ namespace Wabbajack.Lib public override AbsolutePath GamePath { get; } public GameMetaData CompilingGame { get; } - - /// - /// All games available for sourcing during compilation (including the Compiling Game) - /// - public List AvailableGames { get; } + + public CompilerSettings Settings { get; set; } public override AbsolutePath ModListOutputFolder => ((RelativePath)"output_folder").RelativeToEntryPoint(); @@ -66,8 +63,7 @@ namespace Wabbajack.Lib CompilingGame = GameRegistry.Games.First(g => g.Value.MO2Name == mo2game).Value; GamePath = new AbsolutePath((string)MO2Ini.General.gamePath.Replace("\\\\", "\\")); ModListOutputFile = outputFile; - - AvailableGames = CompilingGame.CanSourceFrom.Cons(CompilingGame.Game).Where(g => g.MetaData().IsInstalled).ToList(); + Settings = new CompilerSettings(); } public AbsolutePath MO2DownloadsFolder @@ -91,6 +87,11 @@ namespace Wabbajack.Lib Queue.SetActiveThreadsObservable(ConstructDynamicNumThreads(await RecommendQueueSize())); UpdateTracker.Reset(); UpdateTracker.NextStep("Gathering information"); + + Utils.Log($"Loading compiler Settings"); + Settings = await CompilerSettings.Load(MO2ProfileDir); + Settings.IncludedGames = Settings.IncludedGames.Add(CompilingGame.Game); + Info("Looking for other profiles"); var otherProfilesPath = MO2ProfileDir.Combine("otherprofiles.txt"); SelectedProfiles = new HashSet(); @@ -121,7 +122,7 @@ namespace Wabbajack.Lib { MO2Folder, GamePath, MO2DownloadsFolder }; - roots.AddRange(AvailableGames.Select(g => g.MetaData().GameLocation())); + roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation())); } else { @@ -206,7 +207,7 @@ namespace Wabbajack.Lib if (UseGamePaths) { - foreach (var ag in AvailableGames) + foreach (var ag in Settings.IncludedGames) { try { diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index ced679ae..ad9ce598 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -248,23 +248,19 @@ namespace Wabbajack.Lib private async Task InstallIncludedDownloadMetas() { - await ModList.Directives - .OfType() - .PMap(Queue, async directive => + await ModList.Archives + .PMap(Queue, async archive => { - Status($"Writing .meta file {directive.To}"); - foreach (var archive in ModList.Archives) + if (HashedArchives.TryGetValue(archive.Hash, out var paths)) { - if (HashedArchives.TryGetValue(archive.Hash, out var paths)) + var metaPath = paths.WithExtension(Consts.MetaFileExtension); + if (!metaPath.Exists) { - var metaPath = paths.WithExtension(Consts.MetaFileExtension); - if (!metaPath.Exists) - { - var meta = AddInstalled(archive.State.GetMetaIni()).ToArray(); - await metaPath.WriteAllLinesAsync(meta); - } + Status($"Writing {metaPath.FileName}"); + var meta = AddInstalled(archive.State.GetMetaIni()).ToArray(); + await metaPath.WriteAllLinesAsync(meta); } - } + } }); } diff --git a/Wabbajack.Test/SanityTests.cs b/Wabbajack.Test/SanityTests.cs index 365c8641..b47888e3 100644 --- a/Wabbajack.Test/SanityTests.cs +++ b/Wabbajack.Test/SanityTests.cs @@ -508,6 +508,11 @@ namespace Wabbajack.Test var profile = utils.AddProfile(); var mod = await utils.AddMod(); + await new CompilerSettings() + { + IncludedGames = new []{Game.Morrowind} + }.ToJsonAsync(utils.MO2Folder.Combine("profiles", profile, CompilerSettings.FileName), true); + Game.SkyrimSpecialEdition.MetaData().CanSourceFrom = new[] {Game.Morrowind, Game.Skyrim}; // Morrowind file with different name diff --git a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs index 607410ab..18c39e62 100644 --- a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs @@ -55,10 +55,10 @@ namespace Wabbajack.VirtualFileSystem.Test var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path), _ => true, async (path, sfn) => - { - await using var s = await sfn.GetStream(); - return await s.xxHashAsync(); - }); + { + await using var s = await sfn.GetStream(); + return await s.xxHashAsync(); + }); Assert.Equal(10, results.Count); foreach (var (path, hash) in results) @@ -67,6 +67,35 @@ namespace Wabbajack.VirtualFileSystem.Test } } + [Fact] + public async Task CanExtractEmptyFiles() + { + await using var temp = await TempFolder.Create(); + await using var archive = new TempFile(); + + for (int i = 0; i < 1; i ++) + { + await WriteRandomData(temp.Dir.Combine($"{i}.bin"), _rng.Next(10, 1024)); + } + await (await temp.Dir.Combine("empty.txt").Create()).DisposeAsync(); + + await ZipUpFolder(temp.Dir, archive.Path, false); + + var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path), + _ => true, + async (path, sfn) => + { + await using var s = await sfn.GetStream(); + return await s.xxHashAsync(); + }); + + Assert.Equal(2, results.Count); + foreach (var (path, hash) in results) + { + Assert.Equal(await temp.Dir.Combine(path).FileHashAsync(), hash); + } + } + private static Extension OMODExtension = new Extension(".omod"); private static Extension CRCExtension = new Extension(".crc"); diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs index c86eb922..1fd98d6d 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs @@ -25,7 +25,6 @@ namespace Wabbajack.VirtualFileSystem public GatheringExtractor(Stream stream, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) { - _shouldExtract = shouldExtract; _mapFn = mapfn; _results = new Dictionary(); @@ -89,6 +88,14 @@ namespace Wabbajack.VirtualFileSystem { if (_indexes.ContainsKey(index)) { + var path = _indexes[index].Item1; + Utils.Status($"Extracting {path}", Percent.FactoryPutInRange(_results.Count, _indexes.Count)); + // Empty files are never extracted via a write call, so we have to fake that now + if (_indexes[index].Item2 == 0) + { + var result = _mapFn(path, new MemoryStreamFactory(new MemoryStream(), path)).Result; + _results.Add(path, result); + } outStream = new GatheringExtractorStream(this, index); return 0; }