Merge remote-tracking branch 'origin/master' into master

CHANGELOG.md

@@ -3,7 +3,8 @@
 #### Version - 2.3.0.0 - ???
 * Rewrote the file extraction routines. New code uses less memory, less disk space, and performs less thrashing on HDDs
 * Reworked IPS4 integration to reduce download failures
-
+* Profiles can now contain an (optional) file `compiler_settings.json` that includes options for other games to be used during install.
+This is now the only way to include extra games in the install process, implicit inclusion has been removed. 
 
 #### Version - 2.2.2.0 - 8/31/2020
 * Route CDN requests through a reverse proxy to improve reliability

Wabbajack.Common/GameMetaData.cs

@@ -4,10 +4,13 @@ using System.ComponentModel;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
 using Wabbajack.Common.StoreHandlers;
 
 namespace Wabbajack.Common
 {
+    [JsonConverter(typeof(StringEnumConverter))]
     public enum Game 
     {
         //MO2 GAMES

Wabbajack.Common/Json.cs

@@ -51,18 +51,18 @@ namespace Wabbajack.Common
             File.WriteAllText(filename, JsonConvert.SerializeObject(obj, Formatting.Indented, JsonSettings));
         }
         
-        public static void ToJson<T>(this T obj, Stream stream)
+        public static void ToJson<T>(this T obj, Stream stream, bool useGenericSettings = false)
         {
             using var tw = new StreamWriter(stream, Encoding.UTF8, bufferSize: 1024, leaveOpen: true);
             using var writer = new JsonTextWriter(tw);
-            var ser = JsonSerializer.Create(JsonSettings);
+            var ser = JsonSerializer.Create(useGenericSettings ? GenericJsonSettings : JsonSettings);
             ser.Serialize(writer, obj);
         }
         
-        public static async ValueTask ToJsonAsync<T>(this T obj, AbsolutePath path)
+        public static async ValueTask ToJsonAsync<T>(this T obj, AbsolutePath path, bool useGenericSettings = false)
         {
             await using var fs = await path.Create();
-            obj.ToJson(fs);
+            obj.ToJson(fs, useGenericSettings);
         }
 
         public static string ToJson<T>(this T obj, bool useGenericSettings = false)

Wabbajack.Common/Utils.cs

@@ -783,6 +783,14 @@ namespace Wabbajack.Common
                 .Build();
             return d.Deserialize<T>(new StringReader(s));
         }
+        
+        public static T FromYaml<T>(this AbsolutePath s)
+        {
+            var d = new DeserializerBuilder()
+                .WithNamingConvention(PascalCaseNamingConvention.Instance)
+                .Build();
+            return d.Deserialize<T>(new StringReader((string)s));
+        }
         public static void LogStatus(string s)
         {
             Status(s);

Wabbajack.Lib/AInstaller.cs

@@ -334,7 +334,6 @@ namespace Wabbajack.Lib
                 .PMap(Queue, UpdateTracker, async f =>
                 {
                     var relativeTo = f.RelativeTo(OutputFolder);
-                    Utils.Status($"Checking if ModList file {relativeTo}");
                     if (indexed.ContainsKey(relativeTo) || f.InFolder(DownloadFolder))
                         return;
 
@@ -378,17 +377,19 @@ namespace Wabbajack.Lib
                 Utils.Log("Error when trying to clean empty folders. This doesn't really matter.");
             }
 
+            var existingfiles = OutputFolder.EnumerateFiles().ToHashSet();
+            
             UpdateTracker.NextStep("Looking for unmodified files");
             (await indexed.Values.PMap(Queue, UpdateTracker, async d =>
             {
                 // Bit backwards, but we want to return null for 
                 // all files we *want* installed. We return the files
                 // to remove from the install list.
-                Status($"Optimizing {d.To}");
                 var path = OutputFolder.Combine(d.To);
-                if (!path.Exists) return null;
+                if (!existingfiles.Contains(path)) return null;
 
                 if (path.Size != d.Size) return null;
+                Status($"Optimizing {d.To}");
                 
                 return await path.FileHashCachedAsync() == d.Hash ? d : null;
             }))

Wabbajack.Lib/CompilerSettings.cs

@@ -0,0 +1,19 @@
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib
+{
+    public class CompilerSettings
+    {
+        public const string FileName = "compiler_settings.json";
+
+        public static async Task<CompilerSettings> Load(AbsolutePath folder)
+        {
+            var path = folder.Combine(FileName);
+            return !path.IsFile ? new CompilerSettings() : path.FromJson<CompilerSettings>();
+        }
+
+        public Game[] IncludedGames { get; set; } = new Game[0];
+        public string[] OtherProfiles { get; set; } = new string[0];
+    }
+}

Wabbajack.Lib/MO2Compiler.cs

@@ -31,11 +31,8 @@ namespace Wabbajack.Lib
         public override AbsolutePath GamePath { get; }
 
         public GameMetaData CompilingGame { get; }
-        
+     
-        /// <summary>
+        public CompilerSettings Settings { get; set; }
-        /// All games available for sourcing during compilation (including the Compiling Game)
-        /// </summary>
-        public List<Game> AvailableGames { get; }
 
         public override AbsolutePath ModListOutputFolder => ((RelativePath)"output_folder").RelativeToEntryPoint();
 
@@ -66,8 +63,7 @@ namespace Wabbajack.Lib
             CompilingGame = GameRegistry.Games.First(g => g.Value.MO2Name == mo2game).Value;
             GamePath = new AbsolutePath((string)MO2Ini.General.gamePath.Replace("\\\\", "\\"));
             ModListOutputFile = outputFile;
-
+            Settings = new CompilerSettings();
-            AvailableGames = CompilingGame.CanSourceFrom.Cons(CompilingGame.Game).Where(g => g.MetaData().IsInstalled).ToList();
         }
 
         public AbsolutePath MO2DownloadsFolder
@@ -91,6 +87,11 @@ namespace Wabbajack.Lib
             Queue.SetActiveThreadsObservable(ConstructDynamicNumThreads(await RecommendQueueSize()));
             UpdateTracker.Reset();
             UpdateTracker.NextStep("Gathering information");
+
+            Utils.Log($"Loading compiler Settings");
+            Settings = await CompilerSettings.Load(MO2ProfileDir);
+            Settings.IncludedGames = Settings.IncludedGames.Add(CompilingGame.Game);
+            
             Info("Looking for other profiles");
             var otherProfilesPath = MO2ProfileDir.Combine("otherprofiles.txt");
             SelectedProfiles = new HashSet<string>();
@@ -121,7 +122,7 @@ namespace Wabbajack.Lib
                 {
                     MO2Folder, GamePath, MO2DownloadsFolder
                 };
-                roots.AddRange(AvailableGames.Select(g => g.MetaData().GameLocation()));
+                roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation()));
             }
             else
             {
@@ -206,7 +207,7 @@ namespace Wabbajack.Lib
 
             if (UseGamePaths)
             {
-                foreach (var ag in AvailableGames)
+                foreach (var ag in Settings.IncludedGames)
                 {
                     try
                     {

Wabbajack.Lib/MO2Installer.cs

@@ -248,23 +248,19 @@ namespace Wabbajack.Lib
 
         private async Task InstallIncludedDownloadMetas()
         {
-            await ModList.Directives
+            await ModList.Archives
-                   .OfType<ArchiveMeta>()
+                   .PMap(Queue, async archive =>
-                   .PMap(Queue, async directive =>
                    {
-                       Status($"Writing .meta file {directive.To}");
+                       if (HashedArchives.TryGetValue(archive.Hash, out var paths))
-                       foreach (var archive in ModList.Archives)
                        {
-                           if (HashedArchives.TryGetValue(archive.Hash, out var paths))
+                           var metaPath = paths.WithExtension(Consts.MetaFileExtension);
+                           if (!metaPath.Exists)
                            {
-                               var metaPath = paths.WithExtension(Consts.MetaFileExtension);
+                               Status($"Writing {metaPath.FileName}");
-                               if (!metaPath.Exists)
+                               var meta = AddInstalled(archive.State.GetMetaIni()).ToArray();
-                               {
+                               await metaPath.WriteAllLinesAsync(meta);
-                                   var meta = AddInstalled(archive.State.GetMetaIni()).ToArray();
-                                   await metaPath.WriteAllLinesAsync(meta);
-                               }
                            }
-                       } 
+                       }
                    });
         }
 

Wabbajack.Test/SanityTests.cs

@@ -508,6 +508,11 @@ namespace Wabbajack.Test
             var profile = utils.AddProfile();
             var mod = await utils.AddMod();
 
+            await new CompilerSettings()
+            {
+                IncludedGames = new []{Game.Morrowind}
+            }.ToJsonAsync(utils.MO2Folder.Combine("profiles", profile, CompilerSettings.FileName), true);
+
             Game.SkyrimSpecialEdition.MetaData().CanSourceFrom = new[] {Game.Morrowind, Game.Skyrim};
             
             // Morrowind file with different name

Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs

@@ -55,10 +55,10 @@ namespace Wabbajack.VirtualFileSystem.Test
             var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path), 
                 _ => true,
                 async (path, sfn) =>
-            {
+                {
-                await using var s = await sfn.GetStream();
+                    await using var s = await sfn.GetStream();
-                return await s.xxHashAsync();
+                    return await s.xxHashAsync();
-            });
+                });
             
             Assert.Equal(10, results.Count);
             foreach (var (path, hash) in results)
@@ -67,6 +67,35 @@ namespace Wabbajack.VirtualFileSystem.Test
             }
         }
         
+        [Fact]
+        public async Task CanExtractEmptyFiles()
+        {
+            await using var temp = await TempFolder.Create();
+            await using var archive = new TempFile();
+            
+            for (int i = 0; i < 1; i ++)
+            {
+                await WriteRandomData(temp.Dir.Combine($"{i}.bin"), _rng.Next(10, 1024));
+            }
+            await (await temp.Dir.Combine("empty.txt").Create()).DisposeAsync();
+
+            await ZipUpFolder(temp.Dir, archive.Path, false);
+            
+            var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path), 
+                _ => true,
+                async (path, sfn) =>
+                {
+                    await using var s = await sfn.GetStream();
+                    return await s.xxHashAsync();
+                });
+            
+            Assert.Equal(2, results.Count);
+            foreach (var (path, hash) in results)
+            {
+                Assert.Equal(await temp.Dir.Combine(path).FileHashAsync(), hash);
+            }
+        }
+        
         private static Extension OMODExtension = new Extension(".omod");
         private static Extension CRCExtension = new Extension(".crc");
 

Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs

@@ -25,7 +25,6 @@ namespace Wabbajack.VirtualFileSystem
 
         public GatheringExtractor(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory, ValueTask<T>> mapfn)
         {
-            
             _shouldExtract = shouldExtract;
             _mapFn = mapfn;
             _results = new Dictionary<RelativePath, T>();
@@ -89,6 +88,14 @@ namespace Wabbajack.VirtualFileSystem
         {
             if (_indexes.ContainsKey(index))
             {
+                var path = _indexes[index].Item1;
+                Utils.Status($"Extracting {path}", Percent.FactoryPutInRange(_results.Count, _indexes.Count));
+                // Empty files are never extracted via a write call, so we have to fake that now
+                if (_indexes[index].Item2 == 0)
+                {
+                    var result = _mapFn(path, new MemoryStreamFactory(new MemoryStream(), path)).Result;
+                    _results.Add(path, result);
+                }
                 outStream =  new GatheringExtractorStream<T>(this, index);
                 return 0;
             }