start of zEdit integration

CHANGELOG.md

@@ -4,6 +4,9 @@
 * Add WABBAJACK_NOMATCH_INCLUDE works like WABBAJACK_INCLUDE but only includes files that are found to be missing at the end of compilation
 * Add a list of all inlined data blobs to the install report, useful for reducing installer sizes
 * Increased dummy EPS detection size to 250 bytes and added .esm files to the filter logic
+* Only sync the VFS cache when it changes.
+* Fix a crash in GroupedByArchive()
+* Detect and zEdit Merges and include binary patches for merges (no install support yet)
 
 #### Version 0.9.2 - 9/18/2013
 * Fixed a bug with BSA string encoding

Compression.BSA.Test/packages.config

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<packages>
+  <package id="SharpZipLib" version="1.2.0" targetFramework="net472" />
+</packages>

VirtualFileSystem/VirtualFileSystem.cs

@@ -23,6 +23,7 @@ namespace VFS
         private bool _disableDiskCache;
         private Dictionary<string, VirtualFile> _files = new Dictionary<string, VirtualFile>();
         private volatile bool _isSyncing;
+        private volatile bool _isDirty = false;
 
         static VirtualFileSystem()
         {
@@ -123,6 +124,7 @@ namespace VFS
             {
                 Utils.Log($"Purging cache due to {ex}");
                 File.Delete("vfs_cache.bson");
+                _isDirty = true;
                 _files.Clear();
             }
         }
@@ -134,6 +136,7 @@ namespace VFS
                 Utils.Status("Syncing VFS Cache");
                 lock (this)
                 {
+                    if (!_isDirty) return;
                     try
                     {
                         _isSyncing = true;
@@ -152,6 +155,7 @@ namespace VFS
                             File.Delete("vfs_cache.bin");
 
                         File.Move("vfs_cache.bin_new", "vfs_cache.bin");
+                        _isDirty = false;
                     }
                     finally
                     {
@@ -178,7 +182,11 @@ namespace VFS
                 _files.Values
                     .Where(v => v.FullPath.StartsWith(path) || v.FullPath == f.FullPath)
                     .ToList()
-                    .Do(r => { _files.Remove(r.FullPath); });
+                    .Do(r =>
+                    {
+                        _isDirty = true;
+                        _files.Remove(r.FullPath);
+                    });
             }
         }
 
@@ -186,6 +194,7 @@ namespace VFS
         {
             lock (this)
             {
+                _isDirty = true;
                 if (_files.ContainsKey(f.FullPath))
                     Purge(f);
                 _files.Add(f.FullPath, f);
@@ -229,7 +238,11 @@ namespace VFS
                         return false;
                     })
                     .ToList()
-                    .Do(f => _files.Remove(f.FullPath));
+                    .Do(f =>
+                    {
+                        _isDirty = true;
+                        _files.Remove(f.FullPath);
+                    });
             }
         }
 
@@ -455,7 +468,9 @@ namespace VFS
 
         public IDictionary<VirtualFile, IEnumerable<VirtualFile>> GroupedByArchive()
         {
-            return _files.Values.GroupBy(f => f.TopLevelArchive)
+            return _files.Values
+                .Where(f => f.TopLevelArchive != null)
+                .GroupBy(f => f.TopLevelArchive)
                 .ToDictionary(f => f.Key, f => (IEnumerable<VirtualFile>) f);
         }
     }

Wabbajack.Common/DynamicIniData.cs

@@ -47,5 +47,18 @@ namespace Wabbajack.Common
             if (result is string) result = Regex.Unescape(((string) result).Trim('"'));
             return true;
         }
+
+        public override bool TryGetIndex(GetIndexBinder binder, object[] indexes, out object result)
+        {
+            if (indexes.Length > 1)
+            {
+                result = null;
+                return false;
+            }
+
+            result = _coll[(string) indexes[0]];
+            if (result is string) result = Regex.Unescape(((string)result).Trim('"'));
+            return true;
+        }
     }
 }

Wabbajack.Common/Utils.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Net.Configuration;
 using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
@@ -478,5 +479,22 @@ namespace Wabbajack.Common
                 $"{foundHash.FromBase64().ToHEX()}_{fileHash.FromBase64().ToHEX()}.patch");
             ePatch = File.Exists(patch_name) ? File.ReadAllBytes(patch_name) : null;
         }
+
+        public static void Warning(string s)
+        {
+            Log($"WARNING: {s}");
+        }
+
+        public static byte[] ConcatArrays(this IEnumerable<byte[]> arrays)
+        {
+            var outarr = new byte[arrays.Sum(a => a.Length)];
+            int offset = 0;
+            foreach (var arr in arrays)
+            {
+                Array.Copy(arr, 0, outarr, offset, arr.Length);
+                offset += arr.Length;
+            }
+            return outarr;
+        }
     }
 }

Wabbajack/Compiler.cs

@@ -175,6 +175,7 @@ namespace Wabbajack
 
             AllFiles = mo2_files.Concat(game_files)
                 .Concat(loot_files)
+                .DistinctBy(f => f.Path)
                 .ToList();
 
             Info("Found {0} files to build into mod list", AllFiles.Count);
@@ -229,6 +230,7 @@ namespace Wabbajack
 
             if (!User.is_premium) Info($"User {User.name} is not a premium Nexus user, cannot continue");
 
+            zEditIntegration.VerifyMerges(this);
 
             GatherArchives();
             BuildPatches();
@@ -566,6 +568,7 @@ namespace Wabbajack
 
                 IncludeAllConfigs(),
                 IncludeTaggedFiles(Consts.WABBAJACK_NOMATCH_INCLUDE),
+                zEditIntegration.IncludezEditPatches(this),
 
                 DropAll()
             };
@@ -821,7 +824,7 @@ namespace Wabbajack
 
                 foreach (var match in matches)
                 {
-                    if (match is IgnoredDirectly) Error($"File required for BSA creation doesn't exist: {match.To}");
+                    if (match is IgnoredDirectly) Error($"File required for BSA {source.Path} creation doesn't exist: {match.To}");
                     ExtraFiles.Add(match);
                 }
 

Wabbajack/Data.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 using VFS;
 
@@ -148,6 +147,21 @@ namespace Wabbajack
         public byte[] Patch;
     }
 
+    [Serializable]
+    public class SourcePatch
+    {
+        public string RelativePath;
+        public string Hash;
+    }
+
+    [Serializable]
+    public class MergedPatch : Directive
+    {
+        public List<SourcePatch> Sources;
+        public string Hash;
+        public byte[] Patch;
+    }
+
     [Serializable]
     public class Archive
     {

Wabbajack/Wabbajack.csproj

@@ -186,6 +186,7 @@
     <Compile Include="Themes\LeftMarginMultiplierConverter.cs" />
     <Compile Include="Themes\TreeViewItemExtensions.cs" />
     <Compile Include="UIUtils.cs" />
+    <Compile Include="zEditIntegration.cs" />
     <Page Include="MainWindow.xaml">
       <Generator>MSBuild:Compile</Generator>
       <SubType>Designer</SubType>

Wabbajack/zEditIntegration.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using VFS;
+using Wabbajack.Common;
+using Directory = Alphaleonis.Win32.Filesystem.Directory;
+using File = Alphaleonis.Win32.Filesystem.File;
+using Path = Alphaleonis.Win32.Filesystem.Path;
+
+namespace Wabbajack
+{
+    public class zEditIntegration
+    {
+        public static string FindzEditPath(Compiler compiler)
+        {
+            var executables = compiler.MO2Ini.customExecutables;
+
+            foreach (var idx in Enumerable.Range(1, int.Parse(executables.size)))
+            {
+                var path = (string)executables[$"{idx}\\binary"];
+                if (path == null) continue;
+
+                if (path.EndsWith("zEdit.exe"))
+                    return Path.GetDirectoryName(path);
+            }
+
+            return null;
+        }
+
+        public static Func<RawSourceFile, Directive> IncludezEditPatches(Compiler compiler)
+        {
+            var zEditPath = FindzEditPath(compiler);
+            var havezEdit = zEditPath != null;
+
+            Utils.Log(havezEdit ? $"Found zEdit at {zEditPath}" : $"zEdit not detected, disabling zEdit routines");
+
+            if (!havezEdit) return source => { return null; };
+
+            var merges = Directory.EnumerateFiles(Path.Combine(zEditPath, "profiles"),
+                    DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive)
+                .Where(f => f.EndsWith("\\merges.json"))
+                .SelectMany(f => f.FromJSON<List<zEditMerge>>())
+                .GroupBy(f => (f.name, f.filename));
+
+            merges.Where(m => m.Count() > 1)
+                .Do(m => {
+                Utils.Warning(
+                    $"WARNING, you have two patches named {m.Key.name}\\{m.Key.filename} in your zEdit profiles. We'll pick one at random, this probably isn't what you want.");
+                });
+
+            var mergesIndexed =
+                merges.ToDictionary(
+                    m => Path.Combine(compiler.MO2Folder, "mods", m.Key.name, m.Key.filename),
+                    m => m.First());
+
+
+
+            return source =>
+            {
+                if (mergesIndexed.TryGetValue(source.AbsolutePath, out var merge))
+                {
+                    var result = source.EvolveTo<MergedPatch>();
+                    result.Sources = merge.plugins.Select(f =>
+                    {
+                        var abs_path = Path.Combine(f.dataFolder, f.filename);
+                        if (!File.Exists(abs_path))
+                            throw new InvalidDataException(
+                                $"File {abs_path} is required to build {merge.filename} but it doesn't exist");
+
+                        return new SourcePatch
+                        {
+                            RelativePath = abs_path.RelativeTo(compiler.MO2Folder),
+                            Hash = compiler.VFS[abs_path].Hash
+                        };
+                    }).ToList();
+
+                    var src_data = merge.plugins.Select(f => File.ReadAllBytes(Path.Combine(f.dataFolder, f.filename)))
+                        .ConcatArrays();
+
+                    var dst_data = File.ReadAllBytes(source.AbsolutePath);
+
+                    using (var ms = new MemoryStream())
+                    {
+                        Utils.CreatePatch(src_data, dst_data, ms);
+                        result.Patch = ms.ToArray();
+                    }
+
+                    return result;
+
+                }
+                return null;
+            };
+
+
+        }
+
+
+        class zEditMerge
+        {
+            public string name;
+            public string filename;
+            public List<zEditMergePlugin> plugins;
+
+        }
+
+        class zEditMergePlugin
+        {
+            public string filename;
+            public string dataFolder;
+        }
+
+        public static void VerifyMerges(Compiler compiler)
+        {
+            var by_name = compiler.InstallDirectives.ToDictionary(f => f.To);
+
+            foreach (var directive in compiler.InstallDirectives.OfType<MergedPatch>())
+            {
+                foreach (var source in directive.Sources)
+                {
+                    if (by_name.TryGetValue(source.RelativePath, out var result))
+                    {
+                        if (result.Hash != source.Hash)
+                            throw new InvalidDataException($"Hashes for {result.To} needed for zEdit merge sources don't match, this shouldn't happen");
+                        continue;
+                    }
+                    throw new InvalidDataException($"{source.RelativePath} is needed for merged patch {directive.To} but is not included in the install.");
+                }
+            }
+        }
+    }
+}