Fixes for WorkBase

CHANGELOG.md

@@ -1,5 +1,10 @@
 ### Changelog
 
+#### Version - 1.1.0.0 - 
+* Binary Patching stores temporary and patch data on disk instead of memory (reducing memory usage)
+* Fix a memory leak with diffing progress reporting
+* Fix a bug with bad data in inferred game INI files. 
+
 #### Verison - 1.0.0.0 - 2/29/2020
 * 1.0, first non-beta release
 

Wabbajack.Common/DynamicIniData.cs

@@ -76,7 +76,9 @@ namespace Wabbajack.Common
 
         private static string UnescapeString(string s)
         {
+            if (s.Trim().StartsWith("\"") || s.Contains("\\\\")) 
                 return Regex.Unescape(s.Trim('"'));
+            return s;
         }
 
         private static string UnescapeUTF8(string s)
@@ -121,7 +123,8 @@ namespace Wabbajack.Common
             }
 
             result = Coll[(string) indexes[0]];
-            if (result is string s) result = Regex.Unescape(s.Trim('"'));
+            if (result is string s && s.Trim().StartsWith("\"")) 
+                result = Regex.Unescape(s.Trim('"'));
             return true;
         }
     }

Wabbajack.Common/OctoDiff.cs

@@ -45,8 +45,12 @@ namespace Wabbajack.Common
 
         private class ProgressReporter : IProgressReporter
         {
+            private DateTime _lastUpdate = DateTime.UnixEpoch;
+            private readonly TimeSpan _updateInterval = TimeSpan.FromMilliseconds(100);
             public void ReportProgress(string operation, long currentPosition, long total)
             {
+                if (DateTime.Now - _lastUpdate < _updateInterval) return;
+                _lastUpdate = DateTime.Now;
                 if (currentPosition >= total || total < 1 || currentPosition < 0)
                     return;
                 Utils.Status(operation, new Percent(total, currentPosition));

Wabbajack.Common/Util/TempStream.cs

@@ -0,0 +1,33 @@
+using System.IO;
+using System.Threading.Tasks;
+using File = Alphaleonis.Win32.Filesystem.File;
+
+namespace Wabbajack.Common
+{
+    public class TempStream : FileStream
+    {
+        private TempFile _file;
+
+        public TempStream(TempFile file) : base(file.File.FullName, FileMode.Create, FileAccess.ReadWrite)
+        {
+            _file = file;
+        }
+
+        public TempStream() : this(new TempFile())
+        {
+            
+        }
+
+        protected override void Dispose(bool disposing)
+        {
+            base.Dispose(disposing);
+            _file.Dispose();
+        }
+
+        public override async ValueTask DisposeAsync()
+        {
+            await base.DisposeAsync();
+            _file.Dispose();
+        }
+    }
+}

Wabbajack.Common/Utils.cs

@@ -6,6 +6,7 @@ using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
+using System.Net.Http.Headers;
 using System.Reactive.Linq;
 using System.Reactive.Subjects;
 using System.Reflection;
@@ -958,6 +959,7 @@ namespace Wabbajack.Common
                     RETRY:
                     try
                     {
+                        
                         File.Move(tmpName, cacheFile, MoveOptions.ReplaceExisting);
                     }
                     catch (UnauthorizedAccessException)
@@ -975,6 +977,35 @@ namespace Wabbajack.Common
             }
         }
 
+        public static async Task CreatePatch(FileStream srcStream, string srcHash, FileStream destStream, string destHash,
+            FileStream patchStream)
+        {
+            await using var sigFile = new TempStream();
+            OctoDiff.Create(srcStream, destStream, sigFile, patchStream);
+            patchStream.Position = 0;
+            var tmpName = Path.Combine(Consts.PatchCacheFolder, Guid.NewGuid() + ".tmp");
+
+            await using (var f = File.Create(tmpName))
+            {
+                await patchStream.CopyToAsync(f);
+                patchStream.Position = 0;
+            }
+            
+            try
+            {
+                var cacheFile = Path.Combine(Consts.PatchCacheFolder, $"{srcHash.FromBase64().ToHex()}_{srcHash.FromBase64().ToHex()}.patch");
+                if (!Directory.Exists(Consts.PatchCacheFolder))
+                    Directory.CreateDirectory(Consts.PatchCacheFolder);
+
+                File.Move(tmpName, cacheFile, MoveOptions.ReplaceExisting);
+            }
+            catch (UnauthorizedAccessException)
+            {
+                if (File.Exists(tmpName)) 
+                    File.Delete(tmpName);
+            }
+        }
+
         public static bool TryGetPatch(string foundHash, string fileHash, out byte[] ePatch)
         {
             var patchName = Path.Combine(Consts.PatchCacheFolder,

Wabbajack.Lib/ACompiler.cs

@@ -71,6 +71,12 @@ namespace Wabbajack.Lib
             return id;
         }
 
+        internal FileStream IncludeFile(out string id)
+        {
+            id = Guid.NewGuid().ToString();
+            return File.Create(Path.Combine(ModListOutputFolder, id));
+        }
+
         internal string IncludeFile(string data)
         {
             var id = Guid.NewGuid().ToString();

Wabbajack.Lib/MO2Compiler.cs

@@ -184,6 +184,7 @@ namespace Wabbajack.Lib
             {
                 gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
                     .Where(p => p.FileExists())
+                    .Where(p => Path.GetExtension(p) != Consts.HashFileExtension)
                     .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p],
                         Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath))));
             }
@@ -448,22 +449,20 @@ namespace Wabbajack.Lib
                 {
                     Info($"Patching {entry.To}");
                     Status($"Patching {entry.To}");
-                    await using var origin = byPath[string.Join("|", entry.ArchiveHashPath.Skip(1))].OpenRead();
+                    var srcFile = byPath[string.Join("|", entry.ArchiveHashPath.Skip(1))];
-                    await using var output = new MemoryStream();
+                    await using var srcStream = srcFile.OpenRead();
-                    var a = origin.ReadAll();
+                    await using var outputStream = IncludeFile(out entry.PatchID);
-                    var b = LoadDataForTo(entry.To, absolutePaths);
+                    await using var destStream = LoadDataForTo(entry.To, absolutePaths);
-                    await Utils.CreatePatch(a, b, output);
+                    await Utils.CreatePatch(srcStream, srcFile.Hash, destStream, entry.Hash, outputStream);
-                    entry.PatchID = IncludeFile(output.ToArray());
+                    Info($"Patch size {outputStream.Length} for {entry.To}");
-                    var fileSize = File.GetSize(Path.Combine(ModListOutputFolder, entry.PatchID));
-                    Info($"Patch size {fileSize} for {entry.To}");
                 });
             }
         }
 
-        private byte[] LoadDataForTo(string to, Dictionary<string, string> absolutePaths)
+        private FileStream LoadDataForTo(string to, Dictionary<string, string> absolutePaths)
         {
             if (absolutePaths.TryGetValue(to, out var absolute))
-                return File.ReadAllBytes(absolute);
+                return File.OpenRead(absolute);
 
             if (to.StartsWith(Consts.BSACreationDir))
             {
@@ -474,11 +473,10 @@ namespace Wabbajack.Lib
                 {
                     var find = Path.Combine(to.Split('\\').Skip(2).ToArray());
                     var file = a.Files.First(e => e.Path.Replace('/', '\\') == find);
-                    using (var ms = new MemoryStream())
+                    var returnStream = new TempStream();
-                    {
+                    file.CopyDataTo(returnStream);
-                        file.CopyDataTo(ms);
+                    returnStream.Position = 0;
-                        return ms.ToArray();
+                    return returnStream;
-                    }
                 }
             }
 
@@ -518,6 +516,7 @@ namespace Wabbajack.Lib
                 new IgnoreStartsWith(this, "downloads\\"),
                 new IgnoreStartsWith(this,"webcache\\"),
                 new IgnoreStartsWith(this, "overwrite\\"),
+                new IgnoreStartsWith(this, "crashDumps\\"),
                 new IgnorePathContains(this,"temporary_logs"),
                 new IgnorePathContains(this, "GPUCache"),
                 new IgnorePathContains(this, "SSEEdit Cache"),

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -335,7 +335,7 @@ namespace Wabbajack.VirtualFileSystem
             return path;
         }
 
-        public Stream OpenRead()
+        public FileStream OpenRead()
         {
             return File.OpenRead(StagedPath);
         }