diff --git a/CHANGELOG.md b/CHANGELOG.md index b972d878..b9e28411 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ ### Changelog -#### Version - 1.0.1.0 - 3/xx/2020 +#### Version - Next +* Binary Patching stores temporary and patch data on disk instead of memory (reducing memory usage) +* Fix a memory leak with diffing progress reporting +* Fix a bug with bad data in inferred game INI files. * Added download support for YouTube * Slideshow can now display mods from non-Nexus sites diff --git a/Wabbajack.Common/DynamicIniData.cs b/Wabbajack.Common/DynamicIniData.cs index bd850aa9..9f5a7c64 100644 --- a/Wabbajack.Common/DynamicIniData.cs +++ b/Wabbajack.Common/DynamicIniData.cs @@ -76,7 +76,9 @@ namespace Wabbajack.Common private static string UnescapeString(string s) { - return Regex.Unescape(s.Trim('"')); + if (s.Trim().StartsWith("\"") || s.Contains("\\\\")) + return Regex.Unescape(s.Trim('"')); + return s; } private static string UnescapeUTF8(string s) @@ -121,7 +123,8 @@ namespace Wabbajack.Common } result = Coll[(string) indexes[0]]; - if (result is string s) result = Regex.Unescape(s.Trim('"')); + if (result is string s && s.Trim().StartsWith("\"")) + result = Regex.Unescape(s.Trim('"')); return true; } } diff --git a/Wabbajack.Common/OctoDiff.cs b/Wabbajack.Common/OctoDiff.cs index b2dcfffe..f03e5d09 100644 --- a/Wabbajack.Common/OctoDiff.cs +++ b/Wabbajack.Common/OctoDiff.cs @@ -45,8 +45,12 @@ namespace Wabbajack.Common private class ProgressReporter : IProgressReporter { + private DateTime _lastUpdate = DateTime.UnixEpoch; + private readonly TimeSpan _updateInterval = TimeSpan.FromMilliseconds(100); public void ReportProgress(string operation, long currentPosition, long total) { + if (DateTime.Now - _lastUpdate < _updateInterval) return; + _lastUpdate = DateTime.Now; if (currentPosition >= total || total < 1 || currentPosition < 0) return; Utils.Status(operation, new Percent(total, currentPosition)); diff --git a/Wabbajack.Common/Util/TempStream.cs b/Wabbajack.Common/Util/TempStream.cs new file mode 100644 index 00000000..b65e4027 --- /dev/null +++ b/Wabbajack.Common/Util/TempStream.cs @@ -0,0 +1,33 @@ +using System.IO; +using System.Threading.Tasks; +using File = Alphaleonis.Win32.Filesystem.File; + +namespace Wabbajack.Common +{ + public class TempStream : FileStream + { + private TempFile _file; + + public TempStream(TempFile file) : base(file.File.FullName, FileMode.Create, FileAccess.ReadWrite) + { + _file = file; + } + + public TempStream() : this(new TempFile()) + { + + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + _file.Dispose(); + } + + public override async ValueTask DisposeAsync() + { + await base.DisposeAsync(); + _file.Dispose(); + } + } +} diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 65abad2b..e666aa40 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.IO; using System.Linq; using System.Net.Http; +using System.Net.Http.Headers; using System.Reactive.Linq; using System.Reactive.Subjects; using System.Reflection; @@ -958,6 +959,7 @@ namespace Wabbajack.Common RETRY: try { + File.Move(tmpName, cacheFile, MoveOptions.ReplaceExisting); } catch (UnauthorizedAccessException) @@ -975,6 +977,35 @@ namespace Wabbajack.Common } } + public static async Task CreatePatch(FileStream srcStream, string srcHash, FileStream destStream, string destHash, + FileStream patchStream) + { + await using var sigFile = new TempStream(); + OctoDiff.Create(srcStream, destStream, sigFile, patchStream); + patchStream.Position = 0; + var tmpName = Path.Combine(Consts.PatchCacheFolder, Guid.NewGuid() + ".tmp"); + + await using (var f = File.Create(tmpName)) + { + await patchStream.CopyToAsync(f); + patchStream.Position = 0; + } + + try + { + var cacheFile = Path.Combine(Consts.PatchCacheFolder, $"{srcHash.FromBase64().ToHex()}_{srcHash.FromBase64().ToHex()}.patch"); + if (!Directory.Exists(Consts.PatchCacheFolder)) + Directory.CreateDirectory(Consts.PatchCacheFolder); + + File.Move(tmpName, cacheFile, MoveOptions.ReplaceExisting); + } + catch (UnauthorizedAccessException) + { + if (File.Exists(tmpName)) + File.Delete(tmpName); + } + } + public static bool TryGetPatch(string foundHash, string fileHash, out byte[] ePatch) { var patchName = Path.Combine(Consts.PatchCacheFolder, diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index a41985d7..f389ec69 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -71,6 +71,12 @@ namespace Wabbajack.Lib return id; } + internal FileStream IncludeFile(out string id) + { + id = Guid.NewGuid().ToString(); + return File.Create(Path.Combine(ModListOutputFolder, id)); + } + internal string IncludeFile(string data) { var id = Guid.NewGuid().ToString(); diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 6a99f918..62fdbe46 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -184,6 +184,7 @@ namespace Wabbajack.Lib { gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories) .Where(p => p.FileExists()) + .Where(p => Path.GetExtension(p) != Consts.HashFileExtension) .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)))); } @@ -451,22 +452,20 @@ namespace Wabbajack.Lib { Info($"Patching {entry.To}"); Status($"Patching {entry.To}"); - await using var origin = byPath[string.Join("|", entry.ArchiveHashPath.Skip(1))].OpenRead(); - await using var output = new MemoryStream(); - var a = origin.ReadAll(); - var b = LoadDataForTo(entry.To, absolutePaths); - await Utils.CreatePatch(a, b, output); - entry.PatchID = IncludeFile(output.ToArray()); - var fileSize = File.GetSize(Path.Combine(ModListOutputFolder, entry.PatchID)); - Info($"Patch size {fileSize} for {entry.To}"); + var srcFile = byPath[string.Join("|", entry.ArchiveHashPath.Skip(1))]; + await using var srcStream = srcFile.OpenRead(); + await using var outputStream = IncludeFile(out entry.PatchID); + await using var destStream = LoadDataForTo(entry.To, absolutePaths); + await Utils.CreatePatch(srcStream, srcFile.Hash, destStream, entry.Hash, outputStream); + Info($"Patch size {outputStream.Length} for {entry.To}"); }); } } - private byte[] LoadDataForTo(string to, Dictionary absolutePaths) + private FileStream LoadDataForTo(string to, Dictionary absolutePaths) { if (absolutePaths.TryGetValue(to, out var absolute)) - return File.ReadAllBytes(absolute); + return File.OpenRead(absolute); if (to.StartsWith(Consts.BSACreationDir)) { @@ -477,11 +476,10 @@ namespace Wabbajack.Lib { var find = Path.Combine(to.Split('\\').Skip(2).ToArray()); var file = a.Files.First(e => e.Path.Replace('/', '\\') == find); - using (var ms = new MemoryStream()) - { - file.CopyDataTo(ms); - return ms.ToArray(); - } + var returnStream = new TempStream(); + file.CopyDataTo(returnStream); + returnStream.Position = 0; + return returnStream; } } @@ -521,6 +519,7 @@ namespace Wabbajack.Lib new IgnoreStartsWith(this, "downloads\\"), new IgnoreStartsWith(this,"webcache\\"), new IgnoreStartsWith(this, "overwrite\\"), + new IgnoreStartsWith(this, "crashDumps\\"), new IgnorePathContains(this,"temporary_logs"), new IgnorePathContains(this, "GPUCache"), new IgnorePathContains(this, "SSEEdit Cache"), diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index db0239ea..17a98bb6 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -335,7 +335,7 @@ namespace Wabbajack.VirtualFileSystem return path; } - public Stream OpenRead() + public FileStream OpenRead() { return File.OpenRead(StagedPath); }