Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #601 from wabbajack-tools/workbase-fixes

Browse Source 
Fixes for WorkBase
This commit is contained in:
Timothy Baldridge 2020-03-04 05:11:48 -07:00 committed by GitHub
commit fe46970e9e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 98 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -1,6 +1,9 @@
### Changelog
#### Version - 1.0.1.0 - 3/xx/2020
#### Version - Next
* Binary Patching stores temporary and patch data on disk instead of memory (reducing memory usage)
* Fix a memory leak with diffing progress reporting
* Fix a bug with bad data in inferred game INI files.
* Added download support for YouTube
* Slideshow can now display mods from non-Nexus sites

7
Wabbajack.Common/DynamicIniData.cs
View File

@ -76,7 +76,9 @@ namespace Wabbajack.Common
private static string UnescapeString(string s)
{
return Regex.Unescape(s.Trim('"'));
if (s.Trim().StartsWith("\"") || s.Contains("\\\\"))
return Regex.Unescape(s.Trim('"'));
return s;
}
private static string UnescapeUTF8(string s)
@ -121,7 +123,8 @@ namespace Wabbajack.Common
}
result = Coll[(string) indexes[0]];
if (result is string s) result = Regex.Unescape(s.Trim('"'));
if (result is string s && s.Trim().StartsWith("\""))
result = Regex.Unescape(s.Trim('"'));
return true;
}
}

4
Wabbajack.Common/OctoDiff.cs
View File

@ -45,8 +45,12 @@ namespace Wabbajack.Common
private class ProgressReporter : IProgressReporter
{
private DateTime _lastUpdate = DateTime.UnixEpoch;
private readonly TimeSpan _updateInterval = TimeSpan.FromMilliseconds(100);
public void ReportProgress(string operation, long currentPosition, long total)
{
if (DateTime.Now - _lastUpdate < _updateInterval) return;
_lastUpdate = DateTime.Now;
if (currentPosition >= total || total < 1 || currentPosition < 0)
return;
Utils.Status(operation, new Percent(total, currentPosition));

33
Wabbajack.Common/Util/TempStream.cs Normal file
View File

@ -0,0 +1,33 @@
using System.IO;
using System.Threading.Tasks;
using File = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.Common
{
public class TempStream : FileStream
{
private TempFile _file;
public TempStream(TempFile file) : base(file.File.FullName, FileMode.Create, FileAccess.ReadWrite)
{
_file = file;
}
public TempStream() : this(new TempFile())
{
}
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
_file.Dispose();
}
public override async ValueTask DisposeAsync()
{
await base.DisposeAsync();
_file.Dispose();
}
}
}

31
Wabbajack.Common/Utils.cs
View File

@ -6,6 +6,7 @@ using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Reflection;
@ -958,6 +959,7 @@ namespace Wabbajack.Common
RETRY:
try
{
File.Move(tmpName, cacheFile, MoveOptions.ReplaceExisting);
}
catch (UnauthorizedAccessException)
@ -975,6 +977,35 @@ namespace Wabbajack.Common
}
}
public static async Task CreatePatch(FileStream srcStream, string srcHash, FileStream destStream, string destHash,
FileStream patchStream)
{
await using var sigFile = new TempStream();
OctoDiff.Create(srcStream, destStream, sigFile, patchStream);
patchStream.Position = 0;
var tmpName = Path.Combine(Consts.PatchCacheFolder, Guid.NewGuid() + ".tmp");
await using (var f = File.Create(tmpName))
{
await patchStream.CopyToAsync(f);
patchStream.Position = 0;
}
try
{
var cacheFile = Path.Combine(Consts.PatchCacheFolder, $"{srcHash.FromBase64().ToHex()}_{srcHash.FromBase64().ToHex()}.patch");
if (!Directory.Exists(Consts.PatchCacheFolder))
Directory.CreateDirectory(Consts.PatchCacheFolder);
File.Move(tmpName, cacheFile, MoveOptions.ReplaceExisting);
}
catch (UnauthorizedAccessException)
{
if (File.Exists(tmpName))
File.Delete(tmpName);
}
}
public static bool TryGetPatch(string foundHash, string fileHash, out byte[] ePatch)
{
var patchName = Path.Combine(Consts.PatchCacheFolder,

6
Wabbajack.Lib/ACompiler.cs
View File

@ -71,6 +71,12 @@ namespace Wabbajack.Lib
return id;
}
internal FileStream IncludeFile(out string id)
{
id = Guid.NewGuid().ToString();
return File.Create(Path.Combine(ModListOutputFolder, id));
}
internal string IncludeFile(string data)
{
var id = Guid.NewGuid().ToString();

29
Wabbajack.Lib/MO2Compiler.cs
View File

@ -184,6 +184,7 @@ namespace Wabbajack.Lib
{
gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Where(p => Path.GetExtension(p) != Consts.HashFileExtension)
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p],
Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath))));
}
@ -451,22 +452,20 @@ namespace Wabbajack.Lib
{
Info($"Patching {entry.To}");
Status($"Patching {entry.To}");
await using var origin = byPath[string.Join("|", entry.ArchiveHashPath.Skip(1))].OpenRead();
await using var output = new MemoryStream();
var a = origin.ReadAll();
var b = LoadDataForTo(entry.To, absolutePaths);
await Utils.CreatePatch(a, b, output);
entry.PatchID = IncludeFile(output.ToArray());
var fileSize = File.GetSize(Path.Combine(ModListOutputFolder, entry.PatchID));
Info($"Patch size {fileSize} for {entry.To}");
var srcFile = byPath[string.Join("|", entry.ArchiveHashPath.Skip(1))];
await using var srcStream = srcFile.OpenRead();
await using var outputStream = IncludeFile(out entry.PatchID);
await using var destStream = LoadDataForTo(entry.To, absolutePaths);
await Utils.CreatePatch(srcStream, srcFile.Hash, destStream, entry.Hash, outputStream);
Info($"Patch size {outputStream.Length} for {entry.To}");
});
}
}
private byte[] LoadDataForTo(string to, Dictionary<string, string> absolutePaths)
private FileStream LoadDataForTo(string to, Dictionary<string, string> absolutePaths)
{
if (absolutePaths.TryGetValue(to, out var absolute))
return File.ReadAllBytes(absolute);
return File.OpenRead(absolute);
if (to.StartsWith(Consts.BSACreationDir))
{
@ -477,11 +476,10 @@ namespace Wabbajack.Lib
{
var find = Path.Combine(to.Split('\\').Skip(2).ToArray());
var file = a.Files.First(e => e.Path.Replace('/', '\\') == find);
using (var ms = new MemoryStream())
{
file.CopyDataTo(ms);
return ms.ToArray();
}
var returnStream = new TempStream();
file.CopyDataTo(returnStream);
returnStream.Position = 0;
return returnStream;
}
}
@ -521,6 +519,7 @@ namespace Wabbajack.Lib
new IgnoreStartsWith(this, "downloads\\"),
new IgnoreStartsWith(this,"webcache\\"),
new IgnoreStartsWith(this, "overwrite\\"),
new IgnoreStartsWith(this, "crashDumps\\"),
new IgnorePathContains(this,"temporary_logs"),
new IgnorePathContains(this, "GPUCache"),
new IgnorePathContains(this, "SSEEdit Cache"),

2
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -335,7 +335,7 @@ namespace Wabbajack.VirtualFileSystem
return path;
}
public Stream OpenRead()
public FileStream OpenRead()
{
return File.OpenRead(StagedPath);
}