Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reduce memory used by open file streams

Browse Source 
Allocating 1 MB for each file stream was causing memory usage
during the install of certain lists to exceed 60 GB.  This is
primarily due to BSA/BA2 creation where tens of thousands of files
are open at the same time.  Additionally, there appears to be a
couple file stream leaks where file streams are not closed until
the entire program is closed.

There appears to still be some discussion on how this affects
performance and a proper fix.  This is more of a dirty "make it work
for more users" fix so people can stop messing with their page files.
This commit is contained in:
Chris Bessent 2021-01-18 06:13:14 -07:00
parent 4a32fc1415
commit e2d5a42254
1 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Wabbajack.Common/Paths/AbsolutePath.cs
View File

@ -177,7 +177,7 @@ namespace Wabbajack.Common
{ {
if (otherPath.Exists && overwrite) if (otherPath.Exists && overwrite)
await otherPath.DeleteAsync(); await otherPath.DeleteAsync();
await CopyToAsync(otherPath); await CopyToAsync(otherPath);
await DeleteAsync(); await DeleteAsync();
return; return;
@ -190,7 +190,7 @@ namespace Wabbajack.Common
public RelativePath RelativeTo(AbsolutePath p) public RelativePath RelativeTo(AbsolutePath p)
{ {
var relPath = Path.GetRelativePath(p._path, _path); var relPath = Path.GetRelativePath(p._path, _path);
if (relPath == _path) if (relPath == _path)
throw new ArgumentException($"{_path} is not a subpath of {p._path}"); throw new ArgumentException($"{_path} is not a subpath of {p._path}");
return new RelativePath(relPath); return new RelativePath(relPath);
} }
@ -262,7 +262,7 @@ namespace Wabbajack.Common
// ignore, it doesn't exist so why delete it? // ignore, it doesn't exist so why delete it?
} }
} }
public void Delete() public void Delete()
{ {
if (!IsFile) return; if (!IsFile) return;
@ -310,7 +310,7 @@ namespace Wabbajack.Common
public AbsolutePath Combine(params string[] paths) public AbsolutePath Combine(params string[] paths)
{ {
return new AbsolutePath(Path.Combine(paths.Cons(_path).ToArray())); return new AbsolutePath(Path.Combine(paths.Cons(_path).ToArray()));
} }
@ -344,9 +344,9 @@ namespace Wabbajack.Common
public async ValueTask HardLinkIfOversize(AbsolutePath destination) public async ValueTask HardLinkIfOversize(AbsolutePath destination)
{ {
if (!destination.Parent.Exists) if (!destination.Parent.Exists)
destination.Parent.CreateDirectory(); destination.Parent.CreateDirectory();
if (Root == destination.Root && Consts.SupportedBSAs.Contains(Extension)) if (Root == destination.Root && Consts.SupportedBSAs.Contains(Extension))
{ {
if (HardLinkTo(destination)) if (HardLinkTo(destination))
@ -398,14 +398,14 @@ namespace Wabbajack.Common
{ {
var path = _path; var path = _path;
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () => return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () =>
File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, bufferSize: 1048576, useAsync: true)); File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, bufferSize: 4096, useAsync: true));
} }
public ValueTask<FileStream> WriteShared() public ValueTask<FileStream> WriteShared()
{ {
var path = _path; var path = _path;
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () => return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () =>
File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite, bufferSize: 1048576, useAsync: true)); File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite, bufferSize: 4096, useAsync: true));
} }
public async Task CopyDirectoryToAsync(AbsolutePath destination) public async Task CopyDirectoryToAsync(AbsolutePath destination)