Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Several tricks to get memory usage under control during list validation.

Browse Source
This commit is contained in:
Timothy Baldridge 2023-10-09 14:25:10 -06:00
parent 63ca2f7c38
commit 0cba392b66
3 changed files with 19 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Wabbajack.CLI/Verbs/ValidateLists.cs
View File

@ -53,6 +53,8 @@ public class ValidateLists
private readonly Networking.WabbajackClientApi.Client _wjClient;
private readonly HttpClient _httpClient;
private readonly IResource<HttpClient> _httpLimiter;
private readonly AsyncLock _imageProcessLock;
public ValidateLists(ILogger<ValidateLists> logger, Networking.WabbajackClientApi.Client wjClient,
Client gitHubClient, TemporaryFileManager temporaryFileManager,
@ -73,6 +75,7 @@ public class ValidateLists
_random = new Random();
_httpClient = httpClient;
_httpLimiter = httpLimiter;
_imageProcessLock = new AsyncLock();
}
public static VerbDefinition Definition = new("validate-lists",
@ -139,6 +142,9 @@ public class ValidateLists
_logger.LogInformation("Loading Modlist");
modListData =
await StandardInstaller.Load(_dtos, _dispatcher, modList, token);
// Clear out the directives to save memory
modListData.Directives = Array.Empty<Directive>();
GC.Collect();
}
catch (Exception ex)
{
@ -147,7 +153,9 @@ public class ValidateLists
return validatedList;
}
_logger.LogInformation("Verifying {Count} archives", modListData.Archives.Length);
_logger.LogInformation("Verifying {Count} archives from {Name}", modListData.Archives.Length, modList.NamespacedName);
var archives = await modListData.Archives.PMapAll(async archive =>
{
@ -248,7 +256,6 @@ public class ValidateLists
await ExportReports(reports, validatedLists, token);
var usedMirroredFiles = validatedLists.SelectMany(a => a.Archives)
.Where(m => m.Status == ArchiveStatus.Mirrored)
.Select(m => m.Original.Hash)
@ -261,6 +268,7 @@ public class ValidateLists
private async Task<(RelativePath SmallImage, RelativePath LargeImage)> ProcessModlistImage(AbsolutePath reports, ModlistMetadata validatedList,
CancellationToken token)
{
using var _ = await _imageProcessLock.WaitAsync();
_logger.LogInformation("Processing Modlist Image for {MachineUrl}", validatedList.NamespacedName);
var baseFolder = reports.Combine(validatedList.NamespacedName);
baseFolder.CreateDirectory();
@ -401,6 +409,7 @@ public class ValidateLists
"skyrimvr.ini",
}.Select(f => f.ToRelativePath()).ToHashSet();
private async Task<Dictionary<RelativePath, byte[]>> GetFiles(ModList modlist, ModlistMetadata metadata, CancellationToken token)
{
var archive = new Archive

4
Wabbajack.Installer/StandardInstaller.cs
View File

@ -587,10 +587,10 @@ public class StandardInstaller : AInstaller<StandardInstaller>
Hash = metadata.DownloadMetadata.Hash
};
var stream = await dispatcher.ChunkedSeekableStream(archive, token);
await using var stream = await dispatcher.ChunkedSeekableStream(archive, token);
await using var reader = new ZipReader(stream);
var entry = (await reader.GetFiles()).First(e => e.FileName == "modlist");
var ms = new MemoryStream();
using var ms = new MemoryStream();
await reader.Extract(entry, ms, token);
ms.Position = 0;
return JsonSerializer.Deserialize<ModList>(ms, dtos.Options)!;

6
Wabbajack.Networking.Http/AChunkedBufferingStream.cs
View File

@ -24,6 +24,12 @@ public abstract class AChunkedBufferingStream : Stream
_chunks = new Dictionary<ulong, byte[]>();
}
protected override void Dispose(bool disposing)
{
_chunks.Clear();
base.Dispose(disposing);
}
public abstract Task<byte[]> LoadChunk(long offset, int size);
public override void Flush()