Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into pre-release

Browse Source
This commit is contained in:
EzioTheDeadPoet 2023-10-12 19:23:35 +02:00
commit a16f808213
3 changed files with 19 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Wabbajack.CLI/Verbs/ValidateLists.cs
View File

@ -53,6 +53,8 @@ public class ValidateLists
private readonly Networking.WabbajackClientApi.Client _wjClient; private readonly Networking.WabbajackClientApi.Client _wjClient;
private readonly HttpClient _httpClient; private readonly HttpClient _httpClient;
private readonly IResource<HttpClient> _httpLimiter; private readonly IResource<HttpClient> _httpLimiter;
private readonly AsyncLock _imageProcessLock;
public ValidateLists(ILogger<ValidateLists> logger, Networking.WabbajackClientApi.Client wjClient, public ValidateLists(ILogger<ValidateLists> logger, Networking.WabbajackClientApi.Client wjClient,
Client gitHubClient, TemporaryFileManager temporaryFileManager, Client gitHubClient, TemporaryFileManager temporaryFileManager,
@ -73,6 +75,7 @@ public class ValidateLists
_random = new Random(); _random = new Random();
_httpClient = httpClient; _httpClient = httpClient;
_httpLimiter = httpLimiter; _httpLimiter = httpLimiter;
_imageProcessLock = new AsyncLock();
} }
public static VerbDefinition Definition = new("validate-lists", public static VerbDefinition Definition = new("validate-lists",
@ -139,6 +142,9 @@ public class ValidateLists
_logger.LogInformation("Loading Modlist"); _logger.LogInformation("Loading Modlist");
modListData = modListData =
await StandardInstaller.Load(_dtos, _dispatcher, modList, token); await StandardInstaller.Load(_dtos, _dispatcher, modList, token);
// Clear out the directives to save memory
modListData.Directives = Array.Empty<Directive>();
GC.Collect();
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -147,7 +153,9 @@ public class ValidateLists
return validatedList; return validatedList;
} }
_logger.LogInformation("Verifying {Count} archives", modListData.Archives.Length);
_logger.LogInformation("Verifying {Count} archives from {Name}", modListData.Archives.Length, modList.NamespacedName);
var archives = await modListData.Archives.PMapAll(async archive => var archives = await modListData.Archives.PMapAll(async archive =>
{ {
@ -248,7 +256,6 @@ public class ValidateLists
await ExportReports(reports, validatedLists, token); await ExportReports(reports, validatedLists, token);
var usedMirroredFiles = validatedLists.SelectMany(a => a.Archives) var usedMirroredFiles = validatedLists.SelectMany(a => a.Archives)
.Where(m => m.Status == ArchiveStatus.Mirrored) .Where(m => m.Status == ArchiveStatus.Mirrored)
.Select(m => m.Original.Hash) .Select(m => m.Original.Hash)
@ -261,6 +268,7 @@ public class ValidateLists
private async Task<(RelativePath SmallImage, RelativePath LargeImage)> ProcessModlistImage(AbsolutePath reports, ModlistMetadata validatedList, private async Task<(RelativePath SmallImage, RelativePath LargeImage)> ProcessModlistImage(AbsolutePath reports, ModlistMetadata validatedList,
CancellationToken token) CancellationToken token)
{ {
using var _ = await _imageProcessLock.WaitAsync();
_logger.LogInformation("Processing Modlist Image for {MachineUrl}", validatedList.NamespacedName); _logger.LogInformation("Processing Modlist Image for {MachineUrl}", validatedList.NamespacedName);
var baseFolder = reports.Combine(validatedList.NamespacedName); var baseFolder = reports.Combine(validatedList.NamespacedName);
baseFolder.CreateDirectory(); baseFolder.CreateDirectory();
@ -401,6 +409,7 @@ public class ValidateLists
"skyrimvr.ini", "skyrimvr.ini",
}.Select(f => f.ToRelativePath()).ToHashSet(); }.Select(f => f.ToRelativePath()).ToHashSet();
private async Task<Dictionary<RelativePath, byte[]>> GetFiles(ModList modlist, ModlistMetadata metadata, CancellationToken token) private async Task<Dictionary<RelativePath, byte[]>> GetFiles(ModList modlist, ModlistMetadata metadata, CancellationToken token)
{ {
var archive = new Archive var archive = new Archive

4
Wabbajack.Installer/StandardInstaller.cs
View File

@ -587,10 +587,10 @@ public class StandardInstaller : AInstaller<StandardInstaller>
Hash = metadata.DownloadMetadata.Hash Hash = metadata.DownloadMetadata.Hash
}; };
var stream = await dispatcher.ChunkedSeekableStream(archive, token); await using var stream = await dispatcher.ChunkedSeekableStream(archive, token);
await using var reader = new ZipReader(stream); await using var reader = new ZipReader(stream);
var entry = (await reader.GetFiles()).First(e => e.FileName == "modlist"); var entry = (await reader.GetFiles()).First(e => e.FileName == "modlist");
var ms = new MemoryStream(); using var ms = new MemoryStream();
await reader.Extract(entry, ms, token); await reader.Extract(entry, ms, token);
ms.Position = 0; ms.Position = 0;
return JsonSerializer.Deserialize<ModList>(ms, dtos.Options)!; return JsonSerializer.Deserialize<ModList>(ms, dtos.Options)!;

6
Wabbajack.Networking.Http/AChunkedBufferingStream.cs
View File

@ -24,6 +24,12 @@ public abstract class AChunkedBufferingStream : Stream
_chunks = new Dictionary<ulong, byte[]>(); _chunks = new Dictionary<ulong, byte[]>();
} }
protected override void Dispose(bool disposing)
{
_chunks.Clear();
base.Dispose(disposing);
}
public abstract Task<byte[]> LoadChunk(long offset, int size); public abstract Task<byte[]> LoadChunk(long offset, int size);
public override void Flush() public override void Flush()