Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Optimize the compilation process

Browse Source
This commit is contained in:
Timothy Baldridge 2021-07-05 15:26:30 -06:00
parent 993b3f63c3
commit 5370d19a8d
2 changed files with 39 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -345,16 +345,27 @@ namespace Wabbajack.VirtualFileSystem
await _unstage(); await _unstage();
} }
} }
public static class EmptyLookup<TKey, TElement>
{
private static readonly ILookup<TKey, TElement> _instance
= Enumerable.Empty<TElement>().ToLookup(x => default(TKey));
public static ILookup<TKey, TElement> Instance
{
get { return _instance; }
}
}
public class IndexRoot public class IndexRoot
{ {
public static IndexRoot Empty = new IndexRoot(); public static IndexRoot Empty = new IndexRoot();
public IndexRoot(ImmutableList<VirtualFile> aFiles, public IndexRoot(IReadOnlyList<VirtualFile> aFiles,
Dictionary<FullPath, VirtualFile> byFullPath, IDictionary<FullPath, VirtualFile> byFullPath,
ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> byHash, ILookup<Hash, VirtualFile> byHash,
ImmutableDictionary<AbsolutePath, VirtualFile> byRoot, IDictionary<AbsolutePath, VirtualFile> byRoot,
ImmutableDictionary<IPath, ImmutableStack<VirtualFile>> byName) ILookup<IPath, VirtualFile> byName)
{ {
AllFiles = aFiles; AllFiles = aFiles;
ByFullPath = byFullPath; ByFullPath = byFullPath;
@ -367,17 +378,17 @@ namespace Wabbajack.VirtualFileSystem
{ {
AllFiles = ImmutableList<VirtualFile>.Empty; AllFiles = ImmutableList<VirtualFile>.Empty;
ByFullPath = new Dictionary<FullPath, VirtualFile>(); ByFullPath = new Dictionary<FullPath, VirtualFile>();
ByHash = ImmutableDictionary<Hash, ImmutableStack<VirtualFile>>.Empty; ByHash = EmptyLookup<Hash, VirtualFile>.Instance;
ByRootPath = ImmutableDictionary<AbsolutePath, VirtualFile>.Empty; ByRootPath = new Dictionary<AbsolutePath, VirtualFile>();
ByName = ImmutableDictionary<IPath, ImmutableStack<VirtualFile>>.Empty; ByName = EmptyLookup<IPath, VirtualFile>.Instance;
} }
public ImmutableList<VirtualFile> AllFiles { get; } public IReadOnlyList<VirtualFile> AllFiles { get; }
public Dictionary<FullPath, VirtualFile> ByFullPath { get; } public IDictionary<FullPath, VirtualFile> ByFullPath { get; }
public ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> ByHash { get; } public ILookup<Hash, VirtualFile> ByHash { get; }
public ImmutableDictionary<IPath, ImmutableStack<VirtualFile>> ByName { get; set; } public ILookup<IPath, VirtualFile> ByName { get; set; }
public ImmutableDictionary<AbsolutePath, VirtualFile> ByRootPath { get; } public IDictionary<AbsolutePath, VirtualFile> ByRootPath { get; }
public async Task<IndexRoot> Integrate(ICollection<VirtualFile> files) public async Task<IndexRoot> Integrate(ICollection<VirtualFile> files)
{ {
@ -385,19 +396,19 @@ namespace Wabbajack.VirtualFileSystem
var allFiles = AllFiles.Concat(files) var allFiles = AllFiles.Concat(files)
.OrderByDescending(f => f.LastModified) .OrderByDescending(f => f.LastModified)
.GroupBy(f => f.FullPath).Select(g => g.Last()) .GroupBy(f => f.FullPath).Select(g => g.Last())
.ToImmutableList(); .ToList();
var byFullPath = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) var byFullPath = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToDictionary(f => f.FullPath)); .ToDictionary(f => f.FullPath));
var byHash = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) var byHash = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.Where(f => f.Hash != Hash.Empty) .Where(f => f.Hash != Hash.Empty)
.ToGroupedImmutableDictionary(f => f.Hash)); .ToLookup(f => f.Hash));
var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToGroupedImmutableDictionary(f => f.Name)); .ToLookup(f => f.Name));
var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.AbsoluteName)); var byRootPath = Task.Run(() => allFiles.ToDictionary(f => f.AbsoluteName));
var result = new IndexRoot(allFiles, var result = new IndexRoot(allFiles,
await byFullPath, await byFullPath,

13
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -235,7 +235,11 @@ namespace Wabbajack.VirtualFileSystem
self.ExtendedHashes = await ExtendedHashes.FromStream(stream); self.ExtendedHashes = await ExtendedHashes.FromStream(stream);
// Can't extract, so return // Can't extract, so return
if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension)) return self; if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension))
{
await WriteToCache(self);
return self;
}
try try
{ {
@ -256,11 +260,16 @@ namespace Wabbajack.VirtualFileSystem
throw; throw;
} }
await WriteToCache(self);
return self;
}
private static async Task WriteToCache(VirtualFile self)
{
await using var ms = new MemoryStream(); await using var ms = new MemoryStream();
self.ToIndexedVirtualFile().Write(ms); self.ToIndexedVirtualFile().Write(ms);
ms.Position = 0; ms.Position = 0;
await InsertIntoVFSCache(self.Hash, ms); await InsertIntoVFSCache(self.Hash, ms);
return self;
} }
private static async Task InsertIntoVFSCache(Hash hash, MemoryStream data) private static async Task InsertIntoVFSCache(Hash hash, MemoryStream data)