From 5370d19a8de705363e926138bab03355f5cadd52 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 5 Jul 2021 15:26:30 -0600 Subject: [PATCH 1/3] Optimize the compilation process --- Wabbajack.VirtualFileSystem/Context.cs | 45 ++++++++++++++-------- Wabbajack.VirtualFileSystem/VirtualFile.cs | 13 ++++++- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index 2d24b797..b30c48e7 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -345,16 +345,27 @@ namespace Wabbajack.VirtualFileSystem await _unstage(); } } + + public static class EmptyLookup + { + private static readonly ILookup _instance + = Enumerable.Empty().ToLookup(x => default(TKey)); + + public static ILookup Instance + { + get { return _instance; } + } + } public class IndexRoot { public static IndexRoot Empty = new IndexRoot(); - public IndexRoot(ImmutableList aFiles, - Dictionary byFullPath, - ImmutableDictionary> byHash, - ImmutableDictionary byRoot, - ImmutableDictionary> byName) + public IndexRoot(IReadOnlyList aFiles, + IDictionary byFullPath, + ILookup byHash, + IDictionary byRoot, + ILookup byName) { AllFiles = aFiles; ByFullPath = byFullPath; @@ -367,17 +378,17 @@ namespace Wabbajack.VirtualFileSystem { AllFiles = ImmutableList.Empty; ByFullPath = new Dictionary(); - ByHash = ImmutableDictionary>.Empty; - ByRootPath = ImmutableDictionary.Empty; - ByName = ImmutableDictionary>.Empty; + ByHash = EmptyLookup.Instance; + ByRootPath = new Dictionary(); + ByName = EmptyLookup.Instance; } - public ImmutableList AllFiles { get; } - public Dictionary ByFullPath { get; } - public ImmutableDictionary> ByHash { get; } - public ImmutableDictionary> ByName { get; set; } - public ImmutableDictionary ByRootPath { get; } + public IReadOnlyList AllFiles { get; } + public IDictionary ByFullPath { get; } + public ILookup ByHash { get; } + public ILookup ByName { get; set; } + public IDictionary ByRootPath { get; } public async Task Integrate(ICollection files) { @@ -385,19 +396,19 @@ namespace Wabbajack.VirtualFileSystem var allFiles = AllFiles.Concat(files) .OrderByDescending(f => f.LastModified) .GroupBy(f => f.FullPath).Select(g => g.Last()) - .ToImmutableList(); + .ToList(); var byFullPath = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) .ToDictionary(f => f.FullPath)); var byHash = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) .Where(f => f.Hash != Hash.Empty) - .ToGroupedImmutableDictionary(f => f.Hash)); + .ToLookup(f => f.Hash)); var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) - .ToGroupedImmutableDictionary(f => f.Name)); + .ToLookup(f => f.Name)); - var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.AbsoluteName)); + var byRootPath = Task.Run(() => allFiles.ToDictionary(f => f.AbsoluteName)); var result = new IndexRoot(allFiles, await byFullPath, diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index c6a6f933..0dcc1459 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -235,7 +235,11 @@ namespace Wabbajack.VirtualFileSystem self.ExtendedHashes = await ExtendedHashes.FromStream(stream); // Can't extract, so return - if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension)) return self; + if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension)) + { + await WriteToCache(self); + return self; + } try { @@ -256,11 +260,16 @@ namespace Wabbajack.VirtualFileSystem throw; } + await WriteToCache(self); + return self; + } + + private static async Task WriteToCache(VirtualFile self) + { await using var ms = new MemoryStream(); self.ToIndexedVirtualFile().Write(ms); ms.Position = 0; await InsertIntoVFSCache(self.Hash, ms); - return self; } private static async Task InsertIntoVFSCache(Hash hash, MemoryStream data) From 2c490f50e7df053ec67e5410e9afb51ee4c4ba5a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 5 Jul 2021 23:26:08 -0600 Subject: [PATCH 2/3] Cache the last modified date in NativeFileStreamFactory --- Wabbajack.Common/IStreamFactory.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Wabbajack.Common/IStreamFactory.cs b/Wabbajack.Common/IStreamFactory.cs index 7dc4cc4d..6ca1c460 100644 --- a/Wabbajack.Common/IStreamFactory.cs +++ b/Wabbajack.Common/IStreamFactory.cs @@ -34,7 +34,16 @@ namespace Wabbajack.Common return await _file.OpenRead(); } - public DateTime LastModifiedUtc => _file.LastModifiedUtc; + private DateTime? _lastModifiedCache = null; + public DateTime LastModifiedUtc + { + get + { + _lastModifiedCache ??= _file.LastModifiedUtc; + return _lastModifiedCache.Value; + } + } + public IPath Name { get; } } From e08108fb46b385a299a48d189a825c166941517d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 6 Jul 2021 05:49:04 -0600 Subject: [PATCH 3/3] 2.5.1.0 --- CHANGELOG.md | 3 +++ Wabbajack.CLI/Wabbajack.CLI.csproj | 4 ++-- Wabbajack.Launcher/Wabbajack.Launcher.csproj | 4 ++-- Wabbajack.Server/Wabbajack.Server.csproj | 4 ++-- Wabbajack.Test/NexusTests.cs | 4 +++- Wabbajack/Wabbajack.csproj | 4 ++-- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c95097b8..fd0c8a28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ### Changelog +#### Version - 2.5.1.0 - 7/6/2021 +* Drastically improve compilation times by the removal of several bugs in the compiler + #### Version - 2.5.0.9 - 7/4/2021 * Use DX11 GPUs for compression when possible * Don't threadbomb the system by creating O(N*M) threads when compressing textures diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 0baa3be1..d3d16e49 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -6,8 +6,8 @@ wabbajack-cli Wabbajack x64 - 2.5.0.9 - 2.5.0.9 + 2.5.1.0 + 2.5.1.0 Copyright © 2019-2020 An automated ModList installer true diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj index a1450b23..d9dfd4e7 100644 --- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj +++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj @@ -4,8 +4,8 @@ Exe net5.0-windows true - 2.5.0.9 - 2.5.0.9 + 2.5.1.0 + 2.5.1.0 Copyright © 2019-2020 Wabbajack Application Launcher true diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index 6d114b89..342f69b9 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -3,8 +3,8 @@ Exe net5.0-windows - 2.5.0.9 - 2.5.0.9 + 2.5.1.0 + 2.5.1.0 Copyright © 2019-2020 Wabbajack Server win-x64 diff --git a/Wabbajack.Test/NexusTests.cs b/Wabbajack.Test/NexusTests.cs index 1e59e9fc..8d04cdcd 100644 --- a/Wabbajack.Test/NexusTests.cs +++ b/Wabbajack.Test/NexusTests.cs @@ -10,6 +10,8 @@ namespace Wabbajack.Test { public class NexusTests : ATestBase { + // Disable for now + /* [Fact] public async Task CanGetNexusRSSUpdates() { @@ -21,7 +23,7 @@ namespace Wabbajack.Test { Assert.True(DateTime.UtcNow - result.TimeStamp < TimeSpan.FromDays(1)); } - } + }*/ public NexusTests(ITestOutputHelper output) : base(output) { diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 2ead3249..f7876329 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -6,8 +6,8 @@ true x64 win10-x64 - 2.5.0.9 - 2.5.0.9 + 2.5.1.0 + 2.5.1.0 Copyright © 2019-2020 An automated ModList installer true