Merge pull request #1530 from wabbajack-tools/optimize-compilation

Optimize the compilation process

CHANGELOG.md

@@ -1,5 +1,8 @@
 ### Changelog
 
+#### Version - 2.5.1.0 - 7/6/2021
+* Drastically improve compilation times by the removal of several bugs in the compiler
+
 #### Version - 2.5.0.9 - 7/4/2021
 * Use DX11 GPUs for compression when possible
 * Don't threadbomb the system by creating O(N*M) threads when compressing textures

Wabbajack.CLI/Wabbajack.CLI.csproj

@@ -6,8 +6,8 @@
         <AssemblyName>wabbajack-cli</AssemblyName>
         <Company>Wabbajack</Company>
         <Platforms>x64</Platforms>
-        <AssemblyVersion>2.5.0.9</AssemblyVersion>
+        <AssemblyVersion>2.5.1.0</AssemblyVersion>
-        <FileVersion>2.5.0.9</FileVersion>
+        <FileVersion>2.5.1.0</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>An automated ModList installer</Description>
         <PublishReadyToRun>true</PublishReadyToRun>        

Wabbajack.Common/IStreamFactory.cs

@@ -34,7 +34,16 @@ namespace Wabbajack.Common
             return await _file.OpenRead();
         }
 
-        public DateTime LastModifiedUtc => _file.LastModifiedUtc;
+        private DateTime? _lastModifiedCache = null;
+        public DateTime LastModifiedUtc
+        {
+            get
+            {
+                _lastModifiedCache ??= _file.LastModifiedUtc;
+                return _lastModifiedCache.Value;
+            }
+        }
+
         public IPath Name { get; }
     }
     

Wabbajack.Launcher/Wabbajack.Launcher.csproj

@@ -4,8 +4,8 @@
         <OutputType>Exe</OutputType>
         <TargetFramework>net5.0-windows</TargetFramework>
         <UseWPF>true</UseWPF>
-        <AssemblyVersion>2.5.0.9</AssemblyVersion>
+        <AssemblyVersion>2.5.1.0</AssemblyVersion>
-        <FileVersion>2.5.0.9</FileVersion>
+        <FileVersion>2.5.1.0</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Application Launcher</Description>
         <PublishReadyToRun>true</PublishReadyToRun>

Wabbajack.Server/Wabbajack.Server.csproj

@@ -3,8 +3,8 @@
     <PropertyGroup>
         <OutputType>Exe</OutputType>
         <TargetFramework>net5.0-windows</TargetFramework>
-        <AssemblyVersion>2.5.0.9</AssemblyVersion>
+        <AssemblyVersion>2.5.1.0</AssemblyVersion>
-        <FileVersion>2.5.0.9</FileVersion>
+        <FileVersion>2.5.1.0</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Server</Description>
         <RuntimeIdentifier>win-x64</RuntimeIdentifier>

Wabbajack.Test/NexusTests.cs

@@ -10,6 +10,8 @@ namespace Wabbajack.Test
 {
     public class NexusTests : ATestBase
     {
+        // Disable for now
+        /*
         [Fact]
         public async Task CanGetNexusRSSUpdates()
         {
@@ -21,7 +23,7 @@ namespace Wabbajack.Test
             {
                 Assert.True(DateTime.UtcNow - result.TimeStamp < TimeSpan.FromDays(1));
             }
-        }
+        }*/
 
         public NexusTests(ITestOutputHelper output) : base(output)
         {

Wabbajack.VirtualFileSystem/Context.cs

@@ -345,16 +345,27 @@ namespace Wabbajack.VirtualFileSystem
             await _unstage();
         }
     }
+    
+    public static class EmptyLookup<TKey, TElement>
+    {
+        private static readonly ILookup<TKey, TElement> _instance
+            = Enumerable.Empty<TElement>().ToLookup(x => default(TKey));
+
+        public static ILookup<TKey, TElement> Instance
+        {
+            get { return _instance; }
+        }
+    }
 
     public class IndexRoot
     {
         public static IndexRoot Empty = new IndexRoot();
 
-        public IndexRoot(ImmutableList<VirtualFile> aFiles,
+        public IndexRoot(IReadOnlyList<VirtualFile> aFiles,
-            Dictionary<FullPath, VirtualFile> byFullPath,
+            IDictionary<FullPath, VirtualFile> byFullPath,
-            ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> byHash,
+            ILookup<Hash, VirtualFile> byHash,
-            ImmutableDictionary<AbsolutePath, VirtualFile> byRoot,
+            IDictionary<AbsolutePath, VirtualFile> byRoot,
-            ImmutableDictionary<IPath, ImmutableStack<VirtualFile>> byName)
+            ILookup<IPath, VirtualFile> byName)
         {
             AllFiles = aFiles;
             ByFullPath = byFullPath;
@@ -367,17 +378,17 @@ namespace Wabbajack.VirtualFileSystem
         {
             AllFiles = ImmutableList<VirtualFile>.Empty;
             ByFullPath = new Dictionary<FullPath, VirtualFile>();
-            ByHash = ImmutableDictionary<Hash, ImmutableStack<VirtualFile>>.Empty;
+            ByHash = EmptyLookup<Hash, VirtualFile>.Instance;
-            ByRootPath = ImmutableDictionary<AbsolutePath, VirtualFile>.Empty;
+            ByRootPath = new Dictionary<AbsolutePath, VirtualFile>();
-            ByName = ImmutableDictionary<IPath, ImmutableStack<VirtualFile>>.Empty;
+            ByName = EmptyLookup<IPath, VirtualFile>.Instance;
         }
 
 
-        public ImmutableList<VirtualFile> AllFiles { get; }
+        public IReadOnlyList<VirtualFile> AllFiles { get; }
-        public Dictionary<FullPath, VirtualFile> ByFullPath { get; }
+        public IDictionary<FullPath, VirtualFile> ByFullPath { get; }
-        public ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> ByHash { get; }
+        public ILookup<Hash, VirtualFile> ByHash { get; }
-        public ImmutableDictionary<IPath, ImmutableStack<VirtualFile>> ByName { get; set; }
+        public ILookup<IPath, VirtualFile> ByName { get; set; }
-        public ImmutableDictionary<AbsolutePath, VirtualFile> ByRootPath { get; }
+        public IDictionary<AbsolutePath, VirtualFile> ByRootPath { get; }
 
         public async Task<IndexRoot> Integrate(ICollection<VirtualFile> files)
         {
@@ -385,19 +396,19 @@ namespace Wabbajack.VirtualFileSystem
             var allFiles = AllFiles.Concat(files)
                 .OrderByDescending(f => f.LastModified)
                 .GroupBy(f => f.FullPath).Select(g => g.Last())
-                .ToImmutableList();
+                .ToList();
 
             var byFullPath = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
                                      .ToDictionary(f => f.FullPath));
 
             var byHash = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
                                  .Where(f => f.Hash != Hash.Empty)
-                                 .ToGroupedImmutableDictionary(f => f.Hash));
+                                 .ToLookup(f => f.Hash));
 
             var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
-                                 .ToGroupedImmutableDictionary(f => f.Name));
+                                 .ToLookup(f => f.Name));
 
-            var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.AbsoluteName));
+            var byRootPath = Task.Run(() => allFiles.ToDictionary(f => f.AbsoluteName));
 
             var result = new IndexRoot(allFiles,
                 await byFullPath,

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -235,7 +235,11 @@ namespace Wabbajack.VirtualFileSystem
                 self.ExtendedHashes = await ExtendedHashes.FromStream(stream);
 
             // Can't extract, so return
-            if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension)) return self;
+            if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension))
+            {
+                await WriteToCache(self);
+                return self;
+            }
 
             try
             {
@@ -256,11 +260,16 @@ namespace Wabbajack.VirtualFileSystem
                 throw;
             }
 
+            await WriteToCache(self);
+            return self;
+        }
+
+        private static async Task WriteToCache(VirtualFile self)
+        {
             await using var ms = new MemoryStream();
             self.ToIndexedVirtualFile().Write(ms);
             ms.Position = 0;
             await InsertIntoVFSCache(self.Hash, ms);
-            return self;
         }
 
         private static async Task InsertIntoVFSCache(Hash hash, MemoryStream data)

Wabbajack/Wabbajack.csproj

@@ -6,8 +6,8 @@
     <UseWPF>true</UseWPF>
     <Platforms>x64</Platforms>
     <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
-      <AssemblyVersion>2.5.0.9</AssemblyVersion>
+      <AssemblyVersion>2.5.1.0</AssemblyVersion>
-      <FileVersion>2.5.0.9</FileVersion>
+      <FileVersion>2.5.1.0</FileVersion>
     <Copyright>Copyright © 2019-2020</Copyright>
     <Description>An automated ModList installer</Description>
     <PublishReadyToRun>true</PublishReadyToRun>