Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1530 from wabbajack-tools/optimize-compilation

Browse Source 
Optimize the compilation process
This commit is contained in:
Timothy Baldridge 2021-07-06 07:10:18 -06:00 committed by GitHub
commit f36916b2fc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 63 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -1,5 +1,8 @@
### Changelog
#### Version - 2.5.1.0 - 7/6/2021
* Drastically improve compilation times by the removal of several bugs in the compiler
#### Version - 2.5.0.9 - 7/4/2021
* Use DX11 GPUs for compression when possible
* Don't threadbomb the system by creating O(N*M) threads when compressing textures

4
Wabbajack.CLI/Wabbajack.CLI.csproj
View File

@ -6,8 +6,8 @@
<AssemblyName>wabbajack-cli</AssemblyName>
<Company>Wabbajack</Company>
<Platforms>x64</Platforms>
<AssemblyVersion>2.5.0.9</AssemblyVersion>
<FileVersion>2.5.0.9</FileVersion>
<AssemblyVersion>2.5.1.0</AssemblyVersion>
<FileVersion>2.5.1.0</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>An automated ModList installer</Description>
<PublishReadyToRun>true</PublishReadyToRun>

11
Wabbajack.Common/IStreamFactory.cs
View File

@ -34,7 +34,16 @@ namespace Wabbajack.Common
return await _file.OpenRead();
}
public DateTime LastModifiedUtc => _file.LastModifiedUtc;
private DateTime? _lastModifiedCache = null;
public DateTime LastModifiedUtc
{
get
{
_lastModifiedCache ??= _file.LastModifiedUtc;
return _lastModifiedCache.Value;
}
}
public IPath Name { get; }
}

4
Wabbajack.Launcher/Wabbajack.Launcher.csproj
View File

@ -4,8 +4,8 @@
<OutputType>Exe</OutputType>
<TargetFramework>net5.0-windows</TargetFramework>
<UseWPF>true</UseWPF>
<AssemblyVersion>2.5.0.9</AssemblyVersion>
<FileVersion>2.5.0.9</FileVersion>
<AssemblyVersion>2.5.1.0</AssemblyVersion>
<FileVersion>2.5.1.0</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>Wabbajack Application Launcher</Description>
<PublishReadyToRun>true</PublishReadyToRun>

4
Wabbajack.Server/Wabbajack.Server.csproj
View File

@ -3,8 +3,8 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net5.0-windows</TargetFramework>
<AssemblyVersion>2.5.0.9</AssemblyVersion>
<FileVersion>2.5.0.9</FileVersion>
<AssemblyVersion>2.5.1.0</AssemblyVersion>
<FileVersion>2.5.1.0</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>Wabbajack Server</Description>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>

4
Wabbajack.Test/NexusTests.cs
View File

@ -10,6 +10,8 @@ namespace Wabbajack.Test
{
public class NexusTests : ATestBase
{
// Disable for now
/*
[Fact]
public async Task CanGetNexusRSSUpdates()
{
@ -21,7 +23,7 @@ namespace Wabbajack.Test
{
Assert.True(DateTime.UtcNow - result.TimeStamp < TimeSpan.FromDays(1));
}
}
}*/
public NexusTests(ITestOutputHelper output) : base(output)
{

45
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -345,16 +345,27 @@ namespace Wabbajack.VirtualFileSystem
await _unstage();
}
}
public static class EmptyLookup<TKey, TElement>
{
private static readonly ILookup<TKey, TElement> _instance
= Enumerable.Empty<TElement>().ToLookup(x => default(TKey));
public static ILookup<TKey, TElement> Instance
{
get { return _instance; }
}
}
public class IndexRoot
{
public static IndexRoot Empty = new IndexRoot();
public IndexRoot(ImmutableList<VirtualFile> aFiles,
Dictionary<FullPath, VirtualFile> byFullPath,
ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> byHash,
ImmutableDictionary<AbsolutePath, VirtualFile> byRoot,
ImmutableDictionary<IPath, ImmutableStack<VirtualFile>> byName)
public IndexRoot(IReadOnlyList<VirtualFile> aFiles,
IDictionary<FullPath, VirtualFile> byFullPath,
ILookup<Hash, VirtualFile> byHash,
IDictionary<AbsolutePath, VirtualFile> byRoot,
ILookup<IPath, VirtualFile> byName)
{
AllFiles = aFiles;
ByFullPath = byFullPath;
@ -367,17 +378,17 @@ namespace Wabbajack.VirtualFileSystem
{
AllFiles = ImmutableList<VirtualFile>.Empty;
ByFullPath = new Dictionary<FullPath, VirtualFile>();
ByHash = ImmutableDictionary<Hash, ImmutableStack<VirtualFile>>.Empty;
ByRootPath = ImmutableDictionary<AbsolutePath, VirtualFile>.Empty;
ByName = ImmutableDictionary<IPath, ImmutableStack<VirtualFile>>.Empty;
ByHash = EmptyLookup<Hash, VirtualFile>.Instance;
ByRootPath = new Dictionary<AbsolutePath, VirtualFile>();
ByName = EmptyLookup<IPath, VirtualFile>.Instance;
}
public ImmutableList<VirtualFile> AllFiles { get; }
public Dictionary<FullPath, VirtualFile> ByFullPath { get; }
public ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> ByHash { get; }
public ImmutableDictionary<IPath, ImmutableStack<VirtualFile>> ByName { get; set; }
public ImmutableDictionary<AbsolutePath, VirtualFile> ByRootPath { get; }
public IReadOnlyList<VirtualFile> AllFiles { get; }
public IDictionary<FullPath, VirtualFile> ByFullPath { get; }
public ILookup<Hash, VirtualFile> ByHash { get; }
public ILookup<IPath, VirtualFile> ByName { get; set; }
public IDictionary<AbsolutePath, VirtualFile> ByRootPath { get; }
public async Task<IndexRoot> Integrate(ICollection<VirtualFile> files)
{
@ -385,19 +396,19 @@ namespace Wabbajack.VirtualFileSystem
var allFiles = AllFiles.Concat(files)
.OrderByDescending(f => f.LastModified)
.GroupBy(f => f.FullPath).Select(g => g.Last())
.ToImmutableList();
.ToList();
var byFullPath = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToDictionary(f => f.FullPath));
var byHash = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.Where(f => f.Hash != Hash.Empty)
.ToGroupedImmutableDictionary(f => f.Hash));
.ToLookup(f => f.Hash));
var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToGroupedImmutableDictionary(f => f.Name));
.ToLookup(f => f.Name));
var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.AbsoluteName));
var byRootPath = Task.Run(() => allFiles.ToDictionary(f => f.AbsoluteName));
var result = new IndexRoot(allFiles,
await byFullPath,

13
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -235,7 +235,11 @@ namespace Wabbajack.VirtualFileSystem
self.ExtendedHashes = await ExtendedHashes.FromStream(stream);
// Can't extract, so return
if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension)) return self;
if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension))
{
await WriteToCache(self);
return self;
}
try
{
@ -256,11 +260,16 @@ namespace Wabbajack.VirtualFileSystem
throw;
}
await WriteToCache(self);
return self;
}
private static async Task WriteToCache(VirtualFile self)
{
await using var ms = new MemoryStream();
self.ToIndexedVirtualFile().Write(ms);
ms.Position = 0;
await InsertIntoVFSCache(self.Hash, ms);
return self;
}
private static async Task InsertIntoVFSCache(Hash hash, MemoryStream data)

4
Wabbajack/Wabbajack.csproj
View File

@ -6,8 +6,8 @@
<UseWPF>true</UseWPF>
<Platforms>x64</Platforms>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
<AssemblyVersion>2.5.0.9</AssemblyVersion>
<FileVersion>2.5.0.9</FileVersion>
<AssemblyVersion>2.5.1.0</AssemblyVersion>
<FileVersion>2.5.1.0</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>An automated ModList installer</Description>
<PublishReadyToRun>true</PublishReadyToRun>