Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make ImageConverter polymorphic and revert back to texcov on Windows (#2281)

Browse Source 
* Make ImageConverter polymorphic and revert back to texcov on Windows

* Add files I forgot to add, make CHANGELOG.md additions

* Don't run texconv tests on Linux/OSX
This commit is contained in:
Timothy Baldridge 2023-01-21 13:36:12 -06:00 committed by GitHub
parent 31ec53aa29
commit 5b77574b5e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 353 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -4,7 +4,7 @@
* Add support for Cubemaps in BA2 files, if you have problems with BA2 recompression, be sure to delete your `GlobalVFSCache3.sqlite` from your AppData before the next compile
* Fixed slides not being shown during installation for lists compile with the 3.0 compiler
* Set the "While loading slide" debug message to be `Trace` level, set the default minimum log level to `Information`
* Switched back to using TexConv for texture converting on Windows, should greatly improve compatability of texture conversion (on windows systems)
#### Version - 3.0.5.0 - 12/22/2022
* Add support for https://www.nexusmods.com/site hosted mods.

1
Wabbajack.Common/Ext.cs
View File

@ -25,4 +25,5 @@ public static class Ext
public static Extension ModlistMetadataExtension = new(".modlist_metadata");
public static Extension Txt = new(".txt");
public static Extension Webp = new(".webp");
public static Extension Png = new(".png");
}

12
Wabbajack.Compiler.Test/SanityTests.cs
View File

@ -28,10 +28,13 @@ public class CompilerSanityTests : IAsyncLifetime
private readonly IServiceProvider _serviceProvider;
private Mod _mod;
private ModList? _modlist;
private readonly IImageLoader _imageLoader;
public CompilerSanityTests(ILogger<CompilerSanityTests> logger, IServiceProvider serviceProvider,
FileExtractor.FileExtractor fileExtractor,
TemporaryFileManager manager, ParallelOptions parallelOptions)
TemporaryFileManager manager,
ParallelOptions parallelOptions,
IImageLoader imageLoader)
{
_logger = logger;
_serviceProvider = serviceProvider;
@ -40,6 +43,7 @@ public class CompilerSanityTests : IAsyncLifetime
_fileExtractor = fileExtractor;
_manager = manager;
_parallelOptions = parallelOptions;
_imageLoader = imageLoader;
}
@ -195,12 +199,12 @@ public class CompilerSanityTests : IAsyncLifetime
foreach (var file in _mod.FullPath.EnumerateFiles())
{
var oldState = await ImageLoader.Load(file);
var oldState = await _imageLoader.Load(file);
Assert.NotEqual(DXGI_FORMAT.UNKNOWN, oldState.Format);
_logger.LogInformation("Recompressing {file}", file.FileName);
await ImageLoader.Recompress(file, 512, 512, DXGI_FORMAT.BC7_UNORM, file, CancellationToken.None);
await _imageLoader.Recompress(file, 512, 512, DXGI_FORMAT.BC7_UNORM, file, CancellationToken.None);
var state = await ImageLoader.Load(file);
var state = await _imageLoader.Load(file);
Assert.Equal(DXGI_FORMAT.BC7_UNORM, state.Format);
}

7
Wabbajack.Compiler/ACompiler.cs
View File

@ -17,6 +17,7 @@ using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.FileExtractor.ExtractedFiles;
using Wabbajack.Hashing.PHash;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Installer;
using Wabbajack.Networking.WabbajackClientApi;
@ -64,7 +65,8 @@ public abstract class ACompiler
TemporaryFileManager manager, CompilerSettings settings,
ParallelOptions parallelOptions, DownloadDispatcher dispatcher, Client wjClient, IGameLocator locator,
DTOSerializer dtos, IResource<ACompiler> compilerLimiter,
IBinaryPatchCache patchCache)
IBinaryPatchCache patchCache,
IImageLoader imageLoader)
{
CompilerLimiter = compilerLimiter;
_logger = logger;
@ -83,9 +85,12 @@ public abstract class ACompiler
_patchOptions = new ConcurrentDictionary<PatchedFromArchive, VirtualFile[]>();
_sourceFileLinks = new ConcurrentDictionary<Directive, RawSourceFile>();
_patchCache = patchCache;
ImageLoader = imageLoader;
_updateStopWatch = new Stopwatch();
}
public IImageLoader ImageLoader { get; }
protected long MaxSteps { get; set; }
public CompilerSettings Settings

4
Wabbajack.Compiler/CompilationSteps/MatchSimilarTextures.cs
View File

@ -34,7 +34,7 @@ public class MatchSimilarTextures : ACompilationStep
_compiler._logger.LogInformation("Looking for texture match for {source}", source.File.FullPath);
(float Similarity, VirtualFile File) found = _byName[source.Path.FileNameWithoutExtension]
.Select(f => (
ImageLoader.ComputeDifference(f.ImageState!.PerceptualHash, source.File.ImageState.PerceptualHash),
IImageLoader.ComputeDifference(f.ImageState!.PerceptualHash, source.File.ImageState.PerceptualHash),
f))
.Select(f => { return f; })
.OrderByDescending(f => f.Item1)
@ -58,7 +58,7 @@ public class MatchSimilarTextures : ACompilationStep
from mainMatch in _byName[mainFile.FullPath.FileName.FileNameWithoutExtension]
where mainMatch.ImageState != null
where mainFile.ImageState != null
let similarity = ImageLoader.ComputeDifference(mainFile.ImageState!.PerceptualHash,
let similarity = IImageLoader.ComputeDifference(mainFile.ImageState!.PerceptualHash,
mainMatch.ImageState!.PerceptualHash)
where similarity >= PerceptualTolerance
orderby similarity descending

9
Wabbajack.Compiler/MO2Compiler.cs
View File

@ -14,6 +14,7 @@ using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Hashing.PHash;
using Wabbajack.Installer;
using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.Paths;
@ -30,9 +31,10 @@ public class MO2Compiler : ACompiler
TemporaryFileManager manager, CompilerSettings settings, ParallelOptions parallelOptions,
DownloadDispatcher dispatcher,
Client wjClient, IGameLocator locator, DTOSerializer dtos, IResource<ACompiler> compilerLimiter,
IBinaryPatchCache patchCache) :
IBinaryPatchCache patchCache,
IImageLoader imageLoader) :
base(logger, extractor, hashCache, vfs, manager, settings, parallelOptions, dispatcher, wjClient, locator, dtos,
compilerLimiter, patchCache)
compilerLimiter, patchCache, imageLoader)
{
MaxSteps = 14;
}
@ -51,7 +53,8 @@ public class MO2Compiler : ACompiler
provider.GetRequiredService<IGameLocator>(),
provider.GetRequiredService<DTOSerializer>(),
provider.GetRequiredService<IResource<ACompiler>>(),
provider.GetRequiredService<IBinaryPatchCache>());
provider.GetRequiredService<IBinaryPatchCache>(),
provider.GetRequiredService<IImageLoader>());
}
public CompilerSettings Mo2Settings => (CompilerSettings) Settings;

84
Wabbajack.Hashing.PHash.Test/FileLoadingTests.cs
View File

@ -1,4 +1,6 @@
using System;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
@ -10,8 +12,31 @@ using Xunit;
namespace Wabbajack.Hashing.PHash.Test;
public class FileLoadingTests
public class FileLoadingTests : IAsyncDisposable
{
private readonly IImageLoader[] _imageLoaders;
private readonly TemporaryFileManager _tmp;
public FileLoadingTests()
{
_tmp = new TemporaryFileManager();
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) {
_imageLoaders = new IImageLoader[]
{
new CrossPlatformImageLoader(),
new TexConvImageLoader(_tmp)
};
}
else
{
_imageLoaders = new IImageLoader[]
{
new CrossPlatformImageLoader(),
};
}
}
[Theory]
[InlineData("test-dxt5.dds", 1.0f)]
[InlineData("test-dxt5-recompressed.dds", 1f)]
@ -19,40 +44,47 @@ public class FileLoadingTests
[InlineData("test-dxt5-small-bc7-vflip.dds", 0.189f)]
public async Task LoadAllFiles(string file, float difference)
{
var baseState =
await ImageLoader.Load("TestData/test-dxt5.dds".ToRelativePath().RelativeTo(KnownFolders.EntryPoint));
var state = await ImageLoader.Load("TestData".ToRelativePath().Combine(file)
.RelativeTo(KnownFolders.EntryPoint));
foreach (var imageLoader in _imageLoaders)
{
var baseState =
await imageLoader.Load("TestData/test-dxt5.dds".ToRelativePath().RelativeTo(KnownFolders.EntryPoint));
var state = await imageLoader.Load("TestData".ToRelativePath().Combine(file)
.RelativeTo(KnownFolders.EntryPoint));
Assert.NotEqual(DXGI_FORMAT.UNKNOWN, baseState.Format);
Assert.NotEqual(DXGI_FORMAT.UNKNOWN, baseState.Format);
Assert.Equal(difference,
ImagePhash.GetCrossCorrelation(
new Digest {Coefficients = baseState.PerceptualHash.Data},
new Digest {Coefficients = state.PerceptualHash.Data}),
1.0);
Assert.Equal(difference,
ImagePhash.GetCrossCorrelation(
new Digest { Coefficients = baseState.PerceptualHash.Data },
new Digest { Coefficients = state.PerceptualHash.Data }),
1.0);
}
}
[Fact]
public async Task CanConvertCubeMaps()
{
// File used here via re-upload permissions found on the mod's Nexus page:
// https://www.nexusmods.com/fallout4/mods/43458?tab=description
// Used for testing purposes only
var path = "TestData/WindowDisabled_CGPlayerHouseCube.dds".ToRelativePath().RelativeTo(KnownFolders.EntryPoint);
foreach (var imageLoader in _imageLoaders)
{
// File used here via re-upload permissions found on the mod's Nexus page:
// https://www.nexusmods.com/fallout4/mods/43458?tab=description
// Used for testing purposes only
var path = "TestData/WindowDisabled_CGPlayerHouseCube.dds".ToRelativePath().RelativeTo(KnownFolders.EntryPoint);
var baseState = await ImageLoader.Load(path);
baseState.Height.Should().Be(128);
baseState.Width.Should().Be(128);
//baseState.Frames.Should().Be(6);
using var ms = new MemoryStream();
await using var ins = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
await ImageLoader.Recompress(ins, 128, 128, DXGI_FORMAT.BC1_UNORM, ms, CancellationToken.None, leaveOpen:true);
ms.Length.Should().Be(ins.Length);
var baseState = await imageLoader.Load(path);
baseState.Height.Should().Be(128);
baseState.Width.Should().Be(128);
//baseState.Frames.Should().Be(6);
using var ms = new MemoryStream();
await using var ins = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
await imageLoader.Recompress(ins, 128, 128, DXGI_FORMAT.BC1_UNORM, ms, CancellationToken.None, leaveOpen:true);
ms.Length.Should().Be(ins.Length);
}
}
public async ValueTask DisposeAsync()
{
await _tmp.DisposeAsync();
}
}

27
Wabbajack.Hashing.PHash/Image.cs → Wabbajack.Hashing.PHash/CrossPlatformImageLoader.cs
View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using BCnEncoder.Decoder;
@ -13,21 +14,22 @@ using Shipwreck.Phash.Imaging;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using Wabbajack.Common;
using Wabbajack.DTOs.Texture;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
namespace Wabbajack.Hashing.PHash;
public class ImageLoader
public class CrossPlatformImageLoader : IImageLoader
{
public static async ValueTask<ImageState> Load(AbsolutePath path)
public async ValueTask<ImageState> Load(AbsolutePath path)
{
await using var fs = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
return await Load(fs);
}
public static async ValueTask<ImageState> Load(Stream stream)
public async ValueTask<ImageState> Load(Stream stream)
{
var decoder = new BcDecoder();
var ddsFile = DdsFile.Load(stream);
@ -57,8 +59,8 @@ public class ImageLoader
new Digest {Coefficients = a.Data},
new Digest {Coefficients = b.Data});
}
public static async Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format,
public async Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format,
AbsolutePath output,
CancellationToken token)
{
@ -67,24 +69,23 @@ public class ImageLoader
await Recompress(new MemoryStream(inData), width, height, format, outStream, token);
}
public static async Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output,
public async Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output,
CancellationToken token, bool leaveOpen = false)
{
var decoder = new BcDecoder();
var ddsFile = DdsFile.Load(input);
if (!leaveOpen) await input.DisposeAsync();
var faces = new List<Image<Rgba32>>();
var origFormat = ddsFile.dx10Header.dxgiFormat == DxgiFormat.DxgiFormatUnknown
? ddsFile.header.ddsPixelFormat.DxgiFormat
: ddsFile.dx10Header.dxgiFormat;
foreach (var face in ddsFile.Faces)
{
var data = await decoder.DecodeRawToImageRgba32Async(face.MipMaps[0].Data,
var data = await decoder.DecodeRawToImageRgba32Async(face.MipMaps[0].Data,
(int)face.Width, (int)face.Height, ToCompressionFormat((DXGI_FORMAT)origFormat), token);
data.Mutate(x => x.Resize(width, height, KnownResamplers.Welch));
@ -101,7 +102,7 @@ public class ImageLoader
FileFormat = OutputFileFormat.Dds
}
};
switch (faces.Count)
{
case 1:
@ -114,11 +115,11 @@ public class ImageLoader
default:
throw new NotImplementedException($"Can't encode dds with {faces.Count} faces");
}
if (!leaveOpen)
await output.DisposeAsync();
}
public static CompressionFormat ToCompressionFormat(DXGI_FORMAT dx)
{
return dx switch

27
Wabbajack.Hashing.PHash/IImageLoader.cs Normal file
View File

@ -0,0 +1,27 @@
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Shipwreck.Phash;
using Wabbajack.DTOs.Texture;
using Wabbajack.Paths;
namespace Wabbajack.Hashing.PHash;
public interface IImageLoader
{
public ValueTask<ImageState> Load(AbsolutePath path);
public ValueTask<ImageState> Load(Stream stream);
public static float ComputeDifference(DTOs.Texture.PHash a, DTOs.Texture.PHash b)
{
return ImagePhash.GetCrossCorrelation(
new Digest {Coefficients = a.Data},
new Digest {Coefficients = b.Data});
}
public Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format,
AbsolutePath output,
CancellationToken token);
public Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output,
CancellationToken token, bool leaveOpen = false);
}

160
Wabbajack.Hashing.PHash/TexConvImageLoader.cs Normal file
View File

@ -0,0 +1,160 @@
using System;
using System.Collections.Concurrent;
using System.IO;
using System.Linq;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using Shipwreck.Phash;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using Wabbajack.Common;
using Wabbajack.Common.FileSignatures;
using Wabbajack.DTOs.Texture;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
namespace Wabbajack.Hashing.PHash;
public class TexConvImageLoader : IImageLoader
{
private readonly SignatureChecker _sigs;
private readonly TemporaryFileManager _tempManager;
public TexConvImageLoader(TemporaryFileManager manager)
{
_tempManager = manager;
_sigs = new SignatureChecker(FileType.DDS, FileType.PNG, FileType.JPG, FileType.BMP);
}
public async ValueTask<ImageState> Load(AbsolutePath path)
{
return await GetState(path);
}
public async ValueTask<ImageState> Load(Stream stream)
{
var ext = await DetermineType(stream);
var temp = _tempManager.CreateFile(ext);
await using var fs = temp.Path.Open(FileMode.Create, FileAccess.Write, FileShare.None);
await stream.CopyToAsync(fs);
fs.Close();
return await GetState(temp.Path);
}
private async Task<Extension> DetermineType(Stream stream)
{
var sig = await _sigs.MatchesAsync(stream);
var ext = new Extension(".tga");
if (sig != null)
ext = new Extension("." + Enum.GetName(sig.Value));
stream.Position = 0;
return ext;
}
public async Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format, AbsolutePath output,
CancellationToken token)
{
var outFolder = _tempManager.CreateFolder();
var outFile = input.FileName.RelativeTo(outFolder.Path);
await ConvertImage(input, outFolder.Path, width, height, format, input.Extension);
await outFile.MoveToAsync(output, token: token, overwrite:true);
}
public async Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output, CancellationToken token,
bool leaveOpen = false)
{
var type = await DetermineType(input);
await using var toFolder = _tempManager.CreateFolder();
await using var fromFile = _tempManager.CreateFile(type);
await input.CopyToAsync(fromFile.Path, token);
var toFile = fromFile.Path.FileName.RelativeTo(toFolder);
await ConvertImage(fromFile.Path, toFolder.Path, width, height, format, type);
await using var fs = toFile.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
await fs.CopyToAsync(output, token);
}
public async Task ConvertImage(AbsolutePath from, AbsolutePath toFolder, int w, int h, DXGI_FORMAT format, Extension fileFormat)
{
// User isn't renaming the file, so we don't have to create a temporary folder
var ph = new ProcessHelper
{
Path = @"Tools\texconv.exe".ToRelativePath().RelativeTo(KnownFolders.EntryPoint),
Arguments = new object[] {from, "-ft", fileFormat.ToString()[1..], "-f", format, "-o", toFolder, "-w", w, "-h", h, "-if", "CUBIC", "-singleproc"},
ThrowOnNonZeroExitCode = true,
LogError = true
};
await ph.Start();
}
public async Task ConvertImage(Stream from, ImageState state, Extension ext, AbsolutePath to)
{
await using var tmpFile = _tempManager.CreateFolder();
var inFile = to.FileName.RelativeTo(tmpFile.Path);
await inFile.WriteAllAsync(from, CancellationToken.None);
await ConvertImage(inFile, to.Parent, state.Width, state.Height, state.Format, ext);
}
// Internals
public async Task<ImageState> GetState(AbsolutePath path)
{
try
{
var ph = new ProcessHelper
{
Path = @"Tools\texdiag.exe".ToRelativePath().RelativeTo(KnownFolders.EntryPoint),
Arguments = new object[] {"info", path, "-nologo"},
ThrowOnNonZeroExitCode = true,
LogError = true
};
var lines = new ConcurrentStack<string>();
using var _ = ph.Output.Where(p => p.Type == ProcessHelper.StreamType.Output)
.Select(p => p.Line)
.Where(p => p.Contains(" = "))
.Subscribe(l => lines.Push(l));
await ph.Start();
var data = lines.Select(l =>
{
var split = l.Split(" = ");
return (split[0].Trim(), split[1].Trim());
}).ToDictionary(p => p.Item1, p => p.Item2);
return new ImageState
{
Width = int.Parse(data["width"]),
Height = int.Parse(data["height"]),
Format = Enum.Parse<DXGI_FORMAT>(data["format"]),
PerceptualHash = await GetPHash(path)
};
}
catch (Exception ex)
{
throw;
}
}
public async Task<DTOs.Texture.PHash> GetPHash(AbsolutePath path)
{
if (!path.FileExists())
throw new FileNotFoundException($"Can't hash non-existent file {path}");
await using var tmp = _tempManager.CreateFolder();
await ConvertImage(path, tmp.Path, 512, 512, DXGI_FORMAT.R8G8B8A8_UNORM, Ext.Png);
using var img = await Image.LoadAsync(path.FileName.RelativeTo(tmp.Path).ReplaceExtension(Ext.Png).ToString());
img.Mutate(x => x.Resize(512, 512, KnownResamplers.Welch).Grayscale(GrayscaleMode.Bt601));
return new DTOs.Texture.PHash(ImagePhash.ComputeDigest(new CrossPlatformImageLoader.ImageBitmap((Image<Rgba32>)img)).Coefficients);
}
}

BIN
Wabbajack.Hashing.PHash/Tools/texconv.exe Normal file
View File

Binary file not shown.

BIN
Wabbajack.Hashing.PHash/Tools/texdiag.exe Normal file
View File

Binary file not shown.

14
Wabbajack.Hashing.PHash/Wabbajack.Hashing.PHash.csproj
View File

@ -13,9 +13,23 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
<ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
<ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
<ProjectReference Include="..\Wabbajack.Paths\Wabbajack.Paths.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Tools" />
</ItemGroup>
<ItemGroup>
<None Update="Tools\texconv.exe">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Tools\texdiag.exe">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

6
Wabbajack.Installer/AInstaller.cs
View File

@ -75,7 +75,8 @@ public abstract class AInstaller<T>
DownloadDispatcher downloadDispatcher,
ParallelOptions parallelOptions,
IResource<IInstaller> limiter,
Client wjClient)
Client wjClient,
IImageLoader imageLoader)
{
_limiter = limiter;
_manager = new TemporaryFileManager(config.Install.Combine("__temp__"));
@ -90,8 +91,11 @@ public abstract class AInstaller<T>
_parallelOptions = parallelOptions;
_gameLocator = gameLocator;
_wjClient = wjClient;
ImageLoader = imageLoader;
}
public IImageLoader ImageLoader { get; }
protected long MaxSteps { get; set; }
public Dictionary<Hash, AbsolutePath> HashedArchives { get; set; } = new();

8
Wabbajack.Installer/StandardInstaller.cs
View File

@ -23,6 +23,7 @@ using Wabbajack.DTOs.BSA.FileStates;
using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Hashing.PHash;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Installer.Utilities;
using Wabbajack.Networking.WabbajackClientApi;
@ -40,9 +41,9 @@ public class StandardInstaller : AInstaller<StandardInstaller>
InstallerConfiguration config,
IGameLocator gameLocator, FileExtractor.FileExtractor extractor,
DTOSerializer jsonSerializer, Context vfs, FileHashCache fileHashCache,
DownloadDispatcher downloadDispatcher, ParallelOptions parallelOptions, IResource<IInstaller> limiter, Client wjClient) :
DownloadDispatcher downloadDispatcher, ParallelOptions parallelOptions, IResource<IInstaller> limiter, Client wjClient, IImageLoader imageLoader) :
base(logger, config, gameLocator, extractor, jsonSerializer, vfs, fileHashCache, downloadDispatcher,
parallelOptions, limiter, wjClient)
parallelOptions, limiter, wjClient, imageLoader)
{
MaxSteps = 14;
}
@ -59,7 +60,8 @@ public class StandardInstaller : AInstaller<StandardInstaller>
provider.GetRequiredService<DownloadDispatcher>(),
provider.GetRequiredService<ParallelOptions>(),
provider.GetRequiredService<IResource<IInstaller>>(),
provider.GetRequiredService<Client>());
provider.GetRequiredService<Client>(),
provider.GetRequiredService<IImageLoader>());
}
public override async Task<bool> Begin(CancellationToken token)

12
Wabbajack.Paths.IO/AbsolutePathExtensions.cs
View File

@ -252,6 +252,18 @@ public static class AbsolutePathExtensions
return file.ToString();
}
public static async Task CopyToAsync(this Stream from, AbsolutePath path, CancellationToken token = default)
{
await using var to = path.Open(FileMode.Create, FileAccess.Write, FileShare.None);
await from.CopyToAsync(to, token);
}
public static async Task CopyToAsync(this AbsolutePath from, Stream to, CancellationToken token = default)
{
await using var fromStream = from.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
await fromStream.CopyToAsync(to, token);
}
#region Directories
public static void CreateDirectory(this AbsolutePath path)

9
Wabbajack.Paths/AbsolutePath.cs
View File

@ -85,7 +85,12 @@ public struct AbsolutePath : IPath, IComparable<AbsolutePath>, IEquatable<Absolu
}
}
public AbsolutePath ReplaceExtension(Extension newExtension)
/// <summary>
/// Returns a new path that is this path with the extension changed.
/// </summary>
/// <param name="newExtension"></param>
/// <returns></returns>
public readonly AbsolutePath ReplaceExtension(Extension newExtension)
{
var paths = new string[Parts.Length];
Array.Copy(Parts, paths, paths.Length);
@ -94,7 +99,7 @@ public struct AbsolutePath : IPath, IComparable<AbsolutePath>, IEquatable<Absolu
paths[^1] = newName;
return new AbsolutePath(paths, PathFormat);
}
public static explicit operator AbsolutePath(string input)
{
return Parse(input);

10
Wabbajack.Paths/RelativePath.cs
View File

@ -48,12 +48,10 @@ public struct RelativePath : IPath, IEquatable<RelativePath>, IComparable<Relati
internal static string ReplaceExtension(string oldName, Extension newExtension)
{
var oldExtLength = oldName.LastIndexOf(".", StringComparison.CurrentCultureIgnoreCase);
if (oldExtLength < 0)
oldExtLength = 0;
else
oldExtLength++;
var newName = oldName[..^oldExtLength] + newExtension;
if (oldExtLength <= 0)
return oldName + newExtension;
var newName = oldName[..oldExtLength] + newExtension;
return newName;
}

7
Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
View File

@ -15,6 +15,7 @@ using Wabbajack.DTOs;
using Wabbajack.DTOs.Interventions;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.DTOs.Logins;
using Wabbajack.Hashing.PHash;
using Wabbajack.Installer;
using Wabbajack.Networking.BethesdaNet;
using Wabbajack.Networking.Discord;
@ -180,6 +181,12 @@ public static class ServiceExtensions
service.AddAllSingleton<IGameLocator, StubbedGameLocator>();
else
service.AddAllSingleton<IGameLocator, GameLocator>();
// ImageLoader
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
service.AddSingleton<IImageLoader, TexConvImageLoader>();
else
service.AddSingleton<IImageLoader, CrossPlatformImageLoader>();
// Installer/Compiler Configuration
service.AddScoped<InstallerConfiguration>();

9
Wabbajack.VFS.Test/Startup.cs
View File

@ -1,7 +1,9 @@
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Wabbajack.DTOs;
using Wabbajack.Hashing.PHash;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
using Wabbajack.VFS.Interfaces;
@ -28,6 +30,13 @@ public class Startup
.AddAllSingleton<IResource, IResource<FileHashCache>, Resource<FileHashCache>>(
s =>
new ("File Hash Cache", 2));
// ImageLoader
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
service.AddSingleton<IImageLoader, TexConvImageLoader>();
else
service.AddSingleton<IImageLoader, CrossPlatformImageLoader>();
// Keep this fixed at 2 so that we can detect deadlocks in the VFS parallelOptions
service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = 2});

8
Wabbajack.VFS/Context.cs
View File

@ -7,6 +7,7 @@ using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.FileExtractor.ExtractedFiles;
using Wabbajack.Hashing.PHash;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
@ -32,7 +33,8 @@ public class Context
public Context(ILogger<Context> logger, ParallelOptions parallelOptions, TemporaryFileManager manager,
IVfsCache vfsCache,
FileHashCache hashCache, IResource<Context> limiter, IResource<FileHashCache> hashLimiter, FileExtractor.FileExtractor extractor)
FileHashCache hashCache, IResource<Context> limiter, IResource<FileHashCache> hashLimiter,
FileExtractor.FileExtractor extractor, IImageLoader imageLoader)
{
Limiter = limiter;
HashLimiter = hashLimiter;
@ -42,16 +44,18 @@ public class Context
VfsCache = vfsCache;
HashCache = hashCache;
_parallelOptions = parallelOptions;
ImageLoader = imageLoader;
}
public Context WithTemporaryFileManager(TemporaryFileManager manager)
{
return new Context(Logger, _parallelOptions, manager, VfsCache, HashCache, Limiter, HashLimiter,
Extractor.WithTemporaryFileManager(manager));
Extractor.WithTemporaryFileManager(manager), ImageLoader);
}
public IndexRoot Index { get; private set; } = IndexRoot.Empty;
public IImageLoader ImageLoader { get; }
public async Task<IndexRoot> AddRoot(AbsolutePath root, CancellationToken token)
{

2
Wabbajack.VFS/VirtualFile.cs
View File

@ -204,7 +204,7 @@ public class VirtualFile
if (TextureExtensions.Contains(relPath.FileName.Extension) && await DDSSig.MatchesAsync(stream) != null)
try
{
self.ImageState = await ImageLoader.Load(stream);
self.ImageState = await context.ImageLoader.Load(stream);
if (job != null)
{
job.Size += self.Size;