diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c2bfcc2..aed246e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ * Add support for Cubemaps in BA2 files, if you have problems with BA2 recompression, be sure to delete your `GlobalVFSCache3.sqlite` from your AppData before the next compile * Fixed slides not being shown during installation for lists compile with the 3.0 compiler * Set the "While loading slide" debug message to be `Trace` level, set the default minimum log level to `Information` - +* Switched back to using TexConv for texture converting on Windows, should greatly improve compatability of texture conversion (on windows systems) #### Version - 3.0.5.0 - 12/22/2022 * Add support for https://www.nexusmods.com/site hosted mods. diff --git a/Wabbajack.Common/Ext.cs b/Wabbajack.Common/Ext.cs index 528ab80c..8d9aa07f 100644 --- a/Wabbajack.Common/Ext.cs +++ b/Wabbajack.Common/Ext.cs @@ -25,4 +25,5 @@ public static class Ext public static Extension ModlistMetadataExtension = new(".modlist_metadata"); public static Extension Txt = new(".txt"); public static Extension Webp = new(".webp"); + public static Extension Png = new(".png"); } \ No newline at end of file diff --git a/Wabbajack.Compiler.Test/SanityTests.cs b/Wabbajack.Compiler.Test/SanityTests.cs index 4154400e..b3c80812 100644 --- a/Wabbajack.Compiler.Test/SanityTests.cs +++ b/Wabbajack.Compiler.Test/SanityTests.cs @@ -28,10 +28,13 @@ public class CompilerSanityTests : IAsyncLifetime private readonly IServiceProvider _serviceProvider; private Mod _mod; private ModList? _modlist; + private readonly IImageLoader _imageLoader; public CompilerSanityTests(ILogger logger, IServiceProvider serviceProvider, FileExtractor.FileExtractor fileExtractor, - TemporaryFileManager manager, ParallelOptions parallelOptions) + TemporaryFileManager manager, + ParallelOptions parallelOptions, + IImageLoader imageLoader) { _logger = logger; _serviceProvider = serviceProvider; @@ -40,6 +43,7 @@ public class CompilerSanityTests : IAsyncLifetime _fileExtractor = fileExtractor; _manager = manager; _parallelOptions = parallelOptions; + _imageLoader = imageLoader; } @@ -195,12 +199,12 @@ public class CompilerSanityTests : IAsyncLifetime foreach (var file in _mod.FullPath.EnumerateFiles()) { - var oldState = await ImageLoader.Load(file); + var oldState = await _imageLoader.Load(file); Assert.NotEqual(DXGI_FORMAT.UNKNOWN, oldState.Format); _logger.LogInformation("Recompressing {file}", file.FileName); - await ImageLoader.Recompress(file, 512, 512, DXGI_FORMAT.BC7_UNORM, file, CancellationToken.None); + await _imageLoader.Recompress(file, 512, 512, DXGI_FORMAT.BC7_UNORM, file, CancellationToken.None); - var state = await ImageLoader.Load(file); + var state = await _imageLoader.Load(file); Assert.Equal(DXGI_FORMAT.BC7_UNORM, state.Format); } diff --git a/Wabbajack.Compiler/ACompiler.cs b/Wabbajack.Compiler/ACompiler.cs index 6665c2b3..d983875e 100644 --- a/Wabbajack.Compiler/ACompiler.cs +++ b/Wabbajack.Compiler/ACompiler.cs @@ -17,6 +17,7 @@ using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.JsonConverters; using Wabbajack.FileExtractor.ExtractedFiles; +using Wabbajack.Hashing.PHash; using Wabbajack.Hashing.xxHash64; using Wabbajack.Installer; using Wabbajack.Networking.WabbajackClientApi; @@ -64,7 +65,8 @@ public abstract class ACompiler TemporaryFileManager manager, CompilerSettings settings, ParallelOptions parallelOptions, DownloadDispatcher dispatcher, Client wjClient, IGameLocator locator, DTOSerializer dtos, IResource compilerLimiter, - IBinaryPatchCache patchCache) + IBinaryPatchCache patchCache, + IImageLoader imageLoader) { CompilerLimiter = compilerLimiter; _logger = logger; @@ -83,9 +85,12 @@ public abstract class ACompiler _patchOptions = new ConcurrentDictionary(); _sourceFileLinks = new ConcurrentDictionary(); _patchCache = patchCache; + ImageLoader = imageLoader; _updateStopWatch = new Stopwatch(); } + public IImageLoader ImageLoader { get; } + protected long MaxSteps { get; set; } public CompilerSettings Settings diff --git a/Wabbajack.Compiler/CompilationSteps/MatchSimilarTextures.cs b/Wabbajack.Compiler/CompilationSteps/MatchSimilarTextures.cs index 68f87d93..259a100f 100644 --- a/Wabbajack.Compiler/CompilationSteps/MatchSimilarTextures.cs +++ b/Wabbajack.Compiler/CompilationSteps/MatchSimilarTextures.cs @@ -34,7 +34,7 @@ public class MatchSimilarTextures : ACompilationStep _compiler._logger.LogInformation("Looking for texture match for {source}", source.File.FullPath); (float Similarity, VirtualFile File) found = _byName[source.Path.FileNameWithoutExtension] .Select(f => ( - ImageLoader.ComputeDifference(f.ImageState!.PerceptualHash, source.File.ImageState.PerceptualHash), + IImageLoader.ComputeDifference(f.ImageState!.PerceptualHash, source.File.ImageState.PerceptualHash), f)) .Select(f => { return f; }) .OrderByDescending(f => f.Item1) @@ -58,7 +58,7 @@ public class MatchSimilarTextures : ACompilationStep from mainMatch in _byName[mainFile.FullPath.FileName.FileNameWithoutExtension] where mainMatch.ImageState != null where mainFile.ImageState != null - let similarity = ImageLoader.ComputeDifference(mainFile.ImageState!.PerceptualHash, + let similarity = IImageLoader.ComputeDifference(mainFile.ImageState!.PerceptualHash, mainMatch.ImageState!.PerceptualHash) where similarity >= PerceptualTolerance orderby similarity descending diff --git a/Wabbajack.Compiler/MO2Compiler.cs b/Wabbajack.Compiler/MO2Compiler.cs index 00bb056a..24e7bc2e 100644 --- a/Wabbajack.Compiler/MO2Compiler.cs +++ b/Wabbajack.Compiler/MO2Compiler.cs @@ -14,6 +14,7 @@ using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Hashing.PHash; using Wabbajack.Installer; using Wabbajack.Networking.WabbajackClientApi; using Wabbajack.Paths; @@ -30,9 +31,10 @@ public class MO2Compiler : ACompiler TemporaryFileManager manager, CompilerSettings settings, ParallelOptions parallelOptions, DownloadDispatcher dispatcher, Client wjClient, IGameLocator locator, DTOSerializer dtos, IResource compilerLimiter, - IBinaryPatchCache patchCache) : + IBinaryPatchCache patchCache, + IImageLoader imageLoader) : base(logger, extractor, hashCache, vfs, manager, settings, parallelOptions, dispatcher, wjClient, locator, dtos, - compilerLimiter, patchCache) + compilerLimiter, patchCache, imageLoader) { MaxSteps = 14; } @@ -51,7 +53,8 @@ public class MO2Compiler : ACompiler provider.GetRequiredService(), provider.GetRequiredService(), provider.GetRequiredService>(), - provider.GetRequiredService()); + provider.GetRequiredService(), + provider.GetRequiredService()); } public CompilerSettings Mo2Settings => (CompilerSettings) Settings; diff --git a/Wabbajack.Hashing.PHash.Test/FileLoadingTests.cs b/Wabbajack.Hashing.PHash.Test/FileLoadingTests.cs index dd8b76ef..1db34a5e 100644 --- a/Wabbajack.Hashing.PHash.Test/FileLoadingTests.cs +++ b/Wabbajack.Hashing.PHash.Test/FileLoadingTests.cs @@ -1,4 +1,6 @@ +using System; using System.IO; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; using FluentAssertions; @@ -10,8 +12,31 @@ using Xunit; namespace Wabbajack.Hashing.PHash.Test; -public class FileLoadingTests +public class FileLoadingTests : IAsyncDisposable { + private readonly IImageLoader[] _imageLoaders; + private readonly TemporaryFileManager _tmp; + + public FileLoadingTests() + { + _tmp = new TemporaryFileManager(); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { + _imageLoaders = new IImageLoader[] + { + new CrossPlatformImageLoader(), + new TexConvImageLoader(_tmp) + }; + } + else + { + _imageLoaders = new IImageLoader[] + { + new CrossPlatformImageLoader(), + }; + } + } + [Theory] [InlineData("test-dxt5.dds", 1.0f)] [InlineData("test-dxt5-recompressed.dds", 1f)] @@ -19,40 +44,47 @@ public class FileLoadingTests [InlineData("test-dxt5-small-bc7-vflip.dds", 0.189f)] public async Task LoadAllFiles(string file, float difference) { - var baseState = - await ImageLoader.Load("TestData/test-dxt5.dds".ToRelativePath().RelativeTo(KnownFolders.EntryPoint)); - var state = await ImageLoader.Load("TestData".ToRelativePath().Combine(file) - .RelativeTo(KnownFolders.EntryPoint)); + foreach (var imageLoader in _imageLoaders) + { + var baseState = + await imageLoader.Load("TestData/test-dxt5.dds".ToRelativePath().RelativeTo(KnownFolders.EntryPoint)); + var state = await imageLoader.Load("TestData".ToRelativePath().Combine(file) + .RelativeTo(KnownFolders.EntryPoint)); - Assert.NotEqual(DXGI_FORMAT.UNKNOWN, baseState.Format); + Assert.NotEqual(DXGI_FORMAT.UNKNOWN, baseState.Format); - Assert.Equal(difference, - ImagePhash.GetCrossCorrelation( - new Digest {Coefficients = baseState.PerceptualHash.Data}, - new Digest {Coefficients = state.PerceptualHash.Data}), - 1.0); + Assert.Equal(difference, + ImagePhash.GetCrossCorrelation( + new Digest { Coefficients = baseState.PerceptualHash.Data }, + new Digest { Coefficients = state.PerceptualHash.Data }), + 1.0); + } } [Fact] public async Task CanConvertCubeMaps() { - // File used here via re-upload permissions found on the mod's Nexus page: - // https://www.nexusmods.com/fallout4/mods/43458?tab=description - // Used for testing purposes only - var path = "TestData/WindowDisabled_CGPlayerHouseCube.dds".ToRelativePath().RelativeTo(KnownFolders.EntryPoint); + foreach (var imageLoader in _imageLoaders) + { + // File used here via re-upload permissions found on the mod's Nexus page: + // https://www.nexusmods.com/fallout4/mods/43458?tab=description + // Used for testing purposes only + var path = "TestData/WindowDisabled_CGPlayerHouseCube.dds".ToRelativePath().RelativeTo(KnownFolders.EntryPoint); - var baseState = await ImageLoader.Load(path); - baseState.Height.Should().Be(128); - baseState.Width.Should().Be(128); - //baseState.Frames.Should().Be(6); - - using var ms = new MemoryStream(); - await using var ins = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read); - await ImageLoader.Recompress(ins, 128, 128, DXGI_FORMAT.BC1_UNORM, ms, CancellationToken.None, leaveOpen:true); - ms.Length.Should().Be(ins.Length); - - + var baseState = await imageLoader.Load(path); + baseState.Height.Should().Be(128); + baseState.Width.Should().Be(128); + //baseState.Frames.Should().Be(6); + using var ms = new MemoryStream(); + await using var ins = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read); + await imageLoader.Recompress(ins, 128, 128, DXGI_FORMAT.BC1_UNORM, ms, CancellationToken.None, leaveOpen:true); + ms.Length.Should().Be(ins.Length); + } } + public async ValueTask DisposeAsync() + { + await _tmp.DisposeAsync(); + } } \ No newline at end of file diff --git a/Wabbajack.Hashing.PHash/Image.cs b/Wabbajack.Hashing.PHash/CrossPlatformImageLoader.cs similarity index 90% rename from Wabbajack.Hashing.PHash/Image.cs rename to Wabbajack.Hashing.PHash/CrossPlatformImageLoader.cs index 75d22e02..eb6cae99 100644 --- a/Wabbajack.Hashing.PHash/Image.cs +++ b/Wabbajack.Hashing.PHash/CrossPlatformImageLoader.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; using BCnEncoder.Decoder; @@ -13,21 +14,22 @@ using Shipwreck.Phash.Imaging; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Processing; +using Wabbajack.Common; using Wabbajack.DTOs.Texture; using Wabbajack.Paths; using Wabbajack.Paths.IO; namespace Wabbajack.Hashing.PHash; -public class ImageLoader +public class CrossPlatformImageLoader : IImageLoader { - public static async ValueTask Load(AbsolutePath path) + public async ValueTask Load(AbsolutePath path) { await using var fs = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read); return await Load(fs); } - public static async ValueTask Load(Stream stream) + public async ValueTask Load(Stream stream) { var decoder = new BcDecoder(); var ddsFile = DdsFile.Load(stream); @@ -57,8 +59,8 @@ public class ImageLoader new Digest {Coefficients = a.Data}, new Digest {Coefficients = b.Data}); } - - public static async Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format, + + public async Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format, AbsolutePath output, CancellationToken token) { @@ -67,24 +69,23 @@ public class ImageLoader await Recompress(new MemoryStream(inData), width, height, format, outStream, token); } - public static async Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output, + public async Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output, CancellationToken token, bool leaveOpen = false) { var decoder = new BcDecoder(); var ddsFile = DdsFile.Load(input); - + if (!leaveOpen) await input.DisposeAsync(); var faces = new List>(); - + var origFormat = ddsFile.dx10Header.dxgiFormat == DxgiFormat.DxgiFormatUnknown ? ddsFile.header.ddsPixelFormat.DxgiFormat : ddsFile.dx10Header.dxgiFormat; foreach (var face in ddsFile.Faces) { - - var data = await decoder.DecodeRawToImageRgba32Async(face.MipMaps[0].Data, + var data = await decoder.DecodeRawToImageRgba32Async(face.MipMaps[0].Data, (int)face.Width, (int)face.Height, ToCompressionFormat((DXGI_FORMAT)origFormat), token); data.Mutate(x => x.Resize(width, height, KnownResamplers.Welch)); @@ -101,7 +102,7 @@ public class ImageLoader FileFormat = OutputFileFormat.Dds } }; - + switch (faces.Count) { case 1: @@ -114,11 +115,11 @@ public class ImageLoader default: throw new NotImplementedException($"Can't encode dds with {faces.Count} faces"); } - + if (!leaveOpen) await output.DisposeAsync(); } - + public static CompressionFormat ToCompressionFormat(DXGI_FORMAT dx) { return dx switch diff --git a/Wabbajack.Hashing.PHash/IImageLoader.cs b/Wabbajack.Hashing.PHash/IImageLoader.cs new file mode 100644 index 00000000..93898e43 --- /dev/null +++ b/Wabbajack.Hashing.PHash/IImageLoader.cs @@ -0,0 +1,27 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Shipwreck.Phash; +using Wabbajack.DTOs.Texture; +using Wabbajack.Paths; + +namespace Wabbajack.Hashing.PHash; + +public interface IImageLoader +{ + public ValueTask Load(AbsolutePath path); + public ValueTask Load(Stream stream); + public static float ComputeDifference(DTOs.Texture.PHash a, DTOs.Texture.PHash b) + { + return ImagePhash.GetCrossCorrelation( + new Digest {Coefficients = a.Data}, + new Digest {Coefficients = b.Data}); + } + + public Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format, + AbsolutePath output, + CancellationToken token); + + public Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output, + CancellationToken token, bool leaveOpen = false); +} \ No newline at end of file diff --git a/Wabbajack.Hashing.PHash/TexConvImageLoader.cs b/Wabbajack.Hashing.PHash/TexConvImageLoader.cs new file mode 100644 index 00000000..a80069ab --- /dev/null +++ b/Wabbajack.Hashing.PHash/TexConvImageLoader.cs @@ -0,0 +1,160 @@ +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Linq; +using System.Reactive.Linq; +using System.Threading; +using System.Threading.Tasks; +using Shipwreck.Phash; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Processing; +using Wabbajack.Common; +using Wabbajack.Common.FileSignatures; +using Wabbajack.DTOs.Texture; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; + + +namespace Wabbajack.Hashing.PHash; + +public class TexConvImageLoader : IImageLoader +{ + private readonly SignatureChecker _sigs; + private readonly TemporaryFileManager _tempManager; + + public TexConvImageLoader(TemporaryFileManager manager) + { + _tempManager = manager; + _sigs = new SignatureChecker(FileType.DDS, FileType.PNG, FileType.JPG, FileType.BMP); + } + + public async ValueTask Load(AbsolutePath path) + { + return await GetState(path); + } + + public async ValueTask Load(Stream stream) + { + + var ext = await DetermineType(stream); + var temp = _tempManager.CreateFile(ext); + await using var fs = temp.Path.Open(FileMode.Create, FileAccess.Write, FileShare.None); + await stream.CopyToAsync(fs); + fs.Close(); + return await GetState(temp.Path); + } + + private async Task DetermineType(Stream stream) + { + var sig = await _sigs.MatchesAsync(stream); + + var ext = new Extension(".tga"); + if (sig != null) + ext = new Extension("." + Enum.GetName(sig.Value)); + + stream.Position = 0; + return ext; + } + + public async Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format, AbsolutePath output, + CancellationToken token) + { + var outFolder = _tempManager.CreateFolder(); + var outFile = input.FileName.RelativeTo(outFolder.Path); + await ConvertImage(input, outFolder.Path, width, height, format, input.Extension); + await outFile.MoveToAsync(output, token: token, overwrite:true); + } + + public async Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output, CancellationToken token, + bool leaveOpen = false) + { + var type = await DetermineType(input); + await using var toFolder = _tempManager.CreateFolder(); + await using var fromFile = _tempManager.CreateFile(type); + await input.CopyToAsync(fromFile.Path, token); + var toFile = fromFile.Path.FileName.RelativeTo(toFolder); + + await ConvertImage(fromFile.Path, toFolder.Path, width, height, format, type); + await using var fs = toFile.Open(FileMode.Open, FileAccess.Read, FileShare.Read); + await fs.CopyToAsync(output, token); + } + + + public async Task ConvertImage(AbsolutePath from, AbsolutePath toFolder, int w, int h, DXGI_FORMAT format, Extension fileFormat) + { + // User isn't renaming the file, so we don't have to create a temporary folder + var ph = new ProcessHelper + { + Path = @"Tools\texconv.exe".ToRelativePath().RelativeTo(KnownFolders.EntryPoint), + Arguments = new object[] {from, "-ft", fileFormat.ToString()[1..], "-f", format, "-o", toFolder, "-w", w, "-h", h, "-if", "CUBIC", "-singleproc"}, + ThrowOnNonZeroExitCode = true, + LogError = true + }; + await ph.Start(); + + } + + public async Task ConvertImage(Stream from, ImageState state, Extension ext, AbsolutePath to) + { + await using var tmpFile = _tempManager.CreateFolder(); + var inFile = to.FileName.RelativeTo(tmpFile.Path); + await inFile.WriteAllAsync(from, CancellationToken.None); + await ConvertImage(inFile, to.Parent, state.Width, state.Height, state.Format, ext); + } + + // Internals + public async Task GetState(AbsolutePath path) + { + try + { + var ph = new ProcessHelper + { + Path = @"Tools\texdiag.exe".ToRelativePath().RelativeTo(KnownFolders.EntryPoint), + Arguments = new object[] {"info", path, "-nologo"}, + ThrowOnNonZeroExitCode = true, + LogError = true + }; + var lines = new ConcurrentStack(); + using var _ = ph.Output.Where(p => p.Type == ProcessHelper.StreamType.Output) + .Select(p => p.Line) + .Where(p => p.Contains(" = ")) + .Subscribe(l => lines.Push(l)); + await ph.Start(); + + var data = lines.Select(l => + { + var split = l.Split(" = "); + return (split[0].Trim(), split[1].Trim()); + }).ToDictionary(p => p.Item1, p => p.Item2); + + return new ImageState + { + Width = int.Parse(data["width"]), + Height = int.Parse(data["height"]), + Format = Enum.Parse(data["format"]), + PerceptualHash = await GetPHash(path) + }; + } + catch (Exception ex) + { + throw; + } + } + + + public async Task GetPHash(AbsolutePath path) + { + if (!path.FileExists()) + throw new FileNotFoundException($"Can't hash non-existent file {path}"); + + await using var tmp = _tempManager.CreateFolder(); + await ConvertImage(path, tmp.Path, 512, 512, DXGI_FORMAT.R8G8B8A8_UNORM, Ext.Png); + + using var img = await Image.LoadAsync(path.FileName.RelativeTo(tmp.Path).ReplaceExtension(Ext.Png).ToString()); + img.Mutate(x => x.Resize(512, 512, KnownResamplers.Welch).Grayscale(GrayscaleMode.Bt601)); + + return new DTOs.Texture.PHash(ImagePhash.ComputeDigest(new CrossPlatformImageLoader.ImageBitmap((Image)img)).Coefficients); + } + +} \ No newline at end of file diff --git a/Wabbajack.Hashing.PHash/Tools/texconv.exe b/Wabbajack.Hashing.PHash/Tools/texconv.exe new file mode 100644 index 00000000..6ff20332 Binary files /dev/null and b/Wabbajack.Hashing.PHash/Tools/texconv.exe differ diff --git a/Wabbajack.Hashing.PHash/Tools/texdiag.exe b/Wabbajack.Hashing.PHash/Tools/texdiag.exe new file mode 100644 index 00000000..604c5d37 Binary files /dev/null and b/Wabbajack.Hashing.PHash/Tools/texdiag.exe differ diff --git a/Wabbajack.Hashing.PHash/Wabbajack.Hashing.PHash.csproj b/Wabbajack.Hashing.PHash/Wabbajack.Hashing.PHash.csproj index 2197a721..0d66a3e1 100644 --- a/Wabbajack.Hashing.PHash/Wabbajack.Hashing.PHash.csproj +++ b/Wabbajack.Hashing.PHash/Wabbajack.Hashing.PHash.csproj @@ -13,9 +13,23 @@ + + + + + + + + PreserveNewest + + + PreserveNewest + + + diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index 1fa6b156..85ba5ee7 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -75,7 +75,8 @@ public abstract class AInstaller DownloadDispatcher downloadDispatcher, ParallelOptions parallelOptions, IResource limiter, - Client wjClient) + Client wjClient, + IImageLoader imageLoader) { _limiter = limiter; _manager = new TemporaryFileManager(config.Install.Combine("__temp__")); @@ -90,8 +91,11 @@ public abstract class AInstaller _parallelOptions = parallelOptions; _gameLocator = gameLocator; _wjClient = wjClient; + ImageLoader = imageLoader; } + public IImageLoader ImageLoader { get; } + protected long MaxSteps { get; set; } public Dictionary HashedArchives { get; set; } = new(); diff --git a/Wabbajack.Installer/StandardInstaller.cs b/Wabbajack.Installer/StandardInstaller.cs index 2f35f818..97beffbd 100644 --- a/Wabbajack.Installer/StandardInstaller.cs +++ b/Wabbajack.Installer/StandardInstaller.cs @@ -23,6 +23,7 @@ using Wabbajack.DTOs.BSA.FileStates; using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Hashing.PHash; using Wabbajack.Hashing.xxHash64; using Wabbajack.Installer.Utilities; using Wabbajack.Networking.WabbajackClientApi; @@ -40,9 +41,9 @@ public class StandardInstaller : AInstaller InstallerConfiguration config, IGameLocator gameLocator, FileExtractor.FileExtractor extractor, DTOSerializer jsonSerializer, Context vfs, FileHashCache fileHashCache, - DownloadDispatcher downloadDispatcher, ParallelOptions parallelOptions, IResource limiter, Client wjClient) : + DownloadDispatcher downloadDispatcher, ParallelOptions parallelOptions, IResource limiter, Client wjClient, IImageLoader imageLoader) : base(logger, config, gameLocator, extractor, jsonSerializer, vfs, fileHashCache, downloadDispatcher, - parallelOptions, limiter, wjClient) + parallelOptions, limiter, wjClient, imageLoader) { MaxSteps = 14; } @@ -59,7 +60,8 @@ public class StandardInstaller : AInstaller provider.GetRequiredService(), provider.GetRequiredService(), provider.GetRequiredService>(), - provider.GetRequiredService()); + provider.GetRequiredService(), + provider.GetRequiredService()); } public override async Task Begin(CancellationToken token) diff --git a/Wabbajack.Paths.IO/AbsolutePathExtensions.cs b/Wabbajack.Paths.IO/AbsolutePathExtensions.cs index 27cd9e90..b7980db9 100644 --- a/Wabbajack.Paths.IO/AbsolutePathExtensions.cs +++ b/Wabbajack.Paths.IO/AbsolutePathExtensions.cs @@ -252,6 +252,18 @@ public static class AbsolutePathExtensions return file.ToString(); } + public static async Task CopyToAsync(this Stream from, AbsolutePath path, CancellationToken token = default) + { + await using var to = path.Open(FileMode.Create, FileAccess.Write, FileShare.None); + await from.CopyToAsync(to, token); + } + + public static async Task CopyToAsync(this AbsolutePath from, Stream to, CancellationToken token = default) + { + await using var fromStream = from.Open(FileMode.Open, FileAccess.Read, FileShare.Read); + await fromStream.CopyToAsync(to, token); + } + #region Directories public static void CreateDirectory(this AbsolutePath path) diff --git a/Wabbajack.Paths/AbsolutePath.cs b/Wabbajack.Paths/AbsolutePath.cs index 352708ef..d200c5a1 100644 --- a/Wabbajack.Paths/AbsolutePath.cs +++ b/Wabbajack.Paths/AbsolutePath.cs @@ -85,7 +85,12 @@ public struct AbsolutePath : IPath, IComparable, IEquatable + /// Returns a new path that is this path with the extension changed. + /// + /// + /// + public readonly AbsolutePath ReplaceExtension(Extension newExtension) { var paths = new string[Parts.Length]; Array.Copy(Parts, paths, paths.Length); @@ -94,7 +99,7 @@ public struct AbsolutePath : IPath, IComparable, IEquatable, IComparable(); else service.AddAllSingleton(); + + // ImageLoader + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + service.AddSingleton(); + else + service.AddSingleton(); // Installer/Compiler Configuration service.AddScoped(); diff --git a/Wabbajack.VFS.Test/Startup.cs b/Wabbajack.VFS.Test/Startup.cs index 7549fe80..44a26e45 100644 --- a/Wabbajack.VFS.Test/Startup.cs +++ b/Wabbajack.VFS.Test/Startup.cs @@ -1,7 +1,9 @@ +using System.Runtime.InteropServices; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Wabbajack.DTOs; +using Wabbajack.Hashing.PHash; using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; using Wabbajack.VFS.Interfaces; @@ -28,6 +30,13 @@ public class Startup .AddAllSingleton, Resource>( s => new ("File Hash Cache", 2)); + + // ImageLoader + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + service.AddSingleton(); + else + service.AddSingleton(); + // Keep this fixed at 2 so that we can detect deadlocks in the VFS parallelOptions service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = 2}); diff --git a/Wabbajack.VFS/Context.cs b/Wabbajack.VFS/Context.cs index 1461ed80..8facf752 100644 --- a/Wabbajack.VFS/Context.cs +++ b/Wabbajack.VFS/Context.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.FileExtractor.ExtractedFiles; +using Wabbajack.Hashing.PHash; using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; using Wabbajack.Paths.IO; @@ -32,7 +33,8 @@ public class Context public Context(ILogger logger, ParallelOptions parallelOptions, TemporaryFileManager manager, IVfsCache vfsCache, - FileHashCache hashCache, IResource limiter, IResource hashLimiter, FileExtractor.FileExtractor extractor) + FileHashCache hashCache, IResource limiter, IResource hashLimiter, + FileExtractor.FileExtractor extractor, IImageLoader imageLoader) { Limiter = limiter; HashLimiter = hashLimiter; @@ -42,16 +44,18 @@ public class Context VfsCache = vfsCache; HashCache = hashCache; _parallelOptions = parallelOptions; + ImageLoader = imageLoader; } public Context WithTemporaryFileManager(TemporaryFileManager manager) { return new Context(Logger, _parallelOptions, manager, VfsCache, HashCache, Limiter, HashLimiter, - Extractor.WithTemporaryFileManager(manager)); + Extractor.WithTemporaryFileManager(manager), ImageLoader); } public IndexRoot Index { get; private set; } = IndexRoot.Empty; + public IImageLoader ImageLoader { get; } public async Task AddRoot(AbsolutePath root, CancellationToken token) { diff --git a/Wabbajack.VFS/VirtualFile.cs b/Wabbajack.VFS/VirtualFile.cs index f15e43d6..ea2324c8 100644 --- a/Wabbajack.VFS/VirtualFile.cs +++ b/Wabbajack.VFS/VirtualFile.cs @@ -204,7 +204,7 @@ public class VirtualFile if (TextureExtensions.Contains(relPath.FileName.Extension) && await DDSSig.MatchesAsync(stream) != null) try { - self.ImageState = await ImageLoader.Load(stream); + self.ImageState = await context.ImageLoader.Load(stream); if (job != null) { job.Size += self.Size;