Merge pull request #1504 from wabbajack-tools/image-hashing

Image hashing

Wabbajack.Common/Consts.cs

@@ -172,5 +172,6 @@ namespace Wabbajack.Common
         
         public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2;
 
+        public static HashSet<Extension> TextureExtensions = new() {new Extension(".dds"), new Extension(".tga")};
     }
 }

Wabbajack.Common/Json.cs

@@ -128,7 +128,7 @@ namespace Wabbajack.Common
                 return Hash.FromBase64((string)reader.Value!);
             }
         }
-
+        
         public class RelativePathConverter : JsonConverter<RelativePath>
         {
             public override void WriteJson(JsonWriter writer, RelativePath value, JsonSerializer serializer)

Wabbajack.ImageHashing.Test/ImageLoadingTests.cs

@@ -0,0 +1,58 @@
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Xunit;
+
+namespace Wabbajack.ImageHashing.Test
+{
+    public class ImageLoadingTests
+    {
+        [Fact]
+        public async Task CanLoadAndCompareDDSImages()
+        {
+            var file1 = DDSImage.FromFile(AbsolutePath.EntryPoint.Combine("Resources", "test-dxt5.dds"));
+            var hash1 = file1.PerceptionHash();
+            
+            var file2 = DDSImage.FromFile(AbsolutePath.EntryPoint.Combine("Resources", "test-dxt5.dds"));
+            var hash2 = file2.PerceptionHash();
+            
+            Assert.Equal(1, hash1.Similarity(hash2));
+        }
+        
+        [Fact]
+        public async Task CanLoadAndCompareResizedImage()
+        {
+            var file1 = DDSImage.FromFile(AbsolutePath.EntryPoint.Combine("Resources", "test-dxt5.dds"));
+            var hash1 = file1.PerceptionHash();
+            
+            var file2 = DDSImage.FromFile(AbsolutePath.EntryPoint.Combine("Resources", "test-dxt5-small-bc7.dds"));
+            var hash2 = file2.PerceptionHash();
+            
+            Assert.Equal(0.956666886806488, hash1.Similarity(hash2));
+        }
+        
+                
+        [Fact]
+        public async Task CanLoadAndCompareResizedVFlipImage()
+        {
+            var file1 = DDSImage.FromFile(AbsolutePath.EntryPoint.Combine("Resources", "test-dxt5.dds"));
+            var hash1 = file1.PerceptionHash();
+            
+            var file2 = DDSImage.FromFile(AbsolutePath.EntryPoint.Combine("Resources", "test-dxt5-small-bc7-vflip.dds"));
+            var hash2 = file2.PerceptionHash();
+            
+            Assert.Equal(0.2465425431728363, hash1.Similarity(hash2));
+        }
+        
+        [Fact]
+        public async Task CanLoadAndCompareRecompressedImage()
+        {
+            var file1 = DDSImage.FromFile(AbsolutePath.EntryPoint.Combine("Resources", "test-dxt5.dds"));
+            var hash1 = file1.PerceptionHash();
+
+            var file2 = DDSImage.FromFile(AbsolutePath.EntryPoint.Combine("Resources", "test-dxt5-recompressed.dds"));
+            var hash2 = file2.PerceptionHash();
+
+            Assert.Equal(0.9999724626541138, hash1.Similarity(hash2));
+        }
+    }
+}

Wabbajack.ImageHashing.Test/Wabbajack.ImageHashing.Test.csproj

@@ -0,0 +1,43 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net5.0-windows</TargetFramework>
+
+        <IsPackable>false</IsPackable>
+
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
+        <PackageReference Include="xunit" Version="2.4.1" />
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+            <PrivateAssets>all</PrivateAssets>
+        </PackageReference>
+        <PackageReference Include="coverlet.collector" Version="1.3.0">
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+            <PrivateAssets>all</PrivateAssets>
+        </PackageReference>
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Wabbajack.ImageHashing\Wabbajack.ImageHashing.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <None Update="Resources\test-dxt5.dds">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
+      <None Update="Resources\test-dxt5-small-bc7.dds">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
+      <None Update="Resources\test-dxt5-small-bc7-vflip.dds">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
+      <None Update="Resources\test-dxt5-recompressed.dds">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
+    </ItemGroup>
+
+</Project>

Wabbajack.ImageHashing/DDSImage.cs

@@ -0,0 +1,208 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Numerics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Threading.Tasks;
+using DirectXTexNet;
+using Shipwreck.Phash;
+using Shipwreck.Phash.Imaging;
+using Wabbajack.Common;
+
+namespace Wabbajack.ImageHashing
+{
+    public class DDSImage : IDisposable
+    {
+        private DDSImage(ScratchImage img, TexMetadata metadata, Extension ext)
+        {
+            _image = img;
+            _metaData = metadata;
+            _extension = ext;
+        }
+
+        private static Extension DDSExtension = new(".dds");
+        private static Extension TGAExtension = new(".tga");
+        private ScratchImage _image;
+        private TexMetadata _metaData;
+
+        public static DDSImage FromFile(AbsolutePath file)
+        {
+            if (file.Extension != DDSExtension)
+                throw new Exception("File does not end in DDS");
+
+            var img = TexHelper.Instance.LoadFromDDSFile(file.ToString(), DDS_FLAGS.NONE);
+            
+            return new DDSImage(img, img.GetMetadata(), new Extension(".dds"));
+        }
+
+        public static DDSImage FromDDSMemory(byte[] data)
+        {
+            unsafe
+            {
+                fixed (byte* ptr = data)
+                {
+                    var img = TexHelper.Instance.LoadFromDDSMemory((IntPtr)ptr, data.Length, DDS_FLAGS.NONE);
+                    return new DDSImage(img, img.GetMetadata(), new Extension(".dds"));
+                }
+            }
+        }
+        
+        public static DDSImage FromTGAMemory(byte[] data)
+        {
+            unsafe
+            {
+                fixed (byte* ptr = data)
+                {
+                    var img = TexHelper.Instance.LoadFromTGAMemory((IntPtr)ptr, data.Length);
+                    return new DDSImage(img, img.GetMetadata(), new Extension(".tga"));
+                }
+            }
+        }
+        
+        
+        public static async Task<DDSImage> FromStream(Stream stream, IPath arg1Name)
+        {
+            var data = await stream.ReadAllAsync();
+            if (arg1Name.FileName.Extension == DDSExtension)
+                return FromDDSMemory(data);
+            if (arg1Name.FileName.Extension == TGAExtension)
+                return FromTGAMemory(data);
+            throw new NotImplementedException("Only DDS and TGA files supported");
+        }
+
+        public void Dispose()
+        {
+            if (!_image.IsDisposed) 
+                _image.Dispose();
+        }
+
+        public int Width => _metaData.Width;
+        public int Height => _metaData.Height;
+
+        // Destructively resize a Image
+        public void ResizeRecompressAndSave(int width, int height, DXGI_FORMAT newFormat, AbsolutePath dest)
+        {
+            ScratchImage? resized = default;
+            try
+            {
+                // First we resize the image, so that changes due to image scaling matter less in the final hash
+                if (CompressedTypes.Contains(_metaData.Format))
+                {
+                    using var decompressed = _image.Decompress(DXGI_FORMAT.UNKNOWN);
+                    resized = decompressed.Resize(width, height, TEX_FILTER_FLAGS.DEFAULT);
+                }
+                else
+                {
+                    resized = _image.Resize(width, height, TEX_FILTER_FLAGS.DEFAULT);
+                }
+
+                using var compressed = resized.Compress(newFormat, TEX_COMPRESS_FLAGS.BC7_QUICK, 0.5f);
+
+                if (dest.Extension == new Extension(".dds"))
+                {
+                    compressed.SaveToDDSFile(DDS_FLAGS.NONE, dest.ToString());
+                }
+            }
+            finally
+            {
+                resized?.Dispose();
+            }
+        }
+
+        private static HashSet<DXGI_FORMAT> CompressedTypes = new HashSet<DXGI_FORMAT>()
+        {
+            DXGI_FORMAT.BC1_TYPELESS,
+            DXGI_FORMAT.BC1_UNORM,
+            DXGI_FORMAT.BC1_UNORM_SRGB,
+            DXGI_FORMAT.BC2_TYPELESS,
+            DXGI_FORMAT.BC2_UNORM,
+            DXGI_FORMAT.BC2_UNORM_SRGB,
+            DXGI_FORMAT.BC3_TYPELESS,
+            DXGI_FORMAT.BC3_UNORM,
+            DXGI_FORMAT.BC3_UNORM_SRGB,
+            DXGI_FORMAT.BC4_TYPELESS,
+            DXGI_FORMAT.BC4_UNORM,
+            DXGI_FORMAT.BC4_SNORM,
+            DXGI_FORMAT.BC5_TYPELESS,
+            DXGI_FORMAT.BC5_UNORM,
+            DXGI_FORMAT.BC5_SNORM,
+            DXGI_FORMAT.BC6H_TYPELESS,
+            DXGI_FORMAT.BC6H_UF16,
+            DXGI_FORMAT.BC6H_SF16,
+            DXGI_FORMAT.BC7_TYPELESS,
+            DXGI_FORMAT.BC7_UNORM,
+            DXGI_FORMAT.BC7_UNORM_SRGB,
+        };
+
+        private Extension _extension;
+
+        public ImageState ImageState()
+        {
+            return new()
+            {
+                Width = _metaData.Width,
+                Height = _metaData.Height,
+                Format = _metaData.Format,
+                PerceptualHash = PerceptionHash()
+            };
+        }
+
+        public PHash PerceptionHash()
+        {
+            ScratchImage? resized = default;
+            try
+            {
+                // First we resize the image, so that changes due to image scaling matter less in the final hash
+                if (CompressedTypes.Contains(_metaData.Format))
+                {
+                    using var decompressed = _image.Decompress(DXGI_FORMAT.UNKNOWN);
+                    resized = decompressed.Resize(512, 512, TEX_FILTER_FLAGS.DEFAULT);
+                }
+                else
+                {
+                    resized = _image.Resize(512, 512, TEX_FILTER_FLAGS.DEFAULT);
+                }
+                
+                var image = new byte[512 * 512];
+
+                unsafe void EvaluatePixels(IntPtr pixels, IntPtr width, IntPtr line)
+                {
+                    float* ptr = (float*)pixels.ToPointer();
+
+                    int widthV = width.ToInt32();
+                    if (widthV != 512) return;
+
+                    var y = line.ToInt32();
+
+                    for (int i = 0; i < widthV; i++)
+                    {
+                        var r = ptr[0] * 0.229f;
+                        var g = ptr[1] * 0.587f;
+                        var b = ptr[2] * 0.114f;
+
+                        var combined = (r + g + b) * 255.0f;
+
+                        image[(y * widthV) + i] = (byte)combined;
+                        ptr += 4;
+                    }
+
+                }
+
+                resized.EvaluateImage(EvaluatePixels);
+
+                var digest = ImagePhash.ComputeDigest(new ByteImage(512, 512, image));
+                return PHash.FromDigest(digest);
+            }
+            finally
+            {
+                resized?.Dispose();
+            }
+        }
+
+        public void ResizeRecompressAndSave(ImageState state, AbsolutePath dest)
+        {
+            ResizeRecompressAndSave(state.Width, state.Height, state.Format, dest);
+        }
+    }
+}

Wabbajack.ImageHashing/IImage.cs

@@ -0,0 +1,6 @@
+using System;
+
+namespace Wabbajack.ImageHashing
+{
+
+}

Wabbajack.ImageHashing/ImageState.cs

@@ -0,0 +1,71 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using DirectXTexNet;
+using Wabbajack.Common;
+using Wabbajack.Common.Serialization.Json;
+
+namespace Wabbajack.ImageHashing
+{
+    [JsonName("ImageState")]
+    public class ImageState
+    {
+        public int Width { get; set; }
+        public int Height { get; set; }
+        public DXGI_FORMAT Format { get; set; }
+        public PHash PerceptualHash { get; set; }
+
+        public static ImageState Read(BinaryReader br)
+        {
+            return new()
+            {
+                Width = br.ReadUInt16(),
+                Height = br.ReadUInt16(),
+                Format = (DXGI_FORMAT)br.ReadByte(),
+                PerceptualHash = PHash.Read(br)
+            };
+        }
+
+        public void Write(BinaryWriter bw)
+        {
+            bw.Write((ushort)Width);
+            bw.Write((ushort)Height);
+            bw.Write((byte)Format);
+            PerceptualHash.Write(bw);
+        }
+
+        public static async Task<ImageState?> FromImageStream(Stream stream, Extension ext, bool takeStreamOwnership = true)
+        {
+            var ms = new MemoryStream();
+            await stream.CopyToAsync(ms);
+            if (takeStreamOwnership) await stream.DisposeAsync();
+
+            DDSImage? img = default;
+            try
+            {
+                if (ext == new Extension(".dds"))
+                    img = DDSImage.FromDDSMemory(ms.GetBuffer());
+                else if (ext == new Extension(".tga"))
+                {
+                    img = DDSImage.FromTGAMemory(ms.GetBuffer());
+                }
+                else
+                {
+                    throw new NotImplementedException("Only DDS and TGA files supported by PHash");
+                }
+
+                return img.ImageState();
+
+            }
+            catch (Exception ex)
+            {
+                Utils.Log($"Error getting ImageState: {ex}");
+                return null;
+            }
+            finally
+            {
+                img?.Dispose();
+            }
+        }
+    }
+}

Wabbajack.ImageHashing/PHash.cs

@@ -0,0 +1,138 @@
+using System;
+using System.Data;
+using System.IO;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Shipwreck.Phash;
+using Wabbajack.Common;
+
+namespace Wabbajack.ImageHashing
+{
+    [JsonConverter(typeof(PHashJsonConverter))]
+    public struct PHash
+    {
+        private const int SIZE = 40;
+        private readonly int _hash;
+
+        public byte[] Data { get; }
+
+        private PHash(byte[] data)
+        {
+            Data = data;
+            if (Data.Length != SIZE)
+                throw new DataException();
+            
+            long h = 0;
+            h |= Data[0];
+            h <<= 8;
+            h |= Data[1];
+            h <<= 8;
+            h |= Data[2];
+            h <<= 8;
+            h |= Data[3];
+            h <<= 8;
+            _hash = (int)h;
+        }
+
+        public static PHash FromBase64(string base64)
+        {
+            var data = base64.FromBase64();
+            if (data.Length != SIZE)
+                throw new DataException();
+            return new PHash(data);
+        }
+        
+        public static PHash Read(BinaryReader br)
+        {
+            return new (br.ReadBytes(SIZE));
+        }
+
+        public void Write(BinaryWriter br)
+        {
+            if (_hash == 0)
+                br.Write(new byte[SIZE]);
+            else 
+                br.Write(Data);
+        }
+        
+        public static PHash FromDigest(Digest digest)
+        {
+            return new(digest.Coefficients);
+        }
+
+        public float Similarity(PHash other)
+        {
+            return ImagePhash.GetCrossCorrelation(this.Data, other.Data);
+        }
+
+        public override string ToString()
+        {
+            return Data.ToBase64();
+        }
+
+        public override int GetHashCode()
+        {
+            long h = 0;
+            h |= Data[0];
+            h <<= 8;
+            h |= Data[1];
+            h <<= 8;
+            h |= Data[2];
+            h <<= 8;
+            h |= Data[3];
+            h <<= 8;
+            return (int)h;
+        }
+
+        public static async Task<PHash> FromStream(Stream stream, Extension ext, bool takeStreamOwnership = true)
+        {
+            try
+            {
+                var ms = new MemoryStream();
+                await stream.CopyToAsync(ms);
+                if (takeStreamOwnership) await stream.DisposeAsync();
+
+                DDSImage img;
+                if (ext == new Extension(".dds"))
+                    img = DDSImage.FromDDSMemory(ms.GetBuffer());
+                else if (ext == new Extension(".tga"))
+                {
+                    img = DDSImage.FromTGAMemory(ms.GetBuffer());
+                }
+                else
+                {
+                    throw new NotImplementedException("Only DDS and TGA files supported by PHash");
+                }
+
+                return img.PerceptionHash();
+            }
+            catch (Exception ex)
+            {
+                Utils.Log($"Error getting PHASH {ex}");
+                return default;
+            }
+        }
+
+        public static async Task<PHash> FromFile(AbsolutePath path)
+        {
+            await using var s = await path.OpenRead();
+            return await FromStream(s, path.Extension);
+
+        }
+    }
+    
+            
+    public class PHashJsonConverter : JsonConverter<PHash>
+    {
+        public override void WriteJson(JsonWriter writer, PHash value, JsonSerializer serializer)
+        {
+            writer.WriteValue(value.Data.ToBase64());
+        }
+
+        public override PHash ReadJson(JsonReader reader, Type objectType, PHash existingValue, bool hasExistingValue,
+            JsonSerializer serializer)
+        {
+            return PHash.FromBase64((string)reader.Value!);
+        }
+    }
+}

Wabbajack.ImageHashing/Wabbajack.ImageHashing.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net5.0-windows</TargetFramework>
+        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="DirectXTexNet" Version="1.0.2" />
+      <PackageReference Include="Shipwreck.Phash" Version="0.5.0" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
+    </ItemGroup>
+
+</Project>

Wabbajack.Lib/AInstaller.cs

@@ -6,6 +6,7 @@ using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Wabbajack.Common;
+using Wabbajack.ImageHashing;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.Validation;
 using Wabbajack.VirtualFileSystem;
@@ -153,6 +154,16 @@ namespace Wabbajack.Lib
                         }
                             break;
 
+                        case TransformedTexture tt:
+                        {
+                            await using var s = await sf.GetStream();
+                            using var img = await DDSImage.FromStream(s, vf.Name);
+                            
+                            img.ResizeRecompressAndSave(tt.ImageState, directive.Directive.To.RelativeTo(OutputFolder));
+
+                        }
+                            break;
+
                         case FromArchive _:
                             if (grouped[vf].Count() == 1)
                             {

Wabbajack.Lib/CompilationSteps/MatchSimilarTextures.cs

@@ -0,0 +1,44 @@
+using System.Linq;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.ImageHashing;
+using Wabbajack.VirtualFileSystem;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class MatchSimilarTextures : ACompilationStep
+    {
+        private ILookup<RelativePath, VirtualFile> _byName;
+        public MatchSimilarTextures(ACompiler compiler) : base(compiler)
+        {
+            _byName = _compiler.IndexedFiles.SelectMany(kv => kv.Value)
+                .Where(f => f.Name.FileName.Extension == DDS)
+                .ToLookup(f => f.Name.FileName.FileNameWithoutExtension);
+        }
+
+        private static Extension DDS = new(".dds");
+
+
+        public override async ValueTask<Directive?> Run(RawSourceFile source)
+        {
+            if (source.Path.Extension == DDS)
+            {
+                var found = _byName[source.Path.FileNameWithoutExtension]
+                    .Select(f => (f.ImageState.PerceptualHash.Similarity(source.File.ImageState.PerceptualHash), f))
+                    .Where(f => f.Item1 >= 0.90f)
+                    .OrderByDescending(f => f.Item1)
+                    .FirstOrDefault();
+
+                if (found == default) return null;
+
+                var rv = source.EvolveTo<TransformedTexture>();
+                rv.ArchiveHashPath = found.f.MakeRelativePaths();
+                rv.ImageState = found.f.ImageState;
+                
+                return rv;
+            }
+
+            return null;
+        }
+    }
+}

Wabbajack.Lib/Data.cs

@@ -6,6 +6,7 @@ using Compression.BSA;
 using Newtonsoft.Json;
 using Wabbajack.Common;
 using Wabbajack.Common.Serialization.Json;
+using Wabbajack.ImageHashing;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.VirtualFileSystem;
 
@@ -37,10 +38,7 @@ namespace Wabbajack.Lib
 
         public T EvolveTo<T>() where T : Directive, new()
         {
-            var v = new T();
-            v.To = Path;
-            v.Hash = File.Hash;
-            v.Size = File.Size;
+            var v = new T {To = Path, Hash = File.Hash, Size = File.Size};
             return v;
         }
     }
@@ -252,6 +250,15 @@ namespace Wabbajack.Lib
         [JsonIgnore]
         public VirtualFile[] Choices { get; set; } = { };
     }
+    
+    [JsonName("TransformedTexture")]
+    public class TransformedTexture : FromArchive
+    {
+        /// <summary>
+        ///     The file to apply to the source file to patch it
+        /// </summary>
+        public ImageState ImageState { get; set; } = new();
+    }
 
     [JsonName("SourcePatch")]
     public class SourcePatch

Wabbajack.Lib/MO2Compiler.cs

@@ -39,6 +39,7 @@ namespace Wabbajack.Lib
         public Dictionary<AbsolutePath, dynamic> ModInis { get; } = new Dictionary<AbsolutePath, dynamic>();
 
         public HashSet<string> SelectedProfiles { get; set; } = new HashSet<string>();
+        public bool DisableTextureResizing { get; set; }
 
         public static AbsolutePath GetTypicalDownloadsFolder(AbsolutePath mo2Folder)
         {
@@ -421,7 +422,7 @@ namespace Wabbajack.Lib
         public override IEnumerable<ICompilationStep> MakeStack()
         {
             Utils.Log("Generating compilation stack");
-            return new List<ICompilationStep>
+            var steps = new List<ICompilationStep>
             {
                 new IgnoreGameFilesIfGameFolderFilesExist(this),
                 new IncludePropertyFiles(this),
@@ -456,6 +457,7 @@ namespace Wabbajack.Lib
                 new IgnoreEndsWith(this, ".log"),
                 new DeconstructBSAs(
                     this), // Deconstruct BSAs before building patches so we don't generate massive patch files
+                new MatchSimilarTextures(this),
                 new IncludePatches(this),
                 new IncludeDummyESPs(this),
 
@@ -486,6 +488,11 @@ namespace Wabbajack.Lib
                 new IgnoreExtension(this, new Extension(".CACHE")),
                 new DropAll(this)
             };
+
+            if (DisableTextureResizing)
+                steps = steps.Where(s => !(s is MatchSimilarTextures)).ToList();
+
+            return steps;
         }
     }
 }

Wabbajack.Test/SanityTests.cs

@@ -5,12 +5,14 @@ using System.Linq;
 using System.Threading.Tasks;
 using Compression.BSA;
 using Wabbajack.Common;
+using Wabbajack.ImageHashing;
 using Wabbajack.Lib;
 using Wabbajack.Lib.CompilationSteps;
 using Wabbajack.Lib.CompilationSteps.CompilationErrors;
 using Xunit;
 using Xunit.Abstractions;
 using Xunit.Sdk;
+using DXGI_FORMAT = DirectXTexNet.DXGI_FORMAT;
 using File = Alphaleonis.Win32.Filesystem.File;
 using Path = Alphaleonis.Win32.Filesystem.Path;
 
@@ -364,6 +366,41 @@ namespace Wabbajack.Test
             
         }
         
+        
+
+        [Fact]
+        public async Task CanRecompressAndResizeDDSImages()
+        {
+            var profile = utils.AddProfile();
+            var mod = await utils.AddMod();
+            var nativeFile = await utils.AddModFile(mod, @"native\whitestagbody.dds", 0);
+            var recompressedFile = await utils.AddModFile(mod, @"recompressed\whitestagbody.dds", 0);
+            var resizedFile = await utils.AddModFile(mod, @"resized\whitestagbody.dds", 0);
+
+            var gameBSA = Game.SkyrimSpecialEdition.MetaData().GameLocation().Combine(@"Data\Skyrim - Textures1.bsa");
+            var bsa = await BSADispatch.OpenRead(gameBSA);
+            var ddsExtension = new Extension(".dds");
+            var firstFile = bsa.Files.First(f => f.Path.Extension == ddsExtension);
+            
+            await using (var nf = await nativeFile.OpenWrite())
+            {
+                await firstFile.CopyDataTo(nf);
+            }
+
+            {
+                using var originalDDS = DDSImage.FromFile(nativeFile);
+                originalDDS.ResizeRecompressAndSave(originalDDS.Width, originalDDS.Height, DXGI_FORMAT.BC7_UNORM, recompressedFile);
+                originalDDS.ResizeRecompressAndSave(128, 128, DXGI_FORMAT.BC7_UNORM, resizedFile);
+            }
+
+            await utils.Configure();
+            
+            await CompileAndInstall(profile, true);
+            await utils.VerifyInstalledFile(mod, @"native\whitestagbody.dds");
+            Assert.True(0.99f <=(await PHash.FromFile(recompressedFile)).Similarity(await PHash.FromFile(utils.InstalledPath(mod, @"recompressed\whitestagbody.dds"))));
+            Assert.True(0.98f <=(await PHash.FromFile(resizedFile)).Similarity(await PHash.FromFile(utils.InstalledPath(mod, @"resized\whitestagbody.dds"))));
+        }
+        
         [Fact]
         public async Task CanNoMatchIncludeFilesFromBSAs()
         {

Wabbajack.Test/TestUtils.cs

@@ -95,14 +95,17 @@ namespace Wabbajack.Test
         /// </summary>
         /// <param name="mod_name"></param>
         /// <param name="path"></param>
-        /// <param name="random_fill"></param>
+        /// <param name="randomFill"></param>
         /// <returns></returns>
-        public async Task<AbsolutePath> AddModFile(string mod_name, string path, int random_fill=128)
+        public async Task<AbsolutePath> AddModFile(string mod_name, string path, int randomFill=128)
         {
-            var full_path = ModsPath.Combine(mod_name, path);
-            full_path.Parent.CreateDirectory();
-            await GenerateRandomFileData(full_path, random_fill);
-            return full_path;
+            var fullPath = ModsPath.Combine(mod_name, path);
+            fullPath.Parent.CreateDirectory();
+            
+            if (randomFill != 0) 
+                await GenerateRandomFileData(fullPath, randomFill);
+            
+            return fullPath;
         }
 
         public async Task GenerateRandomFileData(AbsolutePath full_path, int random_fill)
@@ -198,6 +201,9 @@ namespace Wabbajack.Test
                     Assert.True(false, $"Index {x} of {mod}\\{file} are not the same");
             }
         }
+
+        public AbsolutePath InstalledPath(string mod, string file) =>
+            InstallPath.Combine((string)Consts.MO2ModFolderName, mod, file);
         
         public async Task VerifyInstalledGameFile(string file)
         {

Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs

@@ -1,10 +1,12 @@
 using System.Collections.Generic;
 using System.IO;
+using System.IO.Compression;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Common.Serialization.Json;
+using Wabbajack.ImageHashing;
 
 namespace Wabbajack.VirtualFileSystem
 {
@@ -16,13 +18,24 @@ namespace Wabbajack.VirtualFileSystem
     {
         public IPath Name { get; set; }
         public Hash Hash { get; set; }
+        
+        public ImageState ImageState { get; set; }
         public long Size { get; set; }
-        public List<IndexedVirtualFile> Children { get; set; } = new List<IndexedVirtualFile>();
+        public List<IndexedVirtualFile> Children { get; set; } = new();
 
         private void Write(BinaryWriter bw)
         {
             bw.Write(Name.ToString());
             bw.Write((ulong)Hash);
+            
+            if (ImageState == null)
+                bw.Write(false);
+            else
+            {
+                bw.Write(true);
+                ImageState.Write(bw);
+            }
+
             bw.Write(Size);
             bw.Write(Children.Count);
             foreach (var file in Children)
@@ -31,7 +44,8 @@ namespace Wabbajack.VirtualFileSystem
 
         public void Write(Stream s)
         {
-            using var bw = new BinaryWriter(s, Encoding.UTF8, true);
+            using var cs = new GZipStream(s, CompressionLevel.Optimal , true);
+            using var bw = new BinaryWriter(cs, Encoding.UTF8, true);
             bw.Write(Size);
             bw.Write(Children.Count);
             foreach (var file in Children)
@@ -44,8 +58,13 @@ namespace Wabbajack.VirtualFileSystem
             {
                 Name = (RelativePath)br.ReadString(),
                 Hash = Hash.FromULong(br.ReadUInt64()),
-                Size = br.ReadInt64(),
             };
+
+            if (br.ReadBoolean()) 
+                ivf.ImageState = ImageState.Read(br);
+            
+            ivf.Size = br.ReadInt64();
+            
             var lst = new List<IndexedVirtualFile>();
             ivf.Children = lst;
             var count = br.ReadInt32();
@@ -59,7 +78,8 @@ namespace Wabbajack.VirtualFileSystem
 
         public static IndexedVirtualFile Read(Stream s)
         {
-            using var br = new BinaryReader(s);
+            using var cs = new GZipStream(s, CompressionMode.Decompress, true);
+            using var br = new BinaryReader(cs);
             var ivf = new IndexedVirtualFile
             {
                 Size = br.ReadInt64(),

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -10,12 +10,14 @@ using System.Threading.Tasks;
 using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
 using K4os.Hash.Crc;
 using Wabbajack.Common;
+using Wabbajack.Common.FileSignatures;
+using Wabbajack.ImageHashing;
 
 namespace Wabbajack.VirtualFileSystem
 {
     public class VirtualFile
     {
-        private static AbsolutePath DBLocation = Consts.LocalAppDataPath.Combine("GlobalVFSCache.sqlite");
+        private static AbsolutePath DBLocation = Consts.LocalAppDataPath.Combine("GlobalVFSCache2.sqlite");
         private static string _connectionString;
         private static SQLiteConnection _conn;
 
@@ -47,7 +49,7 @@ namespace Wabbajack.VirtualFileSystem
         public FullPath FullPath { get; private set; }
 
         public Hash Hash { get; internal set; }
-        
+        public ImageState ImageState { get; internal set; }
         public ExtendedHashes ExtendedHashes { get; set; }
         public long Size { get; internal set; }
 
@@ -145,7 +147,8 @@ namespace Wabbajack.VirtualFileSystem
                 Size = file.Size,
                 LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
                 LastAnalyzed = DateTime.Now.AsUnixTime(),
-                Hash = file.Hash
+                Hash = file.Hash,
+                ImageState = file.ImageState
             };
                         
             vself.FillFullPath();
@@ -177,16 +180,17 @@ namespace Wabbajack.VirtualFileSystem
 
         private IndexedVirtualFile ToIndexedVirtualFile()
         {
-            return new IndexedVirtualFile
+            return new()
             {
                 Hash = Hash,
+                ImageState = ImageState,
                 Name = Name,
                 Children = Children.Select(c => c.ToIndexedVirtualFile()).ToList(),
                 Size = Size
             };
         }
-
-
+        
+        
         public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile,
             IPath relPath, int depth = 0)
         {
@@ -219,9 +223,12 @@ namespace Wabbajack.VirtualFileSystem
                 Size = stream.Length,
                 LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
                 LastAnalyzed = DateTime.Now.AsUnixTime(),
-                Hash = hash
+                Hash = hash,
             };
 
+            if (Consts.TextureExtensions.Contains(relPath.FileName.Extension))
+                self.ImageState = await ImageState.FromImageStream(stream, relPath.FileName.Extension, false);
+
             self.FillFullPath(depth);
             
             if (context.UseExtendedHashes)

Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj

@@ -12,6 +12,7 @@
     <ItemGroup>
       <ProjectReference Include="..\Compression.BSA\Compression.BSA.csproj" />
       <ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
+      <ProjectReference Include="..\Wabbajack.ImageHashing\Wabbajack.ImageHashing.csproj" />
     </ItemGroup>
     <ItemGroup>
       <PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />

Wabbajack.sln

@@ -44,6 +44,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Server", "Wabbaja
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Server.Test", "Wabbajack.Server.Test\Wabbajack.Server.Test.csproj", "{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.ImageHashing", "Wabbajack.ImageHashing\Wabbajack.ImageHashing.csproj", "{0C893E5F-1FD8-463E-AC3F-D020213ACD3B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.ImageHashing.Test", "Wabbajack.ImageHashing.Test\Wabbajack.ImageHashing.Test.csproj", "{96892D68-7400-4297-864C-39D0673775DE}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -162,6 +166,22 @@ Global
 		{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|Any CPU.Build.0 = Release|Any CPU
 		{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|x64.ActiveCfg = Release|Any CPU
 		{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|x64.Build.0 = Release|Any CPU
+		{0C893E5F-1FD8-463E-AC3F-D020213ACD3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0C893E5F-1FD8-463E-AC3F-D020213ACD3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0C893E5F-1FD8-463E-AC3F-D020213ACD3B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{0C893E5F-1FD8-463E-AC3F-D020213ACD3B}.Debug|x64.Build.0 = Debug|Any CPU
+		{0C893E5F-1FD8-463E-AC3F-D020213ACD3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0C893E5F-1FD8-463E-AC3F-D020213ACD3B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0C893E5F-1FD8-463E-AC3F-D020213ACD3B}.Release|x64.ActiveCfg = Release|Any CPU
+		{0C893E5F-1FD8-463E-AC3F-D020213ACD3B}.Release|x64.Build.0 = Release|Any CPU
+		{96892D68-7400-4297-864C-39D0673775DE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{96892D68-7400-4297-864C-39D0673775DE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{96892D68-7400-4297-864C-39D0673775DE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{96892D68-7400-4297-864C-39D0673775DE}.Debug|x64.Build.0 = Debug|Any CPU
+		{96892D68-7400-4297-864C-39D0673775DE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{96892D68-7400-4297-864C-39D0673775DE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{96892D68-7400-4297-864C-39D0673775DE}.Release|x64.ActiveCfg = Release|Any CPU
+		{96892D68-7400-4297-864C-39D0673775DE}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

Wabbajack/Settings.cs

@@ -145,6 +145,18 @@ namespace Wabbajack
             }
         }
 
+        
+        private bool _disableTextureResizing;
+        public bool DisableTextureResizing
+        {
+            get => _disableTextureResizing;
+            set
+            {
+                RaiseAndSetIfChanged(ref _disableTextureResizing, value);
+            }
+        }
+
+
 
         public void SetProcessorSettings(ABatchProcessor processor)
         {
@@ -152,6 +164,9 @@ namespace Wabbajack
             processor.DiskThreads = DiskThreads;
             processor.ReduceHDDThreads = ReduceHDDThreads;
             processor.FavorPerfOverRam = FavorPerfOverRam;
+
+            if (processor is MO2Compiler mo2c)
+                mo2c.DisableTextureResizing = DisableTextureResizing;
         }
     }
 

Wabbajack/Views/Settings/PerformanceSettingsView.xaml

@@ -28,6 +28,7 @@
                 <RowDefinition Height="25" />
                 <RowDefinition Height="25" />
                 <RowDefinition Height="25" />
+                <RowDefinition Height="25" />
                 <RowDefinition Height="*" />
             </Grid.RowDefinitions>
             <Grid.ColumnDefinitions>
@@ -99,6 +100,17 @@
                       VerticalAlignment="Center"
                       Foreground="White"
             ></CheckBox>
+            
+            <TextBlock Grid.Row="8" Grid.Column="0"
+                       x:Name="DisableTextureResizingLabel"
+                       VerticalAlignment="Center"
+                       Text="Disable Texture Resizing (during compilation)" />
+            <CheckBox Grid.Row="8" Grid.Column="2"
+                      x:Name="DisableTextureResizing"
+                      HorizontalAlignment="Right"
+                      VerticalAlignment="Center"
+                      Foreground="White"
+            ></CheckBox>
         </Grid>
     </Border>
 </rxui:ReactiveUserControl>

Wabbajack/Views/Settings/PerformanceSettingsView.xaml.cs

@@ -45,6 +45,8 @@ namespace Wabbajack
                     .DisposeWith(disposable);
                 this.BindStrict(this.ViewModel, x => x.NetworkWorkaroundMode, x => x.UseNetworkWorkAround.IsChecked)
                     .DisposeWith(disposable);
+                this.BindStrict(this.ViewModel, x => x.DisableTextureResizing, x => x.DisableTextureResizing.IsChecked)
+                    .DisposeWith(disposable);
             });
         }
     }