Fix ba2 compilation errors (#2286)

* Fix BA2 compilation errors by implementing mipmap detection and usage during texture recompression

* Update CHANGELOG.md

* Fix broken mipmap support on Linux/OSX

CHANGELOG.md

@@ -3,12 +3,13 @@
 #### Version - 3.0.6.1 - TBD
 * Game support:
   * Added Mount & Blade II: Bennerlord support (Steam,GOG) 
+* Fixed BA2 files not being compressed properly due to MipMaps not being detected properly
 
 #### Version - 3.0.6.0 - 1/21/2023
 * Add support for Cubemaps in BA2 files, if you have problems with BA2 recompression, be sure to delete your `GlobalVFSCache3.sqlite` from your AppData before the next compile
 * Fixed slides not being shown during installation for lists compile with the 3.0 compiler
 * Set the "While loading slide" debug message to be `Trace` level, set the default minimum log level to `Information`
-* Switched back to using TexConv for texture converting on Windows, should greatly improve compatability of texture conversion (on windows systems)
+* Switched back to using TexConv for texture converting on Windows, should greatly improve compatibility  of texture conversion (on windows systems)
 
 #### Version - 3.0.5.0 - 12/22/2022
 * Add support for https://www.nexusmods.com/site hosted mods.

Wabbajack.Compiler.Test/SanityTests.cs

@@ -202,7 +202,7 @@ public class CompilerSanityTests : IAsyncLifetime
             var oldState = await _imageLoader.Load(file);
             Assert.NotEqual(DXGI_FORMAT.UNKNOWN, oldState.Format);
             _logger.LogInformation("Recompressing {file}", file.FileName);
-            await _imageLoader.Recompress(file, 512, 512, DXGI_FORMAT.BC7_UNORM, file, CancellationToken.None);
+            await _imageLoader.Recompress(file, 512, 512, 1, DXGI_FORMAT.BC7_UNORM, file, CancellationToken.None);
 
             var state = await _imageLoader.Load(file);
             Assert.Equal(DXGI_FORMAT.BC7_UNORM, state.Format);

Wabbajack.Compression.BSA/FO4Archive/Builder.cs

@@ -20,24 +20,31 @@ public class Builder : IBuilder
 
     public async ValueTask AddFile(AFile state, Stream src, CancellationToken token)
     {
-        switch (_state.Type)
+        try
         {
-            case BA2EntryType.GNRL:
+            switch (_state.Type)
-                var result = await FileEntryBuilder.Create((BA2File) state, src, _slab, token);
+            {
-                lock (_entries)
+                case BA2EntryType.GNRL:
-                {
+                    var result = await FileEntryBuilder.Create((BA2File)state, src, _slab, token);
-                    _entries.Add(result);
+                    lock (_entries)
-                }
+                    {
+                        _entries.Add(result);
+                    }
 
-                break;
+                    break;
-            case BA2EntryType.DX10:
+                case BA2EntryType.DX10:
-                var resultdx10 = await DX10FileEntryBuilder.Create((BA2DX10File) state, src, _slab, token);
+                    var resultdx10 = await DX10FileEntryBuilder.Create((BA2DX10File)state, src, _slab, token);
-                lock (_entries)
+                    lock (_entries)
-                {
+                    {
-                    _entries.Add(resultdx10);
+                        _entries.Add(resultdx10);
-                }
+                    }
 
-                break;
+                    break;
+            }
+        }
+        catch (Exception ex)
+        {
+            throw new InvalidDataException($"Error adding file {state.Path} to archive: {ex.Message}", ex);
         }
     }
 

Wabbajack.DTOs/Texture/ImageState.cs

@@ -7,8 +7,10 @@ public class ImageState
     public DXGI_FORMAT Format { get; set; }
     public PHash PerceptualHash { get; set; }
 
+    public int MipLevels { get; set; }
+
     public override string ToString()
     {
-        return $"ImageState<{Width}, {Height}, {Format}>";
+        return $"ImageState<{Width}, {Height}, {Format}, {MipLevels} levels>";
     }
 }

Wabbajack.Hashing.PHash.Test/FileLoadingTests.cs

@@ -78,7 +78,7 @@ public class FileLoadingTests : IAsyncDisposable
 
             using var ms = new MemoryStream();
             await using var ins = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
-            await imageLoader.Recompress(ins, 128, 128, DXGI_FORMAT.BC1_UNORM, ms, CancellationToken.None, leaveOpen:true);
+            await imageLoader.Recompress(ins, 128, 128, baseState.MipLevels, DXGI_FORMAT.BC1_UNORM, ms, CancellationToken.None, leaveOpen:true);
             ms.Length.Should().Be(ins.Length);
         }
     }

Wabbajack.Hashing.PHash/CrossPlatformImageLoader.cs

@@ -43,6 +43,7 @@ public class CrossPlatformImageLoader : IImageLoader
         {
             Width = data.Width,
             Height = data.Height,
+            MipLevels = (int)ddsFile.header.dwMipMapCount,
             Format = (DXGI_FORMAT) format
         };
 
@@ -60,16 +61,16 @@ public class CrossPlatformImageLoader : IImageLoader
             new Digest {Coefficients = b.Data});
     }
     
-    public async Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format,
+    public async Task Recompress(AbsolutePath input, int width, int height, int mipMaps, DXGI_FORMAT format,
         AbsolutePath output,
         CancellationToken token)
     {
         var inData = await input.ReadAllBytesAsync(token);
         await using var outStream = output.Open(FileMode.Create, FileAccess.Write);
-        await Recompress(new MemoryStream(inData), width, height, format, outStream, token);
+        await Recompress(new MemoryStream(inData), width, height, mipMaps, format, outStream, token);
     }
 
-    public async Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output,
+    public async Task Recompress(Stream input, int width, int height, int mipMaps, DXGI_FORMAT format, Stream output,
         CancellationToken token, bool leaveOpen = false)
     {
         var decoder = new BcDecoder();
@@ -99,7 +100,8 @@ public class CrossPlatformImageLoader : IImageLoader
                 Quality = CompressionQuality.Balanced,
                 GenerateMipMaps = true,
                 Format = ToCompressionFormat(format),
-                FileFormat = OutputFileFormat.Dds
+                FileFormat = OutputFileFormat.Dds,
+                MaxMipMapLevel = mipMaps
             }
         };
 

Wabbajack.Hashing.PHash/IImageLoader.cs

@@ -18,10 +18,10 @@ public interface IImageLoader
             new Digest {Coefficients = b.Data});
     }
 
-    public Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format,
+    public Task Recompress(AbsolutePath input, int width, int height, int mipMaps, DXGI_FORMAT format,
         AbsolutePath output,
         CancellationToken token);
 
-    public Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output,
+    public Task Recompress(Stream input, int width, int height, int mipMaps, DXGI_FORMAT format, Stream output,
         CancellationToken token, bool leaveOpen = false);
 }

Wabbajack.Hashing.PHash/TexConvImageLoader.cs

@@ -57,16 +57,16 @@ public class TexConvImageLoader : IImageLoader
         return ext;
     }
 
-    public async Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format, AbsolutePath output,
+    public async Task Recompress(AbsolutePath input, int width, int height, int mipMaps, DXGI_FORMAT format, AbsolutePath output,
         CancellationToken token)
     {
         var outFolder = _tempManager.CreateFolder();
         var outFile = input.FileName.RelativeTo(outFolder.Path);
-        await ConvertImage(input, outFolder.Path, width, height, format, input.Extension);
+        await ConvertImage(input, outFolder.Path, width, height, mipMaps, format, input.Extension);
         await outFile.MoveToAsync(output, token: token, overwrite:true);
     }
 
-    public async Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output, CancellationToken token,
+    public async Task Recompress(Stream input, int width, int height, int mipMaps, DXGI_FORMAT format, Stream output, CancellationToken token,
         bool leaveOpen = false)
     {
         var type = await DetermineType(input);
@@ -75,19 +75,19 @@ public class TexConvImageLoader : IImageLoader
         await input.CopyToAsync(fromFile.Path, token);
         var toFile = fromFile.Path.FileName.RelativeTo(toFolder);
         
-        await ConvertImage(fromFile.Path, toFolder.Path, width, height, format, type);
+        await ConvertImage(fromFile.Path, toFolder.Path, width, height, mipMaps, format, type);
         await using var fs = toFile.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
         await fs.CopyToAsync(output, token);
     }
     
     
-    public async Task ConvertImage(AbsolutePath from, AbsolutePath toFolder, int w, int h, DXGI_FORMAT format, Extension fileFormat)
+    public async Task ConvertImage(AbsolutePath from, AbsolutePath toFolder, int w, int h, int mipMaps, DXGI_FORMAT format, Extension fileFormat)
     {
         // User isn't renaming the file, so we don't have to create a temporary folder
         var ph = new ProcessHelper
         {
             Path = @"Tools\texconv.exe".ToRelativePath().RelativeTo(KnownFolders.EntryPoint),
-            Arguments = new object[] {from, "-ft", fileFormat.ToString()[1..], "-f", format, "-o", toFolder, "-w", w, "-h", h, "-if", "CUBIC", "-singleproc"},
+            Arguments = new object[] {from, "-ft", fileFormat.ToString()[1..], "-f", format, "-o", toFolder, "-w", w, "-h", h, "-m", mipMaps, "-if", "CUBIC", "-singleproc"},
             ThrowOnNonZeroExitCode = true,
             LogError = true
         }; 
@@ -100,7 +100,7 @@ public class TexConvImageLoader : IImageLoader
         await using var tmpFile = _tempManager.CreateFolder();
         var inFile = to.FileName.RelativeTo(tmpFile.Path);
         await inFile.WriteAllAsync(from, CancellationToken.None);
-        await ConvertImage(inFile, to.Parent, state.Width, state.Height, state.Format, ext);
+        await ConvertImage(inFile, to.Parent, state.Width, state.Height, state.MipLevels, state.Format, ext);
     }
     
     // Internals
@@ -133,7 +133,8 @@ public class TexConvImageLoader : IImageLoader
                 Width = int.Parse(data["width"]),
                 Height = int.Parse(data["height"]),
                 Format = Enum.Parse<DXGI_FORMAT>(data["format"]),
-                PerceptualHash = await GetPHash(path)
+                PerceptualHash = await GetPHash(path),
+                MipLevels = byte.Parse(data["mipLevels"])
             };
         }
         catch (Exception ex)
@@ -149,7 +150,7 @@ public class TexConvImageLoader : IImageLoader
             throw new FileNotFoundException($"Can't hash non-existent file {path}");
             
         await using var tmp = _tempManager.CreateFolder();
-        await ConvertImage(path, tmp.Path, 512, 512, DXGI_FORMAT.R8G8B8A8_UNORM, Ext.Png);
+        await ConvertImage(path, tmp.Path, 512, 512, 1, DXGI_FORMAT.R8G8B8A8_UNORM, Ext.Png);
             
         using var img = await Image.LoadAsync(path.FileName.RelativeTo(tmp.Path).ReplaceExtension(Ext.Png).ToString());
         img.Mutate(x => x.Resize(512, 512, KnownResamplers.Welch).Grayscale(GrayscaleMode.Bt601));

Wabbajack.Installer/AInstaller.cs

@@ -261,7 +261,7 @@ public abstract class AInstaller<T>
                         await using var s = await sf.GetStream();
                         await using var of = destPath.Open(FileMode.Create, FileAccess.Write);
                         _logger.LogInformation("Recompressing {Filename}", tt.To.FileName);
-                        await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.Format,
+                        await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.MipLevels, tt.ImageState.Format,
                             of, token);
                     }
                         break;

Wabbajack.Services.OSIntegrated/ServiceExtensions.cs

@@ -64,7 +64,7 @@ public static class ServiceExtensions
         {
             var diskCache = options.UseLocalCache
                 ? new VFSDiskCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path)
-                : new VFSDiskCache(KnownFolders.WabbajackAppLocal.Combine("GlobalVFSCache3.sqlite"));
+                : new VFSDiskCache(KnownFolders.WabbajackAppLocal.Combine("GlobalVFSCache4.sqlite"));
             var cesiCache = new CesiVFSCache(s.GetRequiredService<ILogger<CesiVFSCache>>(),
                 s.GetRequiredService<Client>());
             return new FallthroughVFSCache(new IVfsCache[] {diskCache});

Wabbajack.VFS/Context.cs

@@ -136,13 +136,13 @@ public class Context
                         token,
                         fileNames.Keys.ToHashSet());
                 }
-                catch (Exception)
+                catch (Exception ex)
                 {
                     await using var stream = await sfn.GetStream();
                     var hash = await stream.HashingCopy(Stream.Null, token);
                     if (hash != file.Hash)
                         throw new Exception(
-                            $"File {file.FullPath} is corrupt, please delete it and retry the installation");
+                            $"File {file.FullPath} is corrupt, please delete it and retry the installation, {ex.Message}", ex);
                     throw;
                 }
             }

Wabbajack.VFS/IndexedVirtualFileExtensions.cs

@@ -36,6 +36,7 @@ public static class IndexedVirtualFileExtensions
     {
         bw.Write((ushort) state.Width);
         bw.Write((ushort) state.Height);
+        bw.Write((byte)state.MipLevels);
         bw.Write((byte) state.Format);
         state.PerceptualHash.Write(bw);
     }
@@ -46,6 +47,7 @@ public static class IndexedVirtualFileExtensions
         {
             Width = br.ReadUInt16(),
             Height = br.ReadUInt16(),
+            MipLevels = br.ReadByte(),
             Format = (DXGI_FORMAT) br.ReadByte(),
             PerceptualHash = PHash.Read(br)
         };