Cube maps (#2260)

* Add some support for cubemaps in BA2

* add read support for cubemaps in BA2 routines

* Fix slide support during compilation

* Set default logging level to Information instead of Trace

* update CHANGELOG.md

CHANGELOG.md

@@ -1,5 +1,10 @@
 ### Changelog
 
+#### Version - 3.0.6.0 - ??
+* Add support for Cubemaps in BA2 files, if you have problems with BA2 recompression, be sure to delete your `GlobalVFSCache3.sqlite` from your AppData before the next compile
+* Fixed slides not being shown during installation for lists compile with the 3.0 compiler
+* Set the "While loading slide" debug message to be `Trace` level, set the default minimum log level to `Information`
+
 #### Version - 3.0.5.0 - 12/22/2022
 * Add support for https://www.nexusmods.com/site hosted mods.
 * Fix Website Links

Wabbajack.App.Wpf/App.xaml.cs

@@ -145,7 +145,7 @@ namespace Wabbajack
             config.AddRuleForAllLevels(uiTarget);
 
             loggingBuilder.ClearProviders();
-            loggingBuilder.SetMinimumLevel(LogLevel.Trace);
+            loggingBuilder.SetMinimumLevel(LogLevel.Information);
             loggingBuilder.AddNLog(config);
         }
 

Wabbajack.App.Wpf/View Models/Installers/InstallerVM.cs

@@ -505,7 +505,7 @@ public class InstallerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM
         }
         catch (Exception ex)
         {
-            _logger.LogWarning(ex, "While loading slide");
+            _logger.LogTrace(ex, "While loading slide");
         }
     }
 

Wabbajack.Compression.BSA/DDS.cs

@@ -1,4 +1,5 @@
-using System.IO;
+using System;
+using System.IO;
 using System.Runtime.InteropServices;
 using Wabbajack.DTOs.Texture;
 
@@ -66,6 +67,19 @@ public enum DXT10_RESOURCE_DIMENSION
     DIMENSION_TEXTURE3D = 4
 }
 
+[Flags]
+public enum DDSCAPS2 : uint
+{
+    CUBEMAP = 0x200,
+    CUBEMAP_POSITIVEX = 0x400,
+    CUBEMAP_NEGATIVEX = 0x800,
+    CUBEMAP_POSITIVEY = 0x1000,
+    CUBEMAP_NEGATIVEY = 0x2000,
+    CUBEMAP_POSITIVEZ = 0x4000,
+    CUBEMAP_NEGATIVEZ = 0x8000,
+    CUBEMAP_ALLFACES = 0xFC00
+}
+
 [StructLayout(LayoutKind.Sequential, Pack = 1)]
 public struct DDS_HEADER
 {

Wabbajack.Compression.BSA/FO4Archive/DX10Entry.cs

@@ -18,19 +18,21 @@ namespace Wabbajack.Compression.BSA.FO4Archive;
 public class DX10Entry : IBA2FileEntry
 {
     private readonly Reader _bsa;
-    internal ushort _chunkHdrLen;
+    private ushort _chunkHdrLen;
-    internal List<TextureChunk> _chunks;
+    private List<TextureChunk> _chunks;
-    internal uint _dirHash;
+    private uint _dirHash;
-    internal string _extension;
+    private string _extension;
-    internal byte _format;
+    private byte _format;
-    internal ushort _height;
+    private ushort _height;
-    internal int _index;
+    private int _index;
-    internal uint _nameHash;
+    private uint _nameHash;
-    internal byte _numChunks;
+    private byte _numChunks;
-    internal byte _numMips;
+    private byte _numMips;
-    internal ushort _unk16;
+    private ushort _unk16;
-    internal byte _unk8;
+    private byte _unk8;
-    internal ushort _width;
+    private ushort _width;
+    private readonly byte _isCubemap;
+    private readonly byte _tileMode;
 
     public DX10Entry(Reader ba2Reader, int idx)
     {
@@ -47,7 +49,8 @@ public class DX10Entry : IBA2FileEntry
         _width = _rdr.ReadUInt16();
         _numMips = _rdr.ReadByte();
         _format = _rdr.ReadByte();
-        _unk16 = _rdr.ReadUInt16();
+        _isCubemap = _rdr.ReadByte();
+        _tileMode = _rdr.ReadByte();
         _index = idx;
 
         _chunks = Enumerable.Range(0, _numChunks)
@@ -74,7 +77,8 @@ public class DX10Entry : IBA2FileEntry
         Width = _width,
         NumMips = _numMips,
         PixelFormat = _format,
-        Unk16 = _unk16,
+        IsCubeMap = _isCubemap,
+        TileMode = _tileMode,
         Index = _index,
         Chunks = _chunks.Select(ch => new BA2Chunk
         {
@@ -139,6 +143,12 @@ public class DX10Entry : IBA2FileEntry
         ddsHeader.PixelFormat.dwSize = ddsHeader.PixelFormat.GetSize();
         ddsHeader.dwDepth = 1;
         ddsHeader.dwSurfaceFlags = DDS.DDS_SURFACE_FLAGS_TEXTURE | DDS.DDS_SURFACE_FLAGS_MIPMAP;
+        ddsHeader.dwCubemapFlags = _isCubemap == 1 ? (uint)(DDSCAPS2.CUBEMAP
+                   | DDSCAPS2.CUBEMAP_NEGATIVEX | DDSCAPS2.CUBEMAP_POSITIVEX
+                   | DDSCAPS2.CUBEMAP_NEGATIVEY | DDSCAPS2.CUBEMAP_POSITIVEY
+                   | DDSCAPS2.CUBEMAP_NEGATIVEZ | DDSCAPS2.CUBEMAP_POSITIVEZ
+                   | DDSCAPS2.CUBEMAP_ALLFACES) : 0u;
+        
 
         switch ((DXGI_FORMAT) _format)
         {

Wabbajack.Compression.BSA/FO4Archive/DX10FileEntryBuilder.cs

@@ -31,7 +31,8 @@ public class DX10FileEntryBuilder : IFileBuilder
         bw.Write(_state.Width);
         bw.Write(_state.NumMips);
         bw.Write(_state.PixelFormat);
-        bw.Write(_state.Unk16);
+        bw.Write((byte)_state.IsCubeMap);
+        bw.Write((byte)_state.TileMode);
 
         foreach (var chunk in _chunks)
             chunk.WriteHeader(bw);

Wabbajack.DTOs/ModList/BSA/FileStates/BA2DX10File.cs

@@ -8,8 +8,6 @@ public class BA2DX10File : AFile
 {
     public BA2Chunk[] Chunks { get; set; }
 
-    public ushort Unk16 { get; set; }
-
     public byte PixelFormat { get; set; }
 
     public byte NumMips { get; set; }
@@ -27,4 +25,6 @@ public class BA2DX10File : AFile
     public string Extension { get; set; }
 
     public uint NameHash { get; set; }
+    public byte IsCubeMap { get; set; }
+    public byte TileMode { get; set; }
 }

Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
+using Wabbajack.Downloaders.VerificationCache;
+using Wabbajack.DTOs.JsonConverters;
 using Wabbajack.Paths.IO;
 using Xunit;
 
@@ -10,32 +12,41 @@ namespace Wabbajack.Downloaders.Dispatcher.Test;
 public class VerificationCacheTests
 {
     private readonly ILogger<VerificationCache.VerificationCache> _logger;
+    private readonly DTOSerializer _dtos;
 
-    public VerificationCacheTests(ILogger<VerificationCache.VerificationCache> logger)
+    public VerificationCacheTests(ILogger<VerificationCache.VerificationCache> logger, DTOSerializer dtos)
     {
         _logger = logger;
+        _dtos = dtos;
     }
 
     [Fact]
     public async Task BasicCacheTests()
     {
-        using var cache = new VerificationCache.VerificationCache(_logger,  KnownFolders.EntryPoint.Combine(Guid.NewGuid().ToString()), TimeSpan.FromSeconds(1));
+        using var cacheBase = new VerificationCache.VerificationCache(_logger,  
+            KnownFolders.EntryPoint.Combine(Guid.NewGuid().ToString()), 
+            TimeSpan.FromSeconds(1),
+            _dtos);
+
+        var cache = (IVerificationCache)cacheBase;
 
         var goodState = new DTOs.DownloadStates.Http { Url = new Uri($"https://some.com/{Guid.NewGuid()}/path") };
         var badState = new DTOs.DownloadStates.Http { Url = new Uri($"https://some.com/{Guid.NewGuid()}/path") };
-        Assert.True(await cache.Get(goodState) == null);
+        Assert.True((await cache.Get(goodState)).IsValid == null);
 
         await cache.Put(goodState, true);
-        Assert.True(await cache.Get(goodState));
+        var result = await cache.Get(goodState);
+        Assert.True(result.IsValid);
+        Assert.IsType<DTOs.DownloadStates.Http>(result.State);
 
         await Task.Delay(TimeSpan.FromSeconds(2));
         
-        Assert.False(await cache.Get(goodState));
+        Assert.False((await cache.Get(goodState)).IsValid);
 
         await cache.Put(badState, true);
-        Assert.True(await cache.Get(badState));
+        Assert.True((await cache.Get(badState)).IsValid);
         await cache.Put(badState, false);
-        Assert.Null(await cache.Get(badState));
+        Assert.Null((await cache.Get(badState)).IsValid);
 
     }
 }

Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs

@@ -112,9 +112,13 @@ public class DownloadDispatcher
     {
         try
         {
-            if (await _verificationCache.Get(a.State) == true)
+            var (valid, newState) = await _verificationCache.Get(a.State);
+            if (valid == true)
+            {
+                a.State = newState;
                 return true;
-            
+            }
+
             a = await MaybeProxy(a, token);
             var downloader = Downloader(a);
             using var job = await _limiter.Begin($"Verifying {a.State.PrimaryKeyString}", -1, token);

Wabbajack.Downloaders.VerificationCache/IVerificationCache.cs

@@ -5,6 +5,6 @@ namespace Wabbajack.Downloaders.VerificationCache;
 
 public interface IVerificationCache
 {
-    Task<bool?> Get(IDownloadState archive);
+    Task<(bool? IsValid, IDownloadState State)> Get(IDownloadState archive);
     Task Put(IDownloadState archive, bool valid);
 }

Wabbajack.Downloaders.VerificationCache/VerificationCache.cs

@@ -1,7 +1,9 @@
 using System.Data.SQLite;
+using System.Text.Json;
 using Microsoft.Extensions.Logging;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.DownloadStates;
+using Wabbajack.DTOs.JsonConverters;
 using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Paths;
 using Wabbajack.Paths.IO;
@@ -15,12 +17,14 @@ public class VerificationCache : IVerificationCache, IDisposable
     private readonly SQLiteConnection _conn;
     private readonly TimeSpan _expiry;
     private readonly ILogger<VerificationCache> _logger;
+    private readonly DTOSerializer _dtos;
 
-    public VerificationCache(ILogger<VerificationCache> logger, AbsolutePath location, TimeSpan expiry)
+    public VerificationCache(ILogger<VerificationCache> logger, AbsolutePath location, TimeSpan expiry, DTOSerializer dtos)
     {
         _logger = logger;
         _location = location;
         _expiry = expiry;
+        _dtos = dtos;
 
         if (!_location.Parent.DirectoryExists())
             _location.Parent.CreateDirectory();
@@ -34,17 +38,18 @@ public class VerificationCache : IVerificationCache, IDisposable
         using var cmd = new SQLiteCommand(_conn);
         cmd.CommandText = @"CREATE TABLE IF NOT EXISTS VerficationCache (
             PKS TEXT PRIMARY KEY,
-            LastModified BIGINT)
+            LastModified BIGINT,
+            State TEXT)
             WITHOUT ROWID";
         cmd.ExecuteNonQuery();
     }
 
-    public async Task<bool?> Get(IDownloadState archive)
+    public async Task<(bool?, IDownloadState?)> Get(IDownloadState archive)
     {
         var key = archive.PrimaryKeyString;
 
         await using var cmd = new SQLiteCommand(_conn);
-        cmd.CommandText = "SELECT LastModified FROM VerficationCache WHERE PKS = @pks";
+        cmd.CommandText = "SELECT LastModified, State FROM VerficationCache WHERE PKS = @pks";
         cmd.Parameters.AddWithValue("@pks", key);
         await cmd.PrepareAsync();
 
@@ -52,10 +57,12 @@ public class VerificationCache : IVerificationCache, IDisposable
         while (await reader.ReadAsync())
         {
             var ts = DateTime.FromFileTimeUtc(reader.GetInt64(0));
-            return DateTime.UtcNow - ts <= _expiry;
+            var state = JsonSerializer.Deserialize<IDownloadState>(reader.GetString(1), _dtos.Options);
+            
+            return (DateTime.UtcNow - ts <= _expiry, state);
         }
 
-        return null;
+        return (null, null);
     }
 
     public async Task Put(IDownloadState state, bool valid)
@@ -64,10 +71,11 @@ public class VerificationCache : IVerificationCache, IDisposable
         if (valid)
         {
             await using var cmd = new SQLiteCommand(_conn);
-            cmd.CommandText = @"INSERT INTO VerficationCache (PKS, LastModified) VALUES (@pks, @lastModified)
+            cmd.CommandText = @"INSERT INTO VerficationCache (PKS, LastModified, State) VALUES (@pks, @lastModified, @state)
-            ON CONFLICT(PKS) DO UPDATE SET LastModified = @lastModified";
+            ON CONFLICT(PKS) DO UPDATE SET LastModified = @lastModified, State = @state";
             cmd.Parameters.AddWithValue("@pks", key);
             cmd.Parameters.AddWithValue("@lastModified", DateTime.UtcNow.ToFileTimeUtc());
+            cmd.Parameters.AddWithValue("@state", JsonSerializer.Serialize(state, _dtos.Options));
             await cmd.PrepareAsync();
 
             await cmd.ExecuteNonQueryAsync();

Wabbajack.Hashing.PHash.Test/FileLoadingTests.cs

@@ -1,4 +1,7 @@
+using System.IO;
+using System.Threading;
 using System.Threading.Tasks;
+using FluentAssertions;
 using Shipwreck.Phash;
 using Wabbajack.DTOs.Texture;
 using Wabbajack.Paths;
@@ -29,4 +32,27 @@ public class FileLoadingTests
                 new Digest {Coefficients = state.PerceptualHash.Data}),
             1.0);
     }
+
+    [Fact]
+    public async Task CanConvertCubeMaps()
+    {
+        // File used here via re-upload permissions found on the mod's Nexus page:
+        // https://www.nexusmods.com/fallout4/mods/43458?tab=description
+        // Used for testing purposes only
+        var path = "TestData/WindowDisabled_CGPlayerHouseCube.dds".ToRelativePath().RelativeTo(KnownFolders.EntryPoint);
+        
+        var baseState = await ImageLoader.Load(path);
+        baseState.Height.Should().Be(128);
+        baseState.Width.Should().Be(128);
+        //baseState.Frames.Should().Be(6);
+
+        using var ms = new MemoryStream();
+        await using var ins = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
+        await ImageLoader.Recompress(ins, 128, 128, DXGI_FORMAT.BC1_UNORM, ms, CancellationToken.None, leaveOpen:true);
+        ms.Length.Should().Be(ins.Length);
+
+
+
+    }
+
 }

Wabbajack.Hashing.PHash.Test/Wabbajack.Hashing.PHash.Test.csproj

@@ -11,6 +11,7 @@
           <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
           <PrivateAssets>all</PrivateAssets>
         </PackageReference>
+        <PackageReference Include="FluentAssertions" Version="6.8.0" />
         <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0-preview-20221003-04" />
         <PackageReference Include="Shipwreck.Phash" Version="0.5.0" />
         <PackageReference Include="SixLabors.ImageSharp" Version="2.1.3" />
@@ -38,6 +39,9 @@
         <None Update="TestData\test-dxt5-small-bc7-vflip.dds">
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         </None>
+        <None Update="TestData\WindowDisabled_CGPlayerHouseCube.dds">
+          <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        </None>
     </ItemGroup>
 
     <ItemGroup>

Wabbajack.Hashing.PHash/Image.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
@@ -71,12 +72,24 @@ public class ImageLoader
     {
         var decoder = new BcDecoder();
         var ddsFile = DdsFile.Load(input);
-
+        
         if (!leaveOpen) await input.DisposeAsync();
 
-        var data = await decoder.DecodeToImageRgba32Async(ddsFile, token);
+        var faces = new List<Image<Rgba32>>();
+        
+        var origFormat = ddsFile.dx10Header.dxgiFormat == DxgiFormat.DxgiFormatUnknown
+            ? ddsFile.header.ddsPixelFormat.DxgiFormat
+            : ddsFile.dx10Header.dxgiFormat;
 
-        data.Mutate(x => x.Resize(width, height, KnownResamplers.Welch));
+        foreach (var face in ddsFile.Faces)
+        {
+
+            var data = await decoder.DecodeRawToImageRgba32Async(face.MipMaps[0].Data, 
+                (int)face.Width, (int)face.Height, ToCompressionFormat((DXGI_FORMAT)origFormat), token);
+
+            data.Mutate(x => x.Resize(width, height, KnownResamplers.Welch));
+            faces.Add(data);
+        }
 
         var encoder = new BcEncoder
         {
@@ -88,9 +101,20 @@ public class ImageLoader
                 FileFormat = OutputFileFormat.Dds
             }
         };
-        var file = await encoder.EncodeToDdsAsync(data, token);
+        
-        file.Write(output);
+        switch (faces.Count)
-
+        {
+            case 1:
+                (await encoder.EncodeToDdsAsync(faces[0], token)).Write(output);
+                break;
+            case 6:
+                (await encoder.EncodeCubeMapToDdsAsync(faces[0], faces[1], faces[2], faces[3], faces[4], faces[5], token))
+                    .Write(output);
+                break;
+            default:
+                throw new NotImplementedException($"Can't encode dds with {faces.Count} faces");
+        }
+        
         if (!leaveOpen)
             await output.DisposeAsync();
     }

Wabbajack.Paths.IO/KnownFolders.cs

@@ -12,6 +12,7 @@ public static class KnownFolders
     {
         get
         {
+            return AppDomain.CurrentDomain.BaseDirectory.ToAbsolutePath();
             var result = Process.GetCurrentProcess().MainModule?.FileName?.ToAbsolutePath() ?? default;
 
             if (result != default &&

Wabbajack.Services.OSIntegrated/ServiceExtensions.cs

@@ -13,6 +13,7 @@ using Wabbajack.Downloaders.GameFile;
 using Wabbajack.Downloaders.VerificationCache;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.Interventions;
+using Wabbajack.DTOs.JsonConverters;
 using Wabbajack.DTOs.Logins;
 using Wabbajack.Installer;
 using Wabbajack.Networking.BethesdaNet;
@@ -73,9 +74,19 @@ public static class ServiceExtensions
             : new BinaryPatchCache(s.GetRequiredService<ILogger<BinaryPatchCache>>(),KnownFolders.WabbajackAppLocal.Combine("PatchCache")));
 
         
-        service.AddSingleton<IVerificationCache>(s => options.UseLocalCache
+        service.AddSingleton<IVerificationCache>(s =>
-            ? new VerificationCache(s.GetRequiredService<ILogger<VerificationCache>>(), s.GetService<TemporaryFileManager>()!.CreateFile().Path, TimeSpan.FromDays(1))
+        {
-            : new VerificationCache(s.GetRequiredService<ILogger<VerificationCache>>(),KnownFolders.WabbajackAppLocal.Combine("VerificationCache.sqlite"), TimeSpan.FromDays(1)));
+            var dtos = s.GetRequiredService<DTOSerializer>();
+            return options.UseLocalCache
+                ? new VerificationCache(s.GetRequiredService<ILogger<VerificationCache>>(),
+                    s.GetService<TemporaryFileManager>()!.CreateFile().Path, 
+                    TimeSpan.FromDays(1),
+                    dtos)
+                : new VerificationCache(s.GetRequiredService<ILogger<VerificationCache>>(),
+                    KnownFolders.WabbajackAppLocal.Combine("VerificationCacheV2.sqlite"), 
+                    TimeSpan.FromDays(1),
+                    dtos);
+        });
 
         service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount});