Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cube maps (#2260)

Browse Source 
* Add some support for cubemaps in BA2

* add read support for cubemaps in BA2 routines

* Fix slide support during compilation

* Set default logging level to Information instead of Trace

* update CHANGELOG.md
This commit is contained in:
Timothy Baldridge 2022-12-28 10:21:58 -06:00 committed by GitHub
parent e5ae5acb08
commit 4bfce0e418
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 968 additions and 754 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -1,5 +1,10 @@
### Changelog
#### Version - 3.0.6.0 - ??
* Add support for Cubemaps in BA2 files, if you have problems with BA2 recompression, be sure to delete your `GlobalVFSCache3.sqlite` from your AppData before the next compile
* Fixed slides not being shown during installation for lists compile with the 3.0 compiler
* Set the "While loading slide" debug message to be `Trace` level, set the default minimum log level to `Information`
#### Version - 3.0.5.0 - 12/22/2022
* Add support for https://www.nexusmods.com/site hosted mods.
* Fix Website Links

2
Wabbajack.App.Wpf/App.xaml.cs
View File

@ -145,7 +145,7 @@ namespace Wabbajack
config.AddRuleForAllLevels(uiTarget);
loggingBuilder.ClearProviders();
loggingBuilder.SetMinimumLevel(LogLevel.Trace);
loggingBuilder.SetMinimumLevel(LogLevel.Information);
loggingBuilder.AddNLog(config);
}

2
Wabbajack.App.Wpf/View Models/Installers/InstallerVM.cs
View File

@ -505,7 +505,7 @@ public class InstallerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM
}
catch (Exception ex)
{
_logger.LogWarning(ex, "While loading slide");
_logger.LogTrace(ex, "While loading slide");
}
}

16
Wabbajack.Compression.BSA/DDS.cs
View File

@ -1,4 +1,5 @@
using System.IO;
using System;
using System.IO;
using System.Runtime.InteropServices;
using Wabbajack.DTOs.Texture;
@ -66,6 +67,19 @@ public enum DXT10_RESOURCE_DIMENSION
DIMENSION_TEXTURE3D = 4
}
[Flags]
public enum DDSCAPS2 : uint
{
CUBEMAP = 0x200,
CUBEMAP_POSITIVEX = 0x400,
CUBEMAP_NEGATIVEX = 0x800,
CUBEMAP_POSITIVEY = 0x1000,
CUBEMAP_NEGATIVEY = 0x2000,
CUBEMAP_POSITIVEZ = 0x4000,
CUBEMAP_NEGATIVEZ = 0x8000,
CUBEMAP_ALLFACES = 0xFC00
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public struct DDS_HEADER
{

40
Wabbajack.Compression.BSA/FO4Archive/DX10Entry.cs
View File

@ -18,19 +18,21 @@ namespace Wabbajack.Compression.BSA.FO4Archive;
public class DX10Entry : IBA2FileEntry
{
private readonly Reader _bsa;
internal ushort _chunkHdrLen;
internal List<TextureChunk> _chunks;
internal uint _dirHash;
internal string _extension;
internal byte _format;
internal ushort _height;
internal int _index;
internal uint _nameHash;
internal byte _numChunks;
internal byte _numMips;
internal ushort _unk16;
internal byte _unk8;
internal ushort _width;
private ushort _chunkHdrLen;
private List<TextureChunk> _chunks;
private uint _dirHash;
private string _extension;
private byte _format;
private ushort _height;
private int _index;
private uint _nameHash;
private byte _numChunks;
private byte _numMips;
private ushort _unk16;
private byte _unk8;
private ushort _width;
private readonly byte _isCubemap;
private readonly byte _tileMode;
public DX10Entry(Reader ba2Reader, int idx)
{
@ -47,7 +49,8 @@ public class DX10Entry : IBA2FileEntry
_width = _rdr.ReadUInt16();
_numMips = _rdr.ReadByte();
_format = _rdr.ReadByte();
_unk16 = _rdr.ReadUInt16();
_isCubemap = _rdr.ReadByte();
_tileMode = _rdr.ReadByte();
_index = idx;
_chunks = Enumerable.Range(0, _numChunks)
@ -74,7 +77,8 @@ public class DX10Entry : IBA2FileEntry
Width = _width,
NumMips = _numMips,
PixelFormat = _format,
Unk16 = _unk16,
IsCubeMap = _isCubemap,
TileMode = _tileMode,
Index = _index,
Chunks = _chunks.Select(ch => new BA2Chunk
{
@ -139,6 +143,12 @@ public class DX10Entry : IBA2FileEntry
ddsHeader.PixelFormat.dwSize = ddsHeader.PixelFormat.GetSize();
ddsHeader.dwDepth = 1;
ddsHeader.dwSurfaceFlags = DDS.DDS_SURFACE_FLAGS_TEXTURE | DDS.DDS_SURFACE_FLAGS_MIPMAP;
ddsHeader.dwCubemapFlags = _isCubemap == 1 ? (uint)(DDSCAPS2.CUBEMAP
| DDSCAPS2.CUBEMAP_NEGATIVEX | DDSCAPS2.CUBEMAP_POSITIVEX
| DDSCAPS2.CUBEMAP_NEGATIVEY | DDSCAPS2.CUBEMAP_POSITIVEY
| DDSCAPS2.CUBEMAP_NEGATIVEZ | DDSCAPS2.CUBEMAP_POSITIVEZ
| DDSCAPS2.CUBEMAP_ALLFACES) : 0u;
switch ((DXGI_FORMAT) _format)
{

3
Wabbajack.Compression.BSA/FO4Archive/DX10FileEntryBuilder.cs
View File

@ -31,7 +31,8 @@ public class DX10FileEntryBuilder : IFileBuilder
bw.Write(_state.Width);
bw.Write(_state.NumMips);
bw.Write(_state.PixelFormat);
bw.Write(_state.Unk16);
bw.Write((byte)_state.IsCubeMap);
bw.Write((byte)_state.TileMode);
foreach (var chunk in _chunks)
chunk.WriteHeader(bw);

1507
Wabbajack.DTOs/JsonConverters/Generated.cs
View File

File diff suppressed because it is too large Load Diff

4
Wabbajack.DTOs/ModList/BSA/FileStates/BA2DX10File.cs
View File

@ -8,8 +8,6 @@ public class BA2DX10File : AFile
{
public BA2Chunk[] Chunks { get; set; }
public ushort Unk16 { get; set; }
public byte PixelFormat { get; set; }
public byte NumMips { get; set; }
@ -27,4 +25,6 @@ public class BA2DX10File : AFile
public string Extension { get; set; }
public uint NameHash { get; set; }
public byte IsCubeMap { get; set; }
public byte TileMode { get; set; }
}

25
Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs
View File

@ -1,6 +1,8 @@
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.Downloaders.VerificationCache;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Paths.IO;
using Xunit;
@ -10,32 +12,41 @@ namespace Wabbajack.Downloaders.Dispatcher.Test;
public class VerificationCacheTests
{
private readonly ILogger<VerificationCache.VerificationCache> _logger;
private readonly DTOSerializer _dtos;
public VerificationCacheTests(ILogger<VerificationCache.VerificationCache> logger)
public VerificationCacheTests(ILogger<VerificationCache.VerificationCache> logger, DTOSerializer dtos)
{
_logger = logger;
_dtos = dtos;
}
[Fact]
public async Task BasicCacheTests()
{
using var cache = new VerificationCache.VerificationCache(_logger, KnownFolders.EntryPoint.Combine(Guid.NewGuid().ToString()), TimeSpan.FromSeconds(1));
using var cacheBase = new VerificationCache.VerificationCache(_logger,
KnownFolders.EntryPoint.Combine(Guid.NewGuid().ToString()),
TimeSpan.FromSeconds(1),
_dtos);
var cache = (IVerificationCache)cacheBase;
var goodState = new DTOs.DownloadStates.Http { Url = new Uri($"https://some.com/{Guid.NewGuid()}/path") };
var badState = new DTOs.DownloadStates.Http { Url = new Uri($"https://some.com/{Guid.NewGuid()}/path") };
Assert.True(await cache.Get(goodState) == null);
Assert.True((await cache.Get(goodState)).IsValid == null);
await cache.Put(goodState, true);
Assert.True(await cache.Get(goodState));
var result = await cache.Get(goodState);
Assert.True(result.IsValid);
Assert.IsType<DTOs.DownloadStates.Http>(result.State);
await Task.Delay(TimeSpan.FromSeconds(2));
Assert.False(await cache.Get(goodState));
Assert.False((await cache.Get(goodState)).IsValid);
await cache.Put(badState, true);
Assert.True(await cache.Get(badState));
Assert.True((await cache.Get(badState)).IsValid);
await cache.Put(badState, false);
Assert.Null(await cache.Get(badState));
Assert.Null((await cache.Get(badState)).IsValid);
}
}

8
Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs
View File

@ -112,9 +112,13 @@ public class DownloadDispatcher
{
try
{
if (await _verificationCache.Get(a.State) == true)
var (valid, newState) = await _verificationCache.Get(a.State);
if (valid == true)
{
a.State = newState;
return true;
}
a = await MaybeProxy(a, token);
var downloader = Downloader(a);
using var job = await _limiter.Begin($"Verifying {a.State.PrimaryKeyString}", -1, token);

2
Wabbajack.Downloaders.VerificationCache/IVerificationCache.cs
View File

@ -5,6 +5,6 @@ namespace Wabbajack.Downloaders.VerificationCache;
public interface IVerificationCache
{
Task<bool?> Get(IDownloadState archive);
Task<(bool? IsValid, IDownloadState State)> Get(IDownloadState archive);
Task Put(IDownloadState archive, bool valid);
}

24
Wabbajack.Downloaders.VerificationCache/VerificationCache.cs
View File

@ -1,7 +1,9 @@
using System.Data.SQLite;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
@ -15,12 +17,14 @@ public class VerificationCache : IVerificationCache, IDisposable
private readonly SQLiteConnection _conn;
private readonly TimeSpan _expiry;
private readonly ILogger<VerificationCache> _logger;
private readonly DTOSerializer _dtos;
public VerificationCache(ILogger<VerificationCache> logger, AbsolutePath location, TimeSpan expiry)
public VerificationCache(ILogger<VerificationCache> logger, AbsolutePath location, TimeSpan expiry, DTOSerializer dtos)
{
_logger = logger;
_location = location;
_expiry = expiry;
_dtos = dtos;
if (!_location.Parent.DirectoryExists())
_location.Parent.CreateDirectory();
@ -34,17 +38,18 @@ public class VerificationCache : IVerificationCache, IDisposable
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"CREATE TABLE IF NOT EXISTS VerficationCache (
PKS TEXT PRIMARY KEY,
LastModified BIGINT)
LastModified BIGINT,
State TEXT)
WITHOUT ROWID";
cmd.ExecuteNonQuery();
}
public async Task<bool?> Get(IDownloadState archive)
public async Task<(bool?, IDownloadState?)> Get(IDownloadState archive)
{
var key = archive.PrimaryKeyString;
await using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = "SELECT LastModified FROM VerficationCache WHERE PKS = @pks";
cmd.CommandText = "SELECT LastModified, State FROM VerficationCache WHERE PKS = @pks";
cmd.Parameters.AddWithValue("@pks", key);
await cmd.PrepareAsync();
@ -52,10 +57,12 @@ public class VerificationCache : IVerificationCache, IDisposable
while (await reader.ReadAsync())
{
var ts = DateTime.FromFileTimeUtc(reader.GetInt64(0));
return DateTime.UtcNow - ts <= _expiry;
var state = JsonSerializer.Deserialize<IDownloadState>(reader.GetString(1), _dtos.Options);
return (DateTime.UtcNow - ts <= _expiry, state);
}
return null;
return (null, null);
}
public async Task Put(IDownloadState state, bool valid)
@ -64,10 +71,11 @@ public class VerificationCache : IVerificationCache, IDisposable
if (valid)
{
await using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"INSERT INTO VerficationCache (PKS, LastModified) VALUES (@pks, @lastModified)
ON CONFLICT(PKS) DO UPDATE SET LastModified = @lastModified";
cmd.CommandText = @"INSERT INTO VerficationCache (PKS, LastModified, State) VALUES (@pks, @lastModified, @state)
ON CONFLICT(PKS) DO UPDATE SET LastModified = @lastModified, State = @state";
cmd.Parameters.AddWithValue("@pks", key);
cmd.Parameters.AddWithValue("@lastModified", DateTime.UtcNow.ToFileTimeUtc());
cmd.Parameters.AddWithValue("@state", JsonSerializer.Serialize(state, _dtos.Options));
await cmd.PrepareAsync();
await cmd.ExecuteNonQueryAsync();

26
Wabbajack.Hashing.PHash.Test/FileLoadingTests.cs
View File

@ -1,4 +1,7 @@
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Shipwreck.Phash;
using Wabbajack.DTOs.Texture;
using Wabbajack.Paths;
@ -29,4 +32,27 @@ public class FileLoadingTests
new Digest {Coefficients = state.PerceptualHash.Data}),
1.0);
}
[Fact]
public async Task CanConvertCubeMaps()
{
// File used here via re-upload permissions found on the mod's Nexus page:
// https://www.nexusmods.com/fallout4/mods/43458?tab=description
// Used for testing purposes only
var path = "TestData/WindowDisabled_CGPlayerHouseCube.dds".ToRelativePath().RelativeTo(KnownFolders.EntryPoint);
var baseState = await ImageLoader.Load(path);
baseState.Height.Should().Be(128);
baseState.Width.Should().Be(128);
//baseState.Frames.Should().Be(6);
using var ms = new MemoryStream();
await using var ins = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
await ImageLoader.Recompress(ins, 128, 128, DXGI_FORMAT.BC1_UNORM, ms, CancellationToken.None, leaveOpen:true);
ms.Length.Should().Be(ins.Length);
}
}

BIN
Wabbajack.Hashing.PHash.Test/TestData/WindowDisabled_CGPlayerHouseCube.dds Normal file
View File

Binary file not shown.

4
Wabbajack.Hashing.PHash.Test/Wabbajack.Hashing.PHash.Test.csproj
View File

@ -11,6 +11,7 @@
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="FluentAssertions" Version="6.8.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0-preview-20221003-04" />
<PackageReference Include="Shipwreck.Phash" Version="0.5.0" />
<PackageReference Include="SixLabors.ImageSharp" Version="2.1.3" />
@ -38,6 +39,9 @@
<None Update="TestData\test-dxt5-small-bc7-vflip.dds">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="TestData\WindowDisabled_CGPlayerHouseCube.dds">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>

36
Wabbajack.Hashing.PHash/Image.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
@ -71,12 +72,24 @@ public class ImageLoader
{
var decoder = new BcDecoder();
var ddsFile = DdsFile.Load(input);
if (!leaveOpen) await input.DisposeAsync();
var data = await decoder.DecodeToImageRgba32Async(ddsFile, token);
var faces = new List<Image<Rgba32>>();
var origFormat = ddsFile.dx10Header.dxgiFormat == DxgiFormat.DxgiFormatUnknown
? ddsFile.header.ddsPixelFormat.DxgiFormat
: ddsFile.dx10Header.dxgiFormat;
data.Mutate(x => x.Resize(width, height, KnownResamplers.Welch));
foreach (var face in ddsFile.Faces)
{
var data = await decoder.DecodeRawToImageRgba32Async(face.MipMaps[0].Data,
(int)face.Width, (int)face.Height, ToCompressionFormat((DXGI_FORMAT)origFormat), token);
data.Mutate(x => x.Resize(width, height, KnownResamplers.Welch));
faces.Add(data);
}
var encoder = new BcEncoder
{
@ -88,9 +101,20 @@ public class ImageLoader
FileFormat = OutputFileFormat.Dds
}
};
var file = await encoder.EncodeToDdsAsync(data, token);
file.Write(output);
switch (faces.Count)
{
case 1:
(await encoder.EncodeToDdsAsync(faces[0], token)).Write(output);
break;
case 6:
(await encoder.EncodeCubeMapToDdsAsync(faces[0], faces[1], faces[2], faces[3], faces[4], faces[5], token))
.Write(output);
break;
default:
throw new NotImplementedException($"Can't encode dds with {faces.Count} faces");
}
if (!leaveOpen)
await output.DisposeAsync();
}

1
Wabbajack.Paths.IO/KnownFolders.cs
View File

@ -12,6 +12,7 @@ public static class KnownFolders
{
get
{
return AppDomain.CurrentDomain.BaseDirectory.ToAbsolutePath();
var result = Process.GetCurrentProcess().MainModule?.FileName?.ToAbsolutePath() ?? default;
if (result != default &&

17
Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
View File

@ -13,6 +13,7 @@ using Wabbajack.Downloaders.GameFile;
using Wabbajack.Downloaders.VerificationCache;
using Wabbajack.DTOs;
using Wabbajack.DTOs.Interventions;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.DTOs.Logins;
using Wabbajack.Installer;
using Wabbajack.Networking.BethesdaNet;
@ -73,9 +74,19 @@ public static class ServiceExtensions
: new BinaryPatchCache(s.GetRequiredService<ILogger<BinaryPatchCache>>(),KnownFolders.WabbajackAppLocal.Combine("PatchCache")));
service.AddSingleton<IVerificationCache>(s => options.UseLocalCache
? new VerificationCache(s.GetRequiredService<ILogger<VerificationCache>>(), s.GetService<TemporaryFileManager>()!.CreateFile().Path, TimeSpan.FromDays(1))
: new VerificationCache(s.GetRequiredService<ILogger<VerificationCache>>(),KnownFolders.WabbajackAppLocal.Combine("VerificationCache.sqlite"), TimeSpan.FromDays(1)));
service.AddSingleton<IVerificationCache>(s =>
{
var dtos = s.GetRequiredService<DTOSerializer>();
return options.UseLocalCache
? new VerificationCache(s.GetRequiredService<ILogger<VerificationCache>>(),
s.GetService<TemporaryFileManager>()!.CreateFile().Path,
TimeSpan.FromDays(1),
dtos)
: new VerificationCache(s.GetRequiredService<ILogger<VerificationCache>>(),
KnownFolders.WabbajackAppLocal.Combine("VerificationCacheV2.sqlite"),
TimeSpan.FromDays(1),
dtos);
});
service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount});