Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix ba2 compilation errors (#2286)

Browse Source 
* Fix BA2 compilation errors by implementing mipmap detection and usage during texture recompression

* Update CHANGELOG.md

* Fix broken mipmap support on Linux/OSX
This commit is contained in:
Timothy Baldridge 2023-01-28 15:42:23 -06:00 committed by GitHub
parent 6a9596c9ab
commit 7b46a88fc0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 2215 additions and 2685 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -3,12 +3,13 @@
#### Version - 3.0.6.1 - TBD
* Game support:
* Added Mount & Blade II: Bennerlord support (Steam,GOG)
* Fixed BA2 files not being compressed properly due to MipMaps not being detected properly
#### Version - 3.0.6.0 - 1/21/2023
* Add support for Cubemaps in BA2 files, if you have problems with BA2 recompression, be sure to delete your `GlobalVFSCache3.sqlite` from your AppData before the next compile
* Fixed slides not being shown during installation for lists compile with the 3.0 compiler
* Set the "While loading slide" debug message to be `Trace` level, set the default minimum log level to `Information`
* Switched back to using TexConv for texture converting on Windows, should greatly improve compatability of texture conversion (on windows systems)
* Switched back to using TexConv for texture converting on Windows, should greatly improve compatibility of texture conversion (on windows systems)
#### Version - 3.0.5.0 - 12/22/2022
* Add support for https://www.nexusmods.com/site hosted mods.

2
Wabbajack.Compiler.Test/SanityTests.cs
View File

@ -202,7 +202,7 @@ public class CompilerSanityTests : IAsyncLifetime
var oldState = await _imageLoader.Load(file);
Assert.NotEqual(DXGI_FORMAT.UNKNOWN, oldState.Format);
_logger.LogInformation("Recompressing {file}", file.FileName);
await _imageLoader.Recompress(file, 512, 512, DXGI_FORMAT.BC7_UNORM, file, CancellationToken.None);
await _imageLoader.Recompress(file, 512, 512, 1, DXGI_FORMAT.BC7_UNORM, file, CancellationToken.None);
var state = await _imageLoader.Load(file);
Assert.Equal(DXGI_FORMAT.BC7_UNORM, state.Format);

37
Wabbajack.Compression.BSA/FO4Archive/Builder.cs
View File

@ -20,24 +20,31 @@ public class Builder : IBuilder
public async ValueTask AddFile(AFile state, Stream src, CancellationToken token)
{
switch (_state.Type)
try
{
case BA2EntryType.GNRL:
var result = await FileEntryBuilder.Create((BA2File) state, src, _slab, token);
lock (_entries)
{
_entries.Add(result);
}
switch (_state.Type)
{
case BA2EntryType.GNRL:
var result = await FileEntryBuilder.Create((BA2File)state, src, _slab, token);
lock (_entries)
{
_entries.Add(result);
}
break;
case BA2EntryType.DX10:
var resultdx10 = await DX10FileEntryBuilder.Create((BA2DX10File) state, src, _slab, token);
lock (_entries)
{
_entries.Add(resultdx10);
}
break;
case BA2EntryType.DX10:
var resultdx10 = await DX10FileEntryBuilder.Create((BA2DX10File)state, src, _slab, token);
lock (_entries)
{
_entries.Add(resultdx10);
}
break;
break;
}
}
catch (Exception ex)
{
throw new InvalidDataException($"Error adding file {state.Path} to archive: {ex.Message}", ex);
}
}

4809
Wabbajack.DTOs/JsonConverters/Generated.cs
View File

File diff suppressed because it is too large Load Diff

4
Wabbajack.DTOs/Texture/ImageState.cs
View File

@ -7,8 +7,10 @@ public class ImageState
public DXGI_FORMAT Format { get; set; }
public PHash PerceptualHash { get; set; }
public int MipLevels { get; set; }
public override string ToString()
{
return $"ImageState<{Width}, {Height}, {Format}>";
return $"ImageState<{Width}, {Height}, {Format}, {MipLevels} levels>";
}
}

2
Wabbajack.Hashing.PHash.Test/FileLoadingTests.cs
View File

@ -78,7 +78,7 @@ public class FileLoadingTests : IAsyncDisposable
using var ms = new MemoryStream();
await using var ins = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
await imageLoader.Recompress(ins, 128, 128, DXGI_FORMAT.BC1_UNORM, ms, CancellationToken.None, leaveOpen:true);
await imageLoader.Recompress(ins, 128, 128, baseState.MipLevels, DXGI_FORMAT.BC1_UNORM, ms, CancellationToken.None, leaveOpen:true);
ms.Length.Should().Be(ins.Length);
}
}

10
Wabbajack.Hashing.PHash/CrossPlatformImageLoader.cs
View File

@ -43,6 +43,7 @@ public class CrossPlatformImageLoader : IImageLoader
{
Width = data.Width,
Height = data.Height,
MipLevels = (int)ddsFile.header.dwMipMapCount,
Format = (DXGI_FORMAT) format
};
@ -60,16 +61,16 @@ public class CrossPlatformImageLoader : IImageLoader
new Digest {Coefficients = b.Data});
}
public async Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format,
public async Task Recompress(AbsolutePath input, int width, int height, int mipMaps, DXGI_FORMAT format,
AbsolutePath output,
CancellationToken token)
{
var inData = await input.ReadAllBytesAsync(token);
await using var outStream = output.Open(FileMode.Create, FileAccess.Write);
await Recompress(new MemoryStream(inData), width, height, format, outStream, token);
await Recompress(new MemoryStream(inData), width, height, mipMaps, format, outStream, token);
}
public async Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output,
public async Task Recompress(Stream input, int width, int height, int mipMaps, DXGI_FORMAT format, Stream output,
CancellationToken token, bool leaveOpen = false)
{
var decoder = new BcDecoder();
@ -99,7 +100,8 @@ public class CrossPlatformImageLoader : IImageLoader
Quality = CompressionQuality.Balanced,
GenerateMipMaps = true,
Format = ToCompressionFormat(format),
FileFormat = OutputFileFormat.Dds
FileFormat = OutputFileFormat.Dds,
MaxMipMapLevel = mipMaps
}
};

4
Wabbajack.Hashing.PHash/IImageLoader.cs
View File

@ -18,10 +18,10 @@ public interface IImageLoader
new Digest {Coefficients = b.Data});
}
public Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format,
public Task Recompress(AbsolutePath input, int width, int height, int mipMaps, DXGI_FORMAT format,
AbsolutePath output,
CancellationToken token);
public Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output,
public Task Recompress(Stream input, int width, int height, int mipMaps, DXGI_FORMAT format, Stream output,
CancellationToken token, bool leaveOpen = false);
}

19
Wabbajack.Hashing.PHash/TexConvImageLoader.cs
View File

@ -57,16 +57,16 @@ public class TexConvImageLoader : IImageLoader
return ext;
}
public async Task Recompress(AbsolutePath input, int width, int height, DXGI_FORMAT format, AbsolutePath output,
public async Task Recompress(AbsolutePath input, int width, int height, int mipMaps, DXGI_FORMAT format, AbsolutePath output,
CancellationToken token)
{
var outFolder = _tempManager.CreateFolder();
var outFile = input.FileName.RelativeTo(outFolder.Path);
await ConvertImage(input, outFolder.Path, width, height, format, input.Extension);
await ConvertImage(input, outFolder.Path, width, height, mipMaps, format, input.Extension);
await outFile.MoveToAsync(output, token: token, overwrite:true);
}
public async Task Recompress(Stream input, int width, int height, DXGI_FORMAT format, Stream output, CancellationToken token,
public async Task Recompress(Stream input, int width, int height, int mipMaps, DXGI_FORMAT format, Stream output, CancellationToken token,
bool leaveOpen = false)
{
var type = await DetermineType(input);
@ -75,19 +75,19 @@ public class TexConvImageLoader : IImageLoader
await input.CopyToAsync(fromFile.Path, token);
var toFile = fromFile.Path.FileName.RelativeTo(toFolder);
await ConvertImage(fromFile.Path, toFolder.Path, width, height, format, type);
await ConvertImage(fromFile.Path, toFolder.Path, width, height, mipMaps, format, type);
await using var fs = toFile.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
await fs.CopyToAsync(output, token);
}
public async Task ConvertImage(AbsolutePath from, AbsolutePath toFolder, int w, int h, DXGI_FORMAT format, Extension fileFormat)
public async Task ConvertImage(AbsolutePath from, AbsolutePath toFolder, int w, int h, int mipMaps, DXGI_FORMAT format, Extension fileFormat)
{
// User isn't renaming the file, so we don't have to create a temporary folder
var ph = new ProcessHelper
{
Path = @"Tools\texconv.exe".ToRelativePath().RelativeTo(KnownFolders.EntryPoint),
Arguments = new object[] {from, "-ft", fileFormat.ToString()[1..], "-f", format, "-o", toFolder, "-w", w, "-h", h, "-if", "CUBIC", "-singleproc"},
Arguments = new object[] {from, "-ft", fileFormat.ToString()[1..], "-f", format, "-o", toFolder, "-w", w, "-h", h, "-m", mipMaps, "-if", "CUBIC", "-singleproc"},
ThrowOnNonZeroExitCode = true,
LogError = true
};
@ -100,7 +100,7 @@ public class TexConvImageLoader : IImageLoader
await using var tmpFile = _tempManager.CreateFolder();
var inFile = to.FileName.RelativeTo(tmpFile.Path);
await inFile.WriteAllAsync(from, CancellationToken.None);
await ConvertImage(inFile, to.Parent, state.Width, state.Height, state.Format, ext);
await ConvertImage(inFile, to.Parent, state.Width, state.Height, state.MipLevels, state.Format, ext);
}
// Internals
@ -133,7 +133,8 @@ public class TexConvImageLoader : IImageLoader
Width = int.Parse(data["width"]),
Height = int.Parse(data["height"]),
Format = Enum.Parse<DXGI_FORMAT>(data["format"]),
PerceptualHash = await GetPHash(path)
PerceptualHash = await GetPHash(path),
MipLevels = byte.Parse(data["mipLevels"])
};
}
catch (Exception ex)
@ -149,7 +150,7 @@ public class TexConvImageLoader : IImageLoader
throw new FileNotFoundException($"Can't hash non-existent file {path}");
await using var tmp = _tempManager.CreateFolder();
await ConvertImage(path, tmp.Path, 512, 512, DXGI_FORMAT.R8G8B8A8_UNORM, Ext.Png);
await ConvertImage(path, tmp.Path, 512, 512, 1, DXGI_FORMAT.R8G8B8A8_UNORM, Ext.Png);
using var img = await Image.LoadAsync(path.FileName.RelativeTo(tmp.Path).ReplaceExtension(Ext.Png).ToString());
img.Mutate(x => x.Resize(512, 512, KnownResamplers.Welch).Grayscale(GrayscaleMode.Bt601));

2
Wabbajack.Installer/AInstaller.cs
View File

@ -261,7 +261,7 @@ public abstract class AInstaller<T>
await using var s = await sf.GetStream();
await using var of = destPath.Open(FileMode.Create, FileAccess.Write);
_logger.LogInformation("Recompressing {Filename}", tt.To.FileName);
await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.Format,
await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.MipLevels, tt.ImageState.Format,
of, token);
}
break;

2
Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
View File

@ -64,7 +64,7 @@ public static class ServiceExtensions
{
var diskCache = options.UseLocalCache
? new VFSDiskCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path)
: new VFSDiskCache(KnownFolders.WabbajackAppLocal.Combine("GlobalVFSCache3.sqlite"));
: new VFSDiskCache(KnownFolders.WabbajackAppLocal.Combine("GlobalVFSCache4.sqlite"));
var cesiCache = new CesiVFSCache(s.GetRequiredService<ILogger<CesiVFSCache>>(),
s.GetRequiredService<Client>());
return new FallthroughVFSCache(new IVfsCache[] {diskCache});

4
Wabbajack.VFS/Context.cs
View File

@ -136,13 +136,13 @@ public class Context
token,
fileNames.Keys.ToHashSet());
}
catch (Exception)
catch (Exception ex)
{
await using var stream = await sfn.GetStream();
var hash = await stream.HashingCopy(Stream.Null, token);
if (hash != file.Hash)
throw new Exception(
$"File {file.FullPath} is corrupt, please delete it and retry the installation");
$"File {file.FullPath} is corrupt, please delete it and retry the installation, {ex.Message}", ex);
throw;
}
}

2
Wabbajack.VFS/IndexedVirtualFileExtensions.cs
View File

@ -36,6 +36,7 @@ public static class IndexedVirtualFileExtensions
{
bw.Write((ushort) state.Width);
bw.Write((ushort) state.Height);
bw.Write((byte)state.MipLevels);
bw.Write((byte) state.Format);
state.PerceptualHash.Write(bw);
}
@ -46,6 +47,7 @@ public static class IndexedVirtualFileExtensions
{
Width = br.ReadUInt16(),
Height = br.ReadUInt16(),
MipLevels = br.ReadByte(),
Format = (DXGI_FORMAT) br.ReadByte(),
PerceptualHash = PHash.Read(br)
};