diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 47a599d4..1c2caf38 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -167,12 +167,12 @@ namespace Wabbajack.Common public static RelativePath ModListTxt = (RelativePath)"modlist.txt"; public static RelativePath ModOrganizer2Exe = (RelativePath)"ModOrganizer.exe"; public static RelativePath ModOrganizer2Ini = (RelativePath)"ModOrganizer.ini"; - public static string AuthorAPIKeyFile = "author-api-key.txt"; + public static readonly string AuthorAPIKeyFile = "author-api-key.txt"; - public static Uri WabbajackOrg = new Uri("https://www.wabbajack.org/"); + public static readonly Uri WabbajackOrg = new Uri("https://www.wabbajack.org/"); - public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2; + public static readonly long UploadedFileBlockSize = (long)1024 * 1024 * 2; - public static HashSet TextureExtensions = new() {new Extension(".dds"), new Extension(".tga")}; + public static readonly HashSet TextureExtensions = new() {new Extension(".dds"), new Extension(".tga")}; } } diff --git a/Wabbajack.Common/Paths/FullPath.cs b/Wabbajack.Common/Paths/FullPath.cs index 2a9879d5..70c32545 100644 --- a/Wabbajack.Common/Paths/FullPath.cs +++ b/Wabbajack.Common/Paths/FullPath.cs @@ -72,5 +72,25 @@ namespace Wabbajack.Common } public RelativePath FileName => Paths.Length == 0 ? Base.FileName : Paths.Last().FileName; + + /// + /// Creates a new full path, with relativePath combined with the deepest leaf in the full path + /// + /// + /// + public FullPath InSameFolder(RelativePath relativePath) + { + if (Paths.Length == 0) + { + return new FullPath(Base.Parent.Combine(relativePath)); + } + else + { + var paths = new RelativePath[Paths.Length]; + Paths.CopyTo(paths, 0); + paths[^1] = paths[^1].Parent.Combine(relativePath); + return new FullPath(Base, paths); + } + } } } diff --git a/Wabbajack.Common/Paths/RelativePath.cs b/Wabbajack.Common/Paths/RelativePath.cs index f90d6a48..d3957152 100644 --- a/Wabbajack.Common/Paths/RelativePath.cs +++ b/Wabbajack.Common/Paths/RelativePath.cs @@ -139,6 +139,11 @@ namespace Wabbajack.Common return _path.StartsWith(s, StringComparison.OrdinalIgnoreCase); } + public bool EndsWith(string s) + { + return _path.EndsWith(s, StringComparison.OrdinalIgnoreCase); + } + public bool StartsWith(RelativePath s) { return _path.StartsWith(s._path, StringComparison.OrdinalIgnoreCase); diff --git a/Wabbajack.ImageHashing/ImageState.cs b/Wabbajack.ImageHashing/ImageState.cs index 4b66a0fd..2e23a7f1 100644 --- a/Wabbajack.ImageHashing/ImageState.cs +++ b/Wabbajack.ImageHashing/ImageState.cs @@ -41,7 +41,7 @@ namespace Wabbajack.ImageHashing PerceptualHash.Write(bw); } - public static async Task FromImageStream(Stream stream, Extension ext, bool takeStreamOwnership = true) + public static async Task FromImageStream(Stream stream, Extension ext, bool takeStreamOwnership = true) { await using var tf = new TempFile(ext); await tf.Path.WriteAllAsync(stream, takeStreamOwnership); @@ -80,7 +80,7 @@ namespace Wabbajack.ImageHashing await ConvertImage(inFile, to.Parent, state.Width, state.Height, state.Format, ext); } - public static async Task GetState(AbsolutePath path) + public static async Task GetState(AbsolutePath path) { var ph = new ProcessHelper { @@ -94,14 +94,7 @@ namespace Wabbajack.ImageHashing .Select(p => p.Line) .Where(p => p.Contains(" = ")) .Subscribe(l => lines.Push(l)); - try - { - await ph.Start(); - } - catch (Exception ex) - { - return null; - } + await ph.Start(); var data = lines.Select(l => { diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 29183caf..92e64a7b 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -21,14 +21,13 @@ namespace Wabbajack.Lib { public abstract class ACompiler : ABatchProcessor { - protected readonly Subject<(string, float)> _progressUpdates = new Subject<(string, float)>(); + protected readonly Subject<(string, float)> _progressUpdates = new(); - public List IndexedArchives = new List(); + public List IndexedArchives = new(); - public Dictionary> IndexedFiles = - new Dictionary>(); + public Dictionary> IndexedFiles = new(); - public ModList ModList = new ModList(); + public ModList ModList = new(); public AbsolutePath ModListImage; public bool ModlistIsNSFW; diff --git a/Wabbajack.Lib/AuthorApi/Client.cs b/Wabbajack.Lib/AuthorApi/Client.cs index 6500afa6..01692ecf 100644 --- a/Wabbajack.Lib/AuthorApi/Client.cs +++ b/Wabbajack.Lib/AuthorApi/Client.cs @@ -44,12 +44,12 @@ namespace Wabbajack.Lib.AuthorApi IEnumerable Blocks(AbsolutePath path) { var size = path.Size; - for (long block = 0; block * Consts.UPLOADED_FILE_BLOCK_SIZE < size; block ++) + for (long block = 0; block * Consts.UploadedFileBlockSize < size; block ++) yield return new CDNFilePartDefinition { Index = block, - Size = Math.Min(Consts.UPLOADED_FILE_BLOCK_SIZE, size - block * Consts.UPLOADED_FILE_BLOCK_SIZE), - Offset = block * Consts.UPLOADED_FILE_BLOCK_SIZE + Size = Math.Min(Consts.UploadedFileBlockSize, size - block * Consts.UploadedFileBlockSize), + Offset = block * Consts.UploadedFileBlockSize }; } diff --git a/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs b/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs index b3c826ae..deb871d3 100644 --- a/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs +++ b/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs @@ -34,6 +34,7 @@ namespace Wabbajack.Lib.CompilationSteps _microstack = bsa => new List { new DirectMatch(_mo2Compiler), + new MatchSimilarTextures(_mo2Compiler), new IncludePatches(_mo2Compiler, bsa), new DropAll(_mo2Compiler) }; @@ -41,6 +42,7 @@ namespace Wabbajack.Lib.CompilationSteps _microstackWithInclude = bsa => new List { new DirectMatch(_mo2Compiler), + new MatchSimilarTextures(_mo2Compiler), new IncludePatches(_mo2Compiler, bsa), new IncludeAll(_mo2Compiler) }; diff --git a/Wabbajack.Lib/CompilationSteps/MatchSimilarTextures.cs b/Wabbajack.Lib/CompilationSteps/MatchSimilarTextures.cs index 60e99e20..2e62e5c7 100644 --- a/Wabbajack.Lib/CompilationSteps/MatchSimilarTextures.cs +++ b/Wabbajack.Lib/CompilationSteps/MatchSimilarTextures.cs @@ -1,5 +1,8 @@ -using System.Linq; +using System; +using System.IO; +using System.Linq; using System.Threading.Tasks; +using Microsoft.VisualBasic.Logging; using Wabbajack.Common; using Wabbajack.ImageHashing; using Wabbajack.VirtualFileSystem; @@ -17,24 +20,63 @@ namespace Wabbajack.Lib.CompilationSteps .ToLookup(f => f.Name.FileName.FileNameWithoutExtension); } + private const float PerceptualTolerance = 0.80f; + private static Extension DDS = new(".dds"); + private static string[] PostFixes = new[] {"_n", "_d", "_s"}; public override async ValueTask Run(RawSourceFile source) { - if (source.Path.Extension == DDS && source.File.ImageState != null) + if (source.File.Name.FileName.Extension == DDS && source.File.ImageState != null) { - var found = _byName[source.Path.FileNameWithoutExtension] + (float Similarity, VirtualFile File) found = _byName[source.Path.FileNameWithoutExtension] .Select(f => (f.ImageState.PerceptualHash.Similarity(source.File.ImageState.PerceptualHash), f)) - .Where(f => f.Item1 >= 0.90f) + .Select(f => + { + Utils.Log($"{f.f.Name.FileName} similar {f.Item1}"); + return f; + }) .OrderByDescending(f => f.Item1) .FirstOrDefault(); - if (found == default) return null; + if (found == default || found.Similarity <= PerceptualTolerance) + { + // This looks bad, but it's fairly simple: normal and displacement textures don't match very well + // via perceptual hashing. So instead we'll try to find a diffuse map with the same name, and look + // for normal maps in the same folders. Example: roof_n.dds didn't match, so find a match betweeen + // roof.dds and a perceptual match in the downloads. Then try to find a roof_n.dds in the same folder + // as the match we found for roof.dds. + found = default; + var r = from postfix in PostFixes + where source.File.Name.FileName.FileNameWithoutExtension.EndsWith(postfix) + let mainFileName = + source.File.Name.FileName.FileNameWithoutExtension.ToString()[..^postfix.Length] + + ".dds" + let mainFile = source.File.InSameFolder(new RelativePath(mainFileName)) + where mainFile != null + from mainMatch in _byName[mainFile.FullPath.FileName.FileNameWithoutExtension] + where mainMatch.ImageState != null + where mainFile.ImageState != null + let similarity = mainFile.ImageState.PerceptualHash.Similarity(mainMatch.ImageState.PerceptualHash) + where similarity >= PerceptualTolerance + orderby similarity descending + let foundFile = mainMatch.InSameFolder(source.Path.FileName) + where foundFile != null + select (similarity, postfix, mainFile, mainMatch, foundFile); + + var foundRec = r.FirstOrDefault(); + if (foundRec == default) + { + return null; + } + + found = (foundRec.similarity, foundRec.foundFile); + } var rv = source.EvolveTo(); - rv.ArchiveHashPath = found.f.MakeRelativePaths(); - rv.ImageState = found.f.ImageState; + rv.ArchiveHashPath = found.File.MakeRelativePaths(); + rv.ImageState = found.File.ImageState; return rv; } diff --git a/Wabbajack.Server.Test/AuthoredFilesTests.cs b/Wabbajack.Server.Test/AuthoredFilesTests.cs index 4c6514f2..d07ce16d 100644 --- a/Wabbajack.Server.Test/AuthoredFilesTests.cs +++ b/Wabbajack.Server.Test/AuthoredFilesTests.cs @@ -28,7 +28,7 @@ namespace Wabbajack.BuildServer.Test var toDelete = await cleanup.FindFilesToDelete(); await using var file = new TempFile(); - await file.Path.WriteAllBytesAsync(RandomData(Consts.UPLOADED_FILE_BLOCK_SIZE * 4 + Consts.UPLOADED_FILE_BLOCK_SIZE / 3)); + await file.Path.WriteAllBytesAsync(RandomData(Consts.UploadedFileBlockSize * 4 + Consts.UploadedFileBlockSize / 3)); var originalHash = await file.Path.FileHashAsync(); var client = await Client.Create(Fixture.APIKey); diff --git a/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs b/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs index b80b24c4..8a7a0137 100644 --- a/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs @@ -46,10 +46,7 @@ namespace Wabbajack.VirtualFileSystem { using var cs = new GZipStream(s, CompressionLevel.Optimal , true); using var bw = new BinaryWriter(cs, Encoding.UTF8, true); - bw.Write(Size); - bw.Write(Children.Count); - foreach (var file in Children) - file.Write(bw); + Write(bw); } private static IndexedVirtualFile Read(BinaryReader br) @@ -80,18 +77,7 @@ namespace Wabbajack.VirtualFileSystem { using var cs = new GZipStream(s, CompressionMode.Decompress, true); using var br = new BinaryReader(cs); - var ivf = new IndexedVirtualFile - { - Size = br.ReadInt64(), - }; - var lst = new List(); - ivf.Children = lst; - var count = br.ReadInt32(); - for (int x = 0; x < count; x++) - { - lst.Add(Read(br)); - } - return ivf; + return Read(br); } } } diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 0dcc1459..b9a379a9 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -17,7 +17,7 @@ namespace Wabbajack.VirtualFileSystem { public class VirtualFile { - private static AbsolutePath DBLocation = Consts.LocalAppDataPath.Combine("GlobalVFSCache2.sqlite"); + private static AbsolutePath DBLocation = Consts.LocalAppDataPath.Combine("GlobalVFSCache3.sqlite"); private static string _connectionString; private static SQLiteConnection _conn; @@ -227,7 +227,10 @@ namespace Wabbajack.VirtualFileSystem }; if (Consts.TextureExtensions.Contains(relPath.FileName.Extension)) + { self.ImageState = await ImageState.FromImageStream(stream, relPath.FileName.Extension, false); + stream.Position = 0; + } self.FillFullPath(depth); @@ -267,7 +270,11 @@ namespace Wabbajack.VirtualFileSystem private static async Task WriteToCache(VirtualFile self) { await using var ms = new MemoryStream(); - self.ToIndexedVirtualFile().Write(ms); + var ivf = self.ToIndexedVirtualFile(); + // Top level path gets renamed when read, we don't want the absolute path + // here else the reader will blow up when it tries to convert the value + ivf.Name = (RelativePath)"not/applicable"; + ivf.Write(ms); ms.Position = 0; await InsertIntoVFSCache(self.Hash, ms); } @@ -436,6 +443,12 @@ namespace Wabbajack.VirtualFileSystem var path = new HashRelativePath(FilesInFullPath.First().Hash, paths); return path; } + + public VirtualFile InSameFolder(RelativePath relativePath) + { + var newPath = FullPath.InSameFolder(relativePath); + return Context.Index.ByFullPath.TryGetValue(newPath, out var found) ? found : null; + } } public class ExtendedHashes