From 72d77bef1a045e34b3cdc9cd49186b8c866e8935 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 24 Mar 2020 15:42:28 -0600 Subject: [PATCH] VFS Tests pass --- Wabbajack.Common/Paths.cs | 58 +++++++++++-- .../VirtualFileSystemTests.cs | 83 ++++++------------- Wabbajack.VirtualFileSystem/Context.cs | 39 ++------- Wabbajack.VirtualFileSystem/VirtualFile.cs | 63 ++++++-------- 4 files changed, 110 insertions(+), 133 deletions(-) diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index 05c151eb..f92c1613 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -197,7 +197,7 @@ namespace Wabbajack.Common } } - public class RelativePath : IPath + public struct RelativePath : IPath, IEquatable { private readonly string _path; private Extension _extension; @@ -205,9 +205,15 @@ namespace Wabbajack.Common public RelativePath(string path) { _path = path.ToLowerInvariant().Replace("/", "\\").Trim('\\'); + _extension = new Extension(Path.GetExtension(path)); Validate(); } + public override int GetHashCode() + { + return (_path != null ? _path.GetHashCode() : 0); + } + public static RelativePath RandomFileName() { return (RelativePath)Guid.NewGuid().ToString(); @@ -252,6 +258,26 @@ namespace Wabbajack.Common public RelativePath Parent => (RelativePath)Path.GetDirectoryName(_path); public RelativePath FileName => new RelativePath(Path.GetFileName(_path)); + + public bool Equals(RelativePath other) + { + return _path == other._path; + } + + public override bool Equals(object obj) + { + return obj is RelativePath other && Equals(other); + } + + public static bool operator ==(RelativePath a, RelativePath b) + { + return a._path == b._path; + } + + public static bool operator !=(RelativePath a, RelativePath b) + { + return !(a == b); + } } public static partial class Utils @@ -449,25 +475,35 @@ namespace Wabbajack.Common } } - public struct FullPath + public struct FullPath : IEquatable { public AbsolutePath Base { get; } public RelativePath[] Paths { get; } + private readonly int _hash; + public FullPath(AbsolutePath basePath, RelativePath[] paths) { Base = basePath; Paths = paths; + _hash = Base.GetHashCode(); + foreach (var itm in Paths) + _hash ^= itm.GetHashCode(); } - public string ToString() + public override string ToString() { - return string.Join("|", Paths.Select(t => t.ToString()).Cons(Base.ToString())); + return string.Join("|", Paths.Select(t => (string)t).Cons((string)Base)); } - + + public override int GetHashCode() + { + return _hash; + } + public static bool operator ==(FullPath a, FullPath b) { - if (a.Base != b.Base || a.Paths.Length == b.Paths.Length) + if (a.Base != b.Base || a.Paths.Length != b.Paths.Length) return false; for (int idx = 0; idx < a.Paths.Length; idx += 1) @@ -481,5 +517,15 @@ namespace Wabbajack.Common { return !(a == b); } + + public bool Equals(FullPath other) + { + return this == other; + } + + public override bool Equals(object obj) + { + return obj is FullPath other && Equals(other); + } } } diff --git a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs index 46d861ec..eca1b235 100644 --- a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs @@ -14,7 +14,7 @@ namespace Wabbajack.VirtualFileSystem.Test private static readonly AbsolutePath VFS_TEST_DIR = "vfs_test_dir".ToPath().RelativeToEntryPoint(); private static readonly AbsolutePath TEST_ZIP = "test.zip".RelativeTo(VFS_TEST_DIR); private static readonly AbsolutePath TEST_TXT = "test.txt".RelativeTo(VFS_TEST_DIR); - private static readonly AbsolutePath ARCHIVE_TEST_TXT = "archive/text.txt".RelativeTo(VFS_TEST_DIR); + private static readonly AbsolutePath ARCHIVE_TEST_TXT = "archive/test.txt".RelativeTo(VFS_TEST_DIR); private Context context; private readonly ITestOutputHelper _helper; @@ -52,8 +52,7 @@ namespace Wabbajack.VirtualFileSystem.Test } - /* - [TestMethod] + [Fact] public async Task ArchiveContentsAreIndexed() { await AddFile(ARCHIVE_TEST_TXT, "This is a test"); @@ -62,19 +61,20 @@ namespace Wabbajack.VirtualFileSystem.Test var absPath = "test.zip".RelativeTo(VFS_TEST_DIR); var file = context.Index.ByRootPath[absPath]; - Assert.IsNotNull(file); + Assert.NotNull(file); - Assert.AreEqual(128, file.Size); - Assert.AreEqual(absPath.FileHash(), file.Hash); + Assert.Equal(128, file.Size); + Assert.Equal(absPath.FileHash(), file.Hash); - Assert.IsTrue(file.IsArchive); + Assert.True(file.IsArchive); var innerFile = file.Children.First(); - Assert.AreEqual(14, innerFile.Size); - Assert.AreEqual("qX0GZvIaTKM=", innerFile.Hash); - Assert.AreSame(file, file.Children.First().Parent); + Assert.Equal(14, innerFile.Size); + Assert.Equal(Hash.FromBase64("qX0GZvIaTKM="), innerFile.Hash); + Assert.Same(file, file.Children.First().Parent); } + - [TestMethod] + [Fact] public async Task DuplicateFileHashes() { await AddFile(ARCHIVE_TEST_TXT, "This is a test"); @@ -85,29 +85,29 @@ namespace Wabbajack.VirtualFileSystem.Test var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")]; - Assert.AreEqual(files.Count(), 2); + Assert.Equal(2, files.Count()); } - [TestMethod] + [Fact] public async Task DeletedFilesAreRemoved() { await AddFile(TEST_TXT, "This is a test"); await AddTestRoot(); var file = context.Index.ByRootPath[TEST_TXT]; - Assert.IsNotNull(file); + Assert.NotNull(file); - Assert.AreEqual(file.Size, 14); - Assert.AreEqual(file.Hash, "qX0GZvIaTKM="); + Assert.Equal(14, file.Size); + Assert.Equal(Hash.FromBase64("qX0GZvIaTKM="), file.Hash); TEST_TXT.Delete(); await AddTestRoot(); - CollectionAssert.DoesNotContain(context.Index.ByFullPath, TEST_TXT); + Assert.DoesNotContain(TEST_TXT, context.Index.AllFiles.Select(f => f.AbsoluteName)); } - [TestMethod] + [Fact] public async Task UnmodifiedFilesAreNotReIndexed() { await AddFile(TEST_TXT, "This is a test"); @@ -120,25 +120,26 @@ namespace Wabbajack.VirtualFileSystem.Test var new_file = context.Index.ByRootPath[TEST_TXT]; - Assert.AreEqual(old_time, new_file.LastAnalyzed); + Assert.Equal(old_time, new_file.LastAnalyzed); } - [TestMethod] + [Fact] public async Task CanStageSimpleArchives() { await AddFile(ARCHIVE_TEST_TXT, "This is a test"); ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP); await AddTestRoot(); - var file = context.Index.ByFullPath[new FullPath(TEST_ZIP, new []{(RelativePath)"test.txt"})]; + var res = new FullPath(TEST_ZIP, new[] {(RelativePath)"test.txt"}); + var file = context.Index.ByFullPath[res]; var cleanup = await context.Stage(new List {file}); - Assert.AreEqual("This is a test", await file.StagedPath.ReadAllTextAsync()); + Assert.Equal("This is a test", await file.StagedPath.ReadAllTextAsync()); cleanup(); } - [TestMethod] + [Fact] public async Task CanStageNestedArchives() { await AddFile(ARCHIVE_TEST_TXT, "This is a test"); @@ -156,45 +157,11 @@ namespace Wabbajack.VirtualFileSystem.Test var cleanup = await context.Stage(files); foreach (var file in files) - Assert.AreEqual("This is a test", await file.StagedPath.ReadAllTextAsync()); + Assert.Equal("This is a test", await file.StagedPath.ReadAllTextAsync()); cleanup(); } - [TestMethod] - public async Task CanRequestPortableFileTrees() - { - await AddFile(ARCHIVE_TEST_TXT, "This is a test"); - ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP); - - @"archive\other\dir".RelativeTo(VFS_TEST_DIR).CreateDirectory(); - TEST_ZIP.MoveTo(@"archive\other\dir\nested.zip".RelativeTo(VFS_TEST_DIR)); - ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP); - - await AddTestRoot(); - - var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")]; - var archive = context.Index.ByRootPath[TEST_ZIP]; - - var state = context.GetPortableState(files); - - var newContext = new Context(Queue); - - await newContext.IntegrateFromPortable(state, - new Dictionary {{archive.Hash, archive.FullPath.Base}}); - - var newFiles = newContext.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")]; - - var close = await newContext.Stage(newFiles); - - foreach (var file in newFiles) - Assert.AreEqual("This is a test", await file.StagedPath.ReadAllTextAsync()); - - close(); - } - - */ - private static async Task AddFile(AbsolutePath filename, string text) { filename.Parent.CreateDirectory(); diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index 4f6b5eaa..fadc1c52 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -71,7 +71,7 @@ namespace Wabbajack.VirtualFileSystem return found; } - return await VirtualFile.Analyze(this, null, f, f, true); + return await VirtualFile.Analyze(this, null, f, f, 0); }); var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList()); @@ -102,7 +102,7 @@ namespace Wabbajack.VirtualFileSystem return found; } - return await VirtualFile.Analyze(this, null, f, f, true); + return await VirtualFile.Analyze(this, null, f, f, 0); }); var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList()); @@ -224,33 +224,6 @@ namespace Wabbajack.VirtualFileSystem }; } - public List GetPortableState(IEnumerable files) - { - return files.SelectMany(f => f.FilesInFullPath) - .Distinct() - .Select(f => new PortableFile - { - Name = f.Parent != null ? f.Name : null, - Hash = f.Hash, - ParentHash = f.Parent?.Hash ?? Hash.Empty, - Size = f.Size - }).ToList(); - } - - public async Task IntegrateFromPortable(List state, Dictionary links) - { - var indexedState = state.GroupBy(f => f.ParentHash) - .ToDictionary(f => f.Key, f => (IEnumerable) f); - var parents = await indexedState[Hash.Empty] - .PMap(Queue,f => VirtualFile.CreateFromPortable(this, indexedState, links, f)); - - var newIndex = await Index.Integrate(parents); - lock (this) - { - Index = newIndex; - } - } - public async Task> StageWith(IEnumerable files) { return new DisposableList(await Stage(files), files); @@ -329,7 +302,7 @@ namespace Wabbajack.VirtualFileSystem public static IndexRoot Empty = new IndexRoot(); public IndexRoot(ImmutableList aFiles, - ImmutableDictionary byFullPath, + Dictionary byFullPath, ImmutableDictionary> byHash, ImmutableDictionary byRoot, ImmutableDictionary> byName) @@ -344,7 +317,7 @@ namespace Wabbajack.VirtualFileSystem public IndexRoot() { AllFiles = ImmutableList.Empty; - ByFullPath = ImmutableDictionary.Empty; + ByFullPath = new Dictionary(); ByHash = ImmutableDictionary>.Empty; ByRootPath = ImmutableDictionary.Empty; ByName = ImmutableDictionary>.Empty; @@ -352,7 +325,7 @@ namespace Wabbajack.VirtualFileSystem public ImmutableList AllFiles { get; } - public ImmutableDictionary ByFullPath { get; } + public Dictionary ByFullPath { get; } public ImmutableDictionary> ByHash { get; } public ImmutableDictionary> ByName { get; set; } public ImmutableDictionary ByRootPath { get; } @@ -363,7 +336,7 @@ namespace Wabbajack.VirtualFileSystem var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList(); var byFullPath = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) - .ToImmutableDictionary(f => f.FullPath)); + .ToDictionary(f => f.FullPath)); var byHash = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) .Where(f => f.Hash != Hash.Empty) diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 23755cea..abee3909 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using K4os.Hash.Crc; using MessagePack; using Wabbajack.Common; +using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.VirtualFileSystem { @@ -132,7 +133,7 @@ namespace Wabbajack.VirtualFileSystem } public static async Task Analyze(Context context, VirtualFile parent, AbsolutePath absPath, - IPath relPath, bool topLevel) + IPath relPath, int depth = 0) { var hash = absPath.FileHash(); @@ -176,6 +177,9 @@ namespace Wabbajack.VirtualFileSystem LastAnalyzed = DateTime.Now.AsUnixTime(), Hash = hash }; + + self.FillFullPath(depth); + if (context.UseExtendedHashes) self.ExtendedHashes = ExtendedHashes.FromFile(absPath); @@ -187,7 +191,7 @@ namespace Wabbajack.VirtualFileSystem var list = await tempFolder.FullName.EnumerateFiles() .PMap(context.Queue, - absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), false)); + absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), depth + 1)); self.Children = list.ToImmutableList(); } @@ -196,6 +200,26 @@ namespace Wabbajack.VirtualFileSystem return self; } + private void FillFullPath(in int depth) + { + if (depth == 0) + { + FullPath = new FullPath((AbsolutePath)Name, new RelativePath[0]); + } + else + { + var paths = new RelativePath[depth]; + var self = this; + for (var idx = depth; idx != 0; idx -= 1) + { + paths[idx - 1] = self.RelativeName; + self = self.Parent; + } + FullPath = new FullPath(self.AbsoluteName, paths); + } + + } + private static async Task TryGetContentsFromServer(Hash hash) { try @@ -250,7 +274,7 @@ namespace Wabbajack.VirtualFileSystem Parent = parent, Children = ImmutableList.Empty }; - + vf.FullPath = new FullPath(vf.AbsoluteName, new RelativePath[0]); var children = br.ReadInt32(); for (var i = 0; i < children; i++) { @@ -286,39 +310,6 @@ namespace Wabbajack.VirtualFileSystem return vf; } - public static VirtualFile CreateFromPortable(Context context, - Dictionary> state, Dictionary links, - PortableFile portableFile) - { - var vf = new VirtualFile - { - Parent = null, - Context = context, - Name = links[portableFile.Hash], - Hash = portableFile.Hash, - Size = portableFile.Size - }; - if (state.TryGetValue(portableFile.Hash, out var children)) - vf.Children = children.Select(child => CreateFromPortable(context, vf, state, child)).ToImmutableList(); - return vf; - } - - public static VirtualFile CreateFromPortable(Context context, VirtualFile parent, - Dictionary> state, PortableFile portableFile) - { - var vf = new VirtualFile - { - Parent = parent, - Context = context, - Name = portableFile.Name, - Hash = portableFile.Hash, - Size = portableFile.Size - }; - if (state.TryGetValue(portableFile.Hash, out var children)) - vf.Children = children.Select(child => CreateFromPortable(context, vf, state, child)).ToImmutableList(); - return vf; - } - public HashRelativePath MakeRelativePaths() { var paths = new RelativePath[FilesInFullPath.Count() - 1];