Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

VFS Tests pass

Browse Source
This commit is contained in:
Timothy Baldridge 2020-03-24 15:42:28 -06:00
parent 2b45210159
commit 72d77bef1a
4 changed files with 110 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
Wabbajack.Common/Paths.cs
View File

@ -197,7 +197,7 @@ namespace Wabbajack.Common
}
}
public class RelativePath : IPath
public struct RelativePath : IPath, IEquatable<RelativePath>
{
private readonly string _path;
private Extension _extension;
@ -205,9 +205,15 @@ namespace Wabbajack.Common
public RelativePath(string path)
{
_path = path.ToLowerInvariant().Replace("/", "\\").Trim('\\');
_extension = new Extension(Path.GetExtension(path));
Validate();
}
public override int GetHashCode()
{
return (_path != null ? _path.GetHashCode() : 0);
}
public static RelativePath RandomFileName()
{
return (RelativePath)Guid.NewGuid().ToString();
@ -252,6 +258,26 @@ namespace Wabbajack.Common
public RelativePath Parent => (RelativePath)Path.GetDirectoryName(_path);
public RelativePath FileName => new RelativePath(Path.GetFileName(_path));
public bool Equals(RelativePath other)
{
return _path == other._path;
}
public override bool Equals(object obj)
{
return obj is RelativePath other && Equals(other);
}
public static bool operator ==(RelativePath a, RelativePath b)
{
return a._path == b._path;
}
public static bool operator !=(RelativePath a, RelativePath b)
{
return !(a == b);
}
}
public static partial class Utils
@ -449,25 +475,35 @@ namespace Wabbajack.Common
}
}
public struct FullPath
public struct FullPath : IEquatable<FullPath>
{
public AbsolutePath Base { get; }
public RelativePath[] Paths { get; }
private readonly int _hash;
public FullPath(AbsolutePath basePath, RelativePath[] paths)
{
Base = basePath;
Paths = paths;
_hash = Base.GetHashCode();
foreach (var itm in Paths)
_hash ^= itm.GetHashCode();
}
public string ToString()
public override string ToString()
{
return string.Join("|", Paths.Select(t => t.ToString()).Cons(Base.ToString()));
return string.Join("|", Paths.Select(t => (string)t).Cons((string)Base));
}
public override int GetHashCode()
{
return _hash;
}
public static bool operator ==(FullPath a, FullPath b)
{
if (a.Base != b.Base || a.Paths.Length == b.Paths.Length)
if (a.Base != b.Base || a.Paths.Length != b.Paths.Length)
return false;
for (int idx = 0; idx < a.Paths.Length; idx += 1)
@ -481,5 +517,15 @@ namespace Wabbajack.Common
{
return !(a == b);
}
public bool Equals(FullPath other)
{
return this == other;
}
public override bool Equals(object obj)
{
return obj is FullPath other && Equals(other);
}
}
}

83
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -14,7 +14,7 @@ namespace Wabbajack.VirtualFileSystem.Test
private static readonly AbsolutePath VFS_TEST_DIR = "vfs_test_dir".ToPath().RelativeToEntryPoint();
private static readonly AbsolutePath TEST_ZIP = "test.zip".RelativeTo(VFS_TEST_DIR);
private static readonly AbsolutePath TEST_TXT = "test.txt".RelativeTo(VFS_TEST_DIR);
private static readonly AbsolutePath ARCHIVE_TEST_TXT = "archive/text.txt".RelativeTo(VFS_TEST_DIR);
private static readonly AbsolutePath ARCHIVE_TEST_TXT = "archive/test.txt".RelativeTo(VFS_TEST_DIR);
private Context context;
private readonly ITestOutputHelper _helper;
@ -52,8 +52,7 @@ namespace Wabbajack.VirtualFileSystem.Test
}
/*
[TestMethod]
[Fact]
public async Task ArchiveContentsAreIndexed()
{
await AddFile(ARCHIVE_TEST_TXT, "This is a test");
@ -62,19 +61,20 @@ namespace Wabbajack.VirtualFileSystem.Test
var absPath = "test.zip".RelativeTo(VFS_TEST_DIR);
var file = context.Index.ByRootPath[absPath];
Assert.IsNotNull(file);
Assert.NotNull(file);
Assert.AreEqual(128, file.Size);
Assert.AreEqual(absPath.FileHash(), file.Hash);
Assert.Equal(128, file.Size);
Assert.Equal(absPath.FileHash(), file.Hash);
Assert.IsTrue(file.IsArchive);
Assert.True(file.IsArchive);
var innerFile = file.Children.First();
Assert.AreEqual(14, innerFile.Size);
Assert.AreEqual("qX0GZvIaTKM=", innerFile.Hash);
Assert.AreSame(file, file.Children.First().Parent);
Assert.Equal(14, innerFile.Size);
Assert.Equal(Hash.FromBase64("qX0GZvIaTKM="), innerFile.Hash);
Assert.Same(file, file.Children.First().Parent);
}
[TestMethod]
[Fact]
public async Task DuplicateFileHashes()
{
await AddFile(ARCHIVE_TEST_TXT, "This is a test");
@ -85,29 +85,29 @@ namespace Wabbajack.VirtualFileSystem.Test
var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")];
Assert.AreEqual(files.Count(), 2);
Assert.Equal(2, files.Count());
}
[TestMethod]
[Fact]
public async Task DeletedFilesAreRemoved()
{
await AddFile(TEST_TXT, "This is a test");
await AddTestRoot();
var file = context.Index.ByRootPath[TEST_TXT];
Assert.IsNotNull(file);
Assert.NotNull(file);
Assert.AreEqual(file.Size, 14);
Assert.AreEqual(file.Hash, "qX0GZvIaTKM=");
Assert.Equal(14, file.Size);
Assert.Equal(Hash.FromBase64("qX0GZvIaTKM="), file.Hash);
TEST_TXT.Delete();
await AddTestRoot();
CollectionAssert.DoesNotContain(context.Index.ByFullPath, TEST_TXT);
Assert.DoesNotContain(TEST_TXT, context.Index.AllFiles.Select(f => f.AbsoluteName));
}
[TestMethod]
[Fact]
public async Task UnmodifiedFilesAreNotReIndexed()
{
await AddFile(TEST_TXT, "This is a test");
@ -120,25 +120,26 @@ namespace Wabbajack.VirtualFileSystem.Test
var new_file = context.Index.ByRootPath[TEST_TXT];
Assert.AreEqual(old_time, new_file.LastAnalyzed);
Assert.Equal(old_time, new_file.LastAnalyzed);
}
[TestMethod]
[Fact]
public async Task CanStageSimpleArchives()
{
await AddFile(ARCHIVE_TEST_TXT, "This is a test");
ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await AddTestRoot();
var file = context.Index.ByFullPath[new FullPath(TEST_ZIP, new []{(RelativePath)"test.txt"})];
var res = new FullPath(TEST_ZIP, new[] {(RelativePath)"test.txt"});
var file = context.Index.ByFullPath[res];
var cleanup = await context.Stage(new List<VirtualFile> {file});
Assert.AreEqual("This is a test", await file.StagedPath.ReadAllTextAsync());
Assert.Equal("This is a test", await file.StagedPath.ReadAllTextAsync());
cleanup();
}
[TestMethod]
[Fact]
public async Task CanStageNestedArchives()
{
await AddFile(ARCHIVE_TEST_TXT, "This is a test");
@ -156,45 +157,11 @@ namespace Wabbajack.VirtualFileSystem.Test
var cleanup = await context.Stage(files);
foreach (var file in files)
Assert.AreEqual("This is a test", await file.StagedPath.ReadAllTextAsync());
Assert.Equal("This is a test", await file.StagedPath.ReadAllTextAsync());
cleanup();
}
[TestMethod]
public async Task CanRequestPortableFileTrees()
{
await AddFile(ARCHIVE_TEST_TXT, "This is a test");
ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
@"archive\other\dir".RelativeTo(VFS_TEST_DIR).CreateDirectory();
TEST_ZIP.MoveTo(@"archive\other\dir\nested.zip".RelativeTo(VFS_TEST_DIR));
ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await AddTestRoot();
var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")];
var archive = context.Index.ByRootPath[TEST_ZIP];
var state = context.GetPortableState(files);
var newContext = new Context(Queue);
await newContext.IntegrateFromPortable(state,
new Dictionary<Hash, AbsolutePath> {{archive.Hash, archive.FullPath.Base}});
var newFiles = newContext.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")];
var close = await newContext.Stage(newFiles);
foreach (var file in newFiles)
Assert.AreEqual("This is a test", await file.StagedPath.ReadAllTextAsync());
close();
}
*/
private static async Task AddFile(AbsolutePath filename, string text)
{
filename.Parent.CreateDirectory();

39
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -71,7 +71,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return await VirtualFile.Analyze(this, null, f, f, true);
return await VirtualFile.Analyze(this, null, f, f, 0);
});
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@ -102,7 +102,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return await VirtualFile.Analyze(this, null, f, f, true);
return await VirtualFile.Analyze(this, null, f, f, 0);
});
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@ -224,33 +224,6 @@ namespace Wabbajack.VirtualFileSystem
};
}
public List<PortableFile> GetPortableState(IEnumerable<VirtualFile> files)
{
return files.SelectMany(f => f.FilesInFullPath)
.Distinct()
.Select(f => new PortableFile
{
Name = f.Parent != null ? f.Name : null,
Hash = f.Hash,
ParentHash = f.Parent?.Hash ?? Hash.Empty,
Size = f.Size
}).ToList();
}
public async Task IntegrateFromPortable(List<PortableFile> state, Dictionary<Hash, AbsolutePath> links)
{
var indexedState = state.GroupBy(f => f.ParentHash)
.ToDictionary(f => f.Key, f => (IEnumerable<PortableFile>) f);
var parents = await indexedState[Hash.Empty]
.PMap(Queue,f => VirtualFile.CreateFromPortable(this, indexedState, links, f));
var newIndex = await Index.Integrate(parents);
lock (this)
{
Index = newIndex;
}
}
public async Task<DisposableList<VirtualFile>> StageWith(IEnumerable<VirtualFile> files)
{
return new DisposableList<VirtualFile>(await Stage(files), files);
@ -329,7 +302,7 @@ namespace Wabbajack.VirtualFileSystem
public static IndexRoot Empty = new IndexRoot();
public IndexRoot(ImmutableList<VirtualFile> aFiles,
ImmutableDictionary<FullPath, VirtualFile> byFullPath,
Dictionary<FullPath, VirtualFile> byFullPath,
ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> byHash,
ImmutableDictionary<AbsolutePath, VirtualFile> byRoot,
ImmutableDictionary<IPath, ImmutableStack<VirtualFile>> byName)
@ -344,7 +317,7 @@ namespace Wabbajack.VirtualFileSystem
public IndexRoot()
{
AllFiles = ImmutableList<VirtualFile>.Empty;
ByFullPath = ImmutableDictionary<FullPath, VirtualFile>.Empty;
ByFullPath = new Dictionary<FullPath, VirtualFile>();
ByHash = ImmutableDictionary<Hash, ImmutableStack<VirtualFile>>.Empty;
ByRootPath = ImmutableDictionary<AbsolutePath, VirtualFile>.Empty;
ByName = ImmutableDictionary<IPath, ImmutableStack<VirtualFile>>.Empty;
@ -352,7 +325,7 @@ namespace Wabbajack.VirtualFileSystem
public ImmutableList<VirtualFile> AllFiles { get; }
public ImmutableDictionary<FullPath, VirtualFile> ByFullPath { get; }
public Dictionary<FullPath, VirtualFile> ByFullPath { get; }
public ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> ByHash { get; }
public ImmutableDictionary<IPath, ImmutableStack<VirtualFile>> ByName { get; set; }
public ImmutableDictionary<AbsolutePath, VirtualFile> ByRootPath { get; }
@ -363,7 +336,7 @@ namespace Wabbajack.VirtualFileSystem
var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList();
var byFullPath = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToImmutableDictionary(f => f.FullPath));
.ToDictionary(f => f.FullPath));
var byHash = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.Where(f => f.Hash != Hash.Empty)

63
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -9,6 +9,7 @@ using System.Threading.Tasks;
using K4os.Hash.Crc;
using MessagePack;
using Wabbajack.Common;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.VirtualFileSystem
{
@ -132,7 +133,7 @@ namespace Wabbajack.VirtualFileSystem
}
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, AbsolutePath absPath,
IPath relPath, bool topLevel)
IPath relPath, int depth = 0)
{
var hash = absPath.FileHash();
@ -176,6 +177,9 @@ namespace Wabbajack.VirtualFileSystem
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = hash
};
self.FillFullPath(depth);
if (context.UseExtendedHashes)
self.ExtendedHashes = ExtendedHashes.FromFile(absPath);
@ -187,7 +191,7 @@ namespace Wabbajack.VirtualFileSystem
var list = await tempFolder.FullName.EnumerateFiles()
.PMap(context.Queue,
absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), false));
absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), depth + 1));
self.Children = list.ToImmutableList();
}
@ -196,6 +200,26 @@ namespace Wabbajack.VirtualFileSystem
return self;
}
private void FillFullPath(in int depth)
{
if (depth == 0)
{
FullPath = new FullPath((AbsolutePath)Name, new RelativePath[0]);
}
else
{
var paths = new RelativePath[depth];
var self = this;
for (var idx = depth; idx != 0; idx -= 1)
{
paths[idx - 1] = self.RelativeName;
self = self.Parent;
}
FullPath = new FullPath(self.AbsoluteName, paths);
}
}
private static async Task<IndexedVirtualFile> TryGetContentsFromServer(Hash hash)
{
try
@ -250,7 +274,7 @@ namespace Wabbajack.VirtualFileSystem
Parent = parent,
Children = ImmutableList<VirtualFile>.Empty
};
vf.FullPath = new FullPath(vf.AbsoluteName, new RelativePath[0]);
var children = br.ReadInt32();
for (var i = 0; i < children; i++)
{
@ -286,39 +310,6 @@ namespace Wabbajack.VirtualFileSystem
return vf;
}
public static VirtualFile CreateFromPortable(Context context,
Dictionary<Hash, IEnumerable<PortableFile>> state, Dictionary<Hash, AbsolutePath> links,
PortableFile portableFile)
{
var vf = new VirtualFile
{
Parent = null,
Context = context,
Name = links[portableFile.Hash],
Hash = portableFile.Hash,
Size = portableFile.Size
};
if (state.TryGetValue(portableFile.Hash, out var children))
vf.Children = children.Select(child => CreateFromPortable(context, vf, state, child)).ToImmutableList();
return vf;
}
public static VirtualFile CreateFromPortable(Context context, VirtualFile parent,
Dictionary<Hash, IEnumerable<PortableFile>> state, PortableFile portableFile)
{
var vf = new VirtualFile
{
Parent = parent,
Context = context,
Name = portableFile.Name,
Hash = portableFile.Hash,
Size = portableFile.Size
};
if (state.TryGetValue(portableFile.Hash, out var children))
vf.Children = children.Select(child => CreateFromPortable(context, vf, state, child)).ToImmutableList();
return vf;
}
public HashRelativePath MakeRelativePaths()
{
var paths = new RelativePath[FilesInFullPath.Count() - 1];