Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

One test passes

Browse Source
This commit is contained in:
Timothy Baldridge 2020-03-24 06:21:19 -06:00
parent a2cf84de54
commit 2b45210159
7 changed files with 227 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
Wabbajack.Common/Paths.cs
View File

@ -14,32 +14,15 @@ using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Common
{
public class AbstractPath
public interface IPath
{
public RelativePath FileName
{
get
{
switch (this)
{
case AbsolutePath abs:
return abs.FileName;
case RelativePath rel:
return rel.FileName;
}
return null;
}
}
}
public class AbsolutePath : AbstractPath
public struct AbsolutePath : IPath
{
#region ObjectEquality
protected bool Equals(AbsolutePath other)
bool Equals(AbsolutePath other)
{
return _path == other._path;
}
@ -76,12 +59,14 @@ namespace Wabbajack.Common
public AbsolutePath(string path)
{
_path = path.ToLowerInvariant().Replace("/", "\\").TrimEnd('\\');
_extension = new Extension(Path.GetExtension(_path));
ValidateAbsolutePath();
}
public AbsolutePath(string path, bool skipValidation)
{
_path = path.ToLowerInvariant().Replace("/", "\\").TrimEnd('\\');
_extension = Extension.FromPath(path);
if (!skipValidation)
ValidateAbsolutePath();
}
@ -89,6 +74,7 @@ namespace Wabbajack.Common
public AbsolutePath(AbsolutePath path)
{
_path = path._path;
_extension = path._extension;
}
private void ValidateAbsolutePath()
@ -96,19 +82,8 @@ namespace Wabbajack.Common
if (Path.IsPathRooted(_path)) return;
throw new InvalidDataException($"Absolute path must be absolute");
}
public Extension Extension
{
get
{
if (_extension != null) return _extension;
var extension = Path.GetExtension(_path);
if (string.IsNullOrEmpty(extension))
return null;
_extension = (Extension)extension;
return _extension;
}
}
public Extension Extension => _extension;
public FileStream OpenRead()
{
@ -222,7 +197,7 @@ namespace Wabbajack.Common
}
}
public class RelativePath : AbstractPath
public class RelativePath : IPath
{
private readonly string _path;
private Extension _extension;
@ -275,6 +250,8 @@ namespace Wabbajack.Common
}
public RelativePath Parent => (RelativePath)Path.GetDirectoryName(_path);
public RelativePath FileName => new RelativePath(Path.GetFileName(_path));
}
public static partial class Utils
@ -288,12 +265,58 @@ namespace Wabbajack.Common
{
return ((RelativePath)str).RelativeTo(path);
}
public static void Write(this BinaryWriter wtr, IPath path)
{
wtr.Write(path is AbsolutePath);
if (path is AbsolutePath)
wtr.Write((AbsolutePath)path);
else
wtr.Write((RelativePath)path);
}
public static void Write(this BinaryWriter wtr, AbsolutePath path)
{
wtr.Write((string)path);
}
public static void Write(this BinaryWriter wtr, RelativePath path)
{
wtr.Write((string)path);
}
public static IPath ReadIPath(this BinaryReader rdr)
{
if (rdr.ReadBoolean())
return rdr.ReadAbsolutePath();
return rdr.ReadRelativePath();
}
public static AbsolutePath ReadAbsolutePath(this BinaryReader rdr)
{
return new AbsolutePath(rdr.ReadString());
}
public static RelativePath ReadRelativePath(this BinaryReader rdr)
{
return new RelativePath(rdr.ReadString());
}
public static T[] Add<T>(this T[] arr, T itm)
{
var newArr = new T[arr.Length + 1];
Array.Copy(arr, 0, newArr, 0, arr.Length);
newArr[arr.Length] = itm;
return newArr;
}
}
public class Extension
public struct Extension
{
public static Extension None = new Extension("", false);
#region ObjectEquality
protected bool Equals(Extension other)
bool Equals(Extension other)
{
return _extension == other._extension;
}
@ -328,9 +351,28 @@ namespace Wabbajack.Common
public Extension(string extension)
{
if (string.IsNullOrWhiteSpace(extension))
{
_extension = None._extension;
return;
}
_extension = string.Intern(extension);
Validate();
}
private Extension(string extension, bool validate)
{
_extension = string.Intern(extension);
if (validate) Validate();
}
public Extension(Extension other)
{
_extension = other._extension;
}
private void Validate()
{
if (!_extension.StartsWith("."))
@ -350,8 +392,8 @@ namespace Wabbajack.Common
public static bool operator ==(Extension a, Extension b)
{
// Super fast comparison because extensions are interned
if (a == null && b == null) return true;
if (a == null || b == null) return false;
if ((object)a == null && (object)b == null) return true;
if ((object)a == null || (object)b == null) return false;
return ReferenceEquals(a._extension, b._extension);
}
@ -359,12 +401,30 @@ namespace Wabbajack.Common
{
return !(a == b);
}
public static Extension FromPath(string path)
{
var ext = Path.GetExtension(path);
return !string.IsNullOrWhiteSpace(ext) ? new Extension(ext) : None;
}
}
public class HashRelativePath
public struct HashRelativePath
{
public Hash BaseHash { get; set; }
public RelativePath[] Paths { get; set; } = new RelativePath[0];
private static RelativePath[] EMPTY_PATH;
public Hash BaseHash { get; }
public RelativePath[] Paths { get; }
static HashRelativePath()
{
EMPTY_PATH = new RelativePath[0];
}
public HashRelativePath(Hash baseHash, params RelativePath[] paths)
{
BaseHash = baseHash;
Paths = paths;
}
public string ToString()
{
@ -389,7 +449,7 @@ namespace Wabbajack.Common
}
}
public class FullPath
public struct FullPath
{
public AbsolutePath Base { get; }
public RelativePath[] Paths { get; }
@ -397,7 +457,7 @@ namespace Wabbajack.Common
public FullPath(AbsolutePath basePath, RelativePath[] paths)
{
Base = basePath;
Paths = Paths;
Paths = paths;
}
public string ToString()

27
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -3,13 +3,12 @@ using System.Collections.Generic;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.VirtualFileSystem.Test
{
[TestClass]
public class VFSTests
{
private static readonly AbsolutePath VFS_TEST_DIR = "vfs_test_dir".ToPath().RelativeToEntryPoint();
@ -18,32 +17,33 @@ namespace Wabbajack.VirtualFileSystem.Test
private static readonly AbsolutePath ARCHIVE_TEST_TXT = "archive/text.txt".RelativeTo(VFS_TEST_DIR);
private Context context;
public TestContext TestContext { get; set; }
public WorkQueue Queue { get; set; }
private readonly ITestOutputHelper _helper;
private WorkQueue Queue { get; }
[TestInitialize]
public void Setup()
public VFSTests(ITestOutputHelper helper)
{
Utils.LogMessages.Subscribe(f => TestContext.WriteLine(f.ShortDescription));
_helper = helper;
Utils.LogMessages.Subscribe(f => _helper.WriteLine(f.ShortDescription));
VFS_TEST_DIR.DeleteDirectory();
VFS_TEST_DIR.CreateDirectory();
Queue = new WorkQueue();
context = new Context(Queue);
}
[TestMethod]
[Fact]
public async Task FilesAreIndexed()
{
await AddFile(TEST_TXT, "This is a test");
await AddTestRoot();
var file = context.Index.ByRootPath["test.txt".ToPath().RelativeTo(VFS_TEST_DIR)];
Assert.IsNotNull(file);
Assert.NotNull(file);
Assert.AreEqual(file.Size, 14);
Assert.AreEqual(file.Hash, "qX0GZvIaTKM=");
Assert.Equal(14, file.Size);
Assert.Equal(file.Hash, Hash.FromBase64("qX0GZvIaTKM="));
}
private async Task AddTestRoot()
{
await context.AddRoot(VFS_TEST_DIR);
@ -52,6 +52,7 @@ namespace Wabbajack.VirtualFileSystem.Test
}
/*
[TestMethod]
public async Task ArchiveContentsAreIndexed()
{
@ -191,6 +192,8 @@ namespace Wabbajack.VirtualFileSystem.Test
close();
}
*/
private static async Task AddFile(AbsolutePath filename, string text)
{

6
Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj
View File

@ -8,9 +8,9 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.4.0" />
<PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
<PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.console" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
</ItemGroup>
<ItemGroup>

11
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -148,7 +148,8 @@ namespace Wabbajack.VirtualFileSystem
.PMap(Queue, f =>
{
var ms = new MemoryStream();
f.Write(ms);
using var ibw = new BinaryWriter(ms, Encoding.UTF8, true);
f.Write(ibw);
return ms;
}))
.Do(ms =>
@ -331,7 +332,7 @@ namespace Wabbajack.VirtualFileSystem
ImmutableDictionary<FullPath, VirtualFile> byFullPath,
ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> byHash,
ImmutableDictionary<AbsolutePath, VirtualFile> byRoot,
ImmutableDictionary<AbstractPath, ImmutableStack<VirtualFile>> byName)
ImmutableDictionary<IPath, ImmutableStack<VirtualFile>> byName)
{
AllFiles = aFiles;
ByFullPath = byFullPath;
@ -346,14 +347,14 @@ namespace Wabbajack.VirtualFileSystem
ByFullPath = ImmutableDictionary<FullPath, VirtualFile>.Empty;
ByHash = ImmutableDictionary<Hash, ImmutableStack<VirtualFile>>.Empty;
ByRootPath = ImmutableDictionary<AbsolutePath, VirtualFile>.Empty;
ByName = ImmutableDictionary<AbstractPath, ImmutableStack<VirtualFile>>.Empty;
ByName = ImmutableDictionary<IPath, ImmutableStack<VirtualFile>>.Empty;
}
public ImmutableList<VirtualFile> AllFiles { get; }
public ImmutableDictionary<FullPath, VirtualFile> ByFullPath { get; }
public ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> ByHash { get; }
public ImmutableDictionary<AbstractPath, ImmutableStack<VirtualFile>> ByName { get; set; }
public ImmutableDictionary<IPath, ImmutableStack<VirtualFile>> ByName { get; set; }
public ImmutableDictionary<AbsolutePath, VirtualFile> ByRootPath { get; }
public async Task<IndexRoot> Integrate(ICollection<VirtualFile> files)
@ -371,7 +372,7 @@ namespace Wabbajack.VirtualFileSystem
var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToGroupedImmutableDictionary(f => f.Name));
var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.Name as AbsolutePath));
var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.AbsoluteName));
var result = new IndexRoot(allFiles,
await byFullPath,

2
Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs
View File

@ -8,7 +8,7 @@ namespace Wabbajack.VirtualFileSystem
/// </summary>
public class IndexedVirtualFile
{
public AbstractPath Name { get; set; }
public IPath Name { get; set; }
public Hash Hash { get; set; }
public long Size { get; set; }
public List<IndexedVirtualFile> Children { get; set; } = new List<IndexedVirtualFile>();

2
Wabbajack.VirtualFileSystem/PortableFile.cs
View File

@ -4,7 +4,7 @@ namespace Wabbajack.VirtualFileSystem
{
public class PortableFile
{
public AbstractPath Name { get; set; }
public IPath Name { get; set; }
public Hash Hash { get; set; }
public Hash ParentHash { get; set; }
public long Size { get; set; }

169
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -4,48 +4,31 @@ using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using K4os.Hash.Crc;
using MessagePack;
using Wabbajack.Common;
using Wabbajack.Common.CSP;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.VirtualFileSystem
{
public class VirtualFile
{
private FullPath _fullPath;
private AbsolutePath _stagedPath;
public AbstractPath Name { get; internal set; }
public RelativePath RelativeName => Name as RelativePath;
public AbsolutePath AbsoluteName => Name as AbsolutePath;
public FullPath FullPath
{
get
{
if (_fullPath != null) return _fullPath;
private IEnumerable<VirtualFile> _thisAndAllChildren;
var cur = this;
var acc = new LinkedList<AbstractPath>();
while (cur != null)
{
acc.AddFirst(cur.Name);
cur = cur.Parent;
}
public IPath Name { get; internal set; }
_fullPath = new FullPath(acc.First() as AbsolutePath, acc.Skip(1).OfType<RelativePath>().ToArray());
return _fullPath;
}
}
public RelativePath RelativeName => (RelativePath)Name;
public AbsolutePath AbsoluteName => (AbsolutePath)Name;
public FullPath FullPath { get; private set; }
public Hash Hash { get; internal set; }
public ExtendedHashes ExtendedHashes { get; set; }
public long Size { get; internal set; }
@ -62,7 +45,7 @@ namespace Wabbajack.VirtualFileSystem
get
{
if (IsNative)
return Name as AbsolutePath;
return (AbsolutePath)Name;
if (_stagedPath == null)
throw new UnstagedFileException(FullPath);
return _stagedPath;
@ -101,8 +84,6 @@ namespace Wabbajack.VirtualFileSystem
public bool IsNative => Parent == null;
private IEnumerable<VirtualFile> _thisAndAllChildren = null;
public IEnumerable<VirtualFile> ThisAndAllChildren
{
get
@ -115,20 +96,6 @@ namespace Wabbajack.VirtualFileSystem
return _thisAndAllChildren;
}
}
public T ThisAndAllChildrenReduced<T>(T acc, Func<T, VirtualFile, T> fn)
{
acc = fn(acc, this);
return this.Children.Aggregate(acc, (current, itm) => itm.ThisAndAllChildrenReduced<T>(current, fn));
}
public void ThisAndAllChildrenReduced(Action<VirtualFile> fn)
{
fn(this);
foreach (var itm in Children)
itm.ThisAndAllChildrenReduced(fn);
}
/// <summary>
@ -150,8 +117,22 @@ namespace Wabbajack.VirtualFileSystem
}
}
public T ThisAndAllChildrenReduced<T>(T acc, Func<T, VirtualFile, T> fn)
{
acc = fn(acc, this);
return Children.Aggregate(acc, (current, itm) => itm.ThisAndAllChildrenReduced(current, fn));
}
public void ThisAndAllChildrenReduced(Action<VirtualFile> fn)
{
fn(this);
foreach (var itm in Children)
itm.ThisAndAllChildrenReduced(fn);
}
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, AbsolutePath absPath,
AbstractPath relPath, bool topLevel)
IPath relPath, bool topLevel)
{
var hash = absPath.FileHash();
@ -162,7 +143,8 @@ namespace Wabbajack.VirtualFileSystem
if (result != null)
{
Utils.Log($"Downloaded VFS data for {(string)absPath}");
VirtualFile Convert(IndexedVirtualFile file, AbstractPath path, VirtualFile vparent)
VirtualFile Convert(IndexedVirtualFile file, IPath path, VirtualFile vparent)
{
var vself = new VirtualFile
{
@ -172,8 +154,7 @@ namespace Wabbajack.VirtualFileSystem
Size = file.Size,
LastModified = absPath.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = file.Hash,
Hash = file.Hash
};
vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList();
@ -200,17 +181,16 @@ namespace Wabbajack.VirtualFileSystem
if (FileExtractor.CanExtract(absPath))
{
using (var tempFolder = Context.GetTemporaryFolder())
{
await FileExtractor.ExtractAll(context.Queue, absPath, tempFolder.FullName);
var list = await tempFolder.FullName.EnumerateFiles()
.PMap(context.Queue, absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), false));
.PMap(context.Queue,
absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), false));
self.Children = list.ToImmutableList();
}
}
return self;
@ -221,7 +201,8 @@ namespace Wabbajack.VirtualFileSystem
try
{
var client = new HttpClient();
var response = await client.GetAsync($"http://{Consts.WabbajackCacheHostname}/indexed_files/{hash.ToHex()}");
var response =
await client.GetAsync($"http://{Consts.WabbajackCacheHostname}/indexed_files/{hash.ToHex()}");
if (!response.IsSuccessStatusCode)
return null;
@ -229,7 +210,6 @@ namespace Wabbajack.VirtualFileSystem
{
return stream.FromJSON<IndexedVirtualFile>();
}
}
catch (Exception ex)
{
@ -238,24 +218,71 @@ namespace Wabbajack.VirtualFileSystem
}
public void Write(Stream stream)
public void Write(BinaryWriter bw)
{
stream.WriteAsMessagePack(this);
bw.Write(Name);
bw.Write(Size);
bw.Write(LastModified);
bw.Write(LastModified);
bw.Write(Hash);
bw.Write(Children.Count);
foreach (var child in Children)
child.Write(bw);
}
public static VirtualFile Read(Context context, byte[] data)
{
using var ms = new MemoryStream(data);
return Read(context, null, ms);
using var br = new BinaryReader(ms);
return Read(context, null, br);
}
private static VirtualFile Read(Context context, VirtualFile parent, Stream br)
private static VirtualFile Read(Context context, VirtualFile parent, BinaryReader br)
{
var vf = br.ReadAsMessagePack<VirtualFile>();
vf.Parent = parent;
vf.Context = context;
vf.Children ??= ImmutableList<VirtualFile>.Empty;
var vf = new VirtualFile
{
Name = br.ReadIPath(),
Size = br.ReadInt64(),
LastModified = br.ReadUInt64(),
LastAnalyzed = br.ReadUInt64(),
Hash = br.ReadHash(),
Context = context,
Parent = parent,
Children = ImmutableList<VirtualFile>.Empty
};
var children = br.ReadInt32();
for (var i = 0; i < children; i++)
{
var child = Read(context, vf, br, (AbsolutePath)vf.Name, new RelativePath[0]);
vf.Children = vf.Children.Add(child);
}
return vf;
}
private static VirtualFile Read(Context context, VirtualFile parent, BinaryReader br, AbsolutePath top, RelativePath[] subpaths)
{
var name = (RelativePath)br.ReadIPath();
subpaths = subpaths.Add(name);
var vf = new VirtualFile
{
Name = name,
Size = br.ReadInt64(),
LastModified = br.ReadUInt64(),
LastAnalyzed = br.ReadUInt64(),
Hash = br.ReadHash(),
Context = context,
Parent = parent,
Children = ImmutableList<VirtualFile>.Empty,
FullPath = new FullPath(top, subpaths)
};
var children = br.ReadInt32();
for (var i = 0; i < children; i++)
{
var child = Read(context, vf, br,top, subpaths);
vf.Children = vf.Children.Add(child);
}
return vf;
}
@ -294,16 +321,16 @@ namespace Wabbajack.VirtualFileSystem
public HashRelativePath MakeRelativePaths()
{
var path = new HashRelativePath();
path.BaseHash = FilesInFullPath.First().Hash;
path.Paths = new RelativePath[FilesInFullPath.Count() - 1];
var paths = new RelativePath[FilesInFullPath.Count() - 1];
var idx = 0;
foreach (var itm in FilesInFullPath.Skip(1))
{
path.Paths[idx] = itm.Name as RelativePath;
paths[idx] = (RelativePath)itm.Name;
idx += 1;
}
var path = new HashRelativePath(FilesInFullPath.First().Hash, paths);
return path;
}
@ -315,6 +342,11 @@ namespace Wabbajack.VirtualFileSystem
public class ExtendedHashes
{
public string SHA256 { get; set; }
public string SHA1 { get; set; }
public string MD5 { get; set; }
public string CRC { get; set; }
public static ExtendedHashes FromFile(AbsolutePath file)
{
var hashes = new ExtendedHashes();
@ -341,11 +373,6 @@ namespace Wabbajack.VirtualFileSystem
return hashes;
}
public string SHA256 { get; set; }
public string SHA1 { get; set; }
public string MD5 { get; set; }
public string CRC { get; set; }
}