Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

WIP, VFS compiles, need to work on VFS tests

Browse Source
This commit is contained in:
Timothy Baldridge 2020-03-23 17:03:26 -06:00
parent b37728eefd
commit 80195b5620
5 changed files with 101 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
Wabbajack.Common/Paths.cs
View File

@ -134,6 +134,12 @@ namespace Wabbajack.Common
public bool Exists => File.Exists(_path) || Directory.Exists(_path);
public bool IsFile => File.Exists(_path);
public bool IsDirectory => Directory.Exists(_path);
public void DeleteDirectory()
{
if (IsDirectory)
Utils.DeleteDirectory(this);
}
public long Size => (new FileInfo(_path)).Length;
@ -203,6 +209,12 @@ namespace Wabbajack.Common
{
Directory.CreateDirectory(_path);
}
public void Delete()
{
if (IsFile)
File.Delete(_path);
}
}
public class RelativePath : AbstractPath
@ -260,6 +272,19 @@ namespace Wabbajack.Common
public RelativePath Parent => (RelativePath)Path.GetDirectoryName(_path);
}
public static partial class Utils
{
public static RelativePath ToPath(this string str)
{
return (RelativePath)str;
}
public static AbsolutePath RelativeTo(this string str, AbsolutePath path)
{
return ((RelativePath)str).RelativeTo(path);
}
}
public class Extension
{
#region ObjectEquality
@ -331,8 +356,8 @@ namespace Wabbajack.Common
public class HashRelativePath
{
public Hash BaseHash { get; }
public RelativePath[] Paths { get; }
public Hash BaseHash { get; set; }
public RelativePath[] Paths { get; set; } = new RelativePath[0];
public string ToString()
{

2
Wabbajack.Lib/Data.cs
View File

@ -22,7 +22,7 @@ namespace Wabbajack.Lib
Path = path;
}
public string AbsolutePath => File.StagedPath;
public AbsolutePath AbsolutePath => File.StagedPath;
public VirtualFile File { get; }

26
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -12,8 +12,7 @@ namespace Wabbajack.VirtualFileSystem.Test
[TestClass]
public class VFSTests
{
private const string VFS_TEST_DIR = "vfs_test_dir";
private static readonly string VFS_TEST_DIR_FULL = Path.Combine(Directory.GetCurrentDirectory(), VFS_TEST_DIR);
private readonly AbsolutePath VFS_TEST_DIR = "vfs_test_dir".ToPath().RelativeToEntryPoint();
private Context context;
public TestContext TestContext { get; set; }
@ -23,9 +22,8 @@ namespace Wabbajack.VirtualFileSystem.Test
public void Setup()
{
Utils.LogMessages.Subscribe(f => TestContext.WriteLine(f.ShortDescription));
if (Directory.Exists(VFS_TEST_DIR))
Utils.DeleteDirectory(VFS_TEST_DIR);
Directory.CreateDirectory(VFS_TEST_DIR);
VFS_TEST_DIR.DeleteDirectory();
VFS_TEST_DIR.CreateDirectory();
Queue = new WorkQueue();
context = new Context(Queue);
}
@ -36,7 +34,7 @@ namespace Wabbajack.VirtualFileSystem.Test
AddFile("test.txt", "This is a test");
await AddTestRoot();
var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
var file = context.Index.ByRootPath["test.txt".ToPath().RelativeTo(VFS_TEST_DIR)];
Assert.IsNotNull(file);
Assert.AreEqual(file.Size, 14);
@ -45,9 +43,9 @@ namespace Wabbajack.VirtualFileSystem.Test
private async Task AddTestRoot()
{
await context.AddRoot(VFS_TEST_DIR_FULL);
await context.WriteToFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
await context.IntegrateFromFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
await context.AddRoot(VFS_TEST_DIR);
await context.WriteToFile("vfs_cache.bin".RelativeTo(VFS_TEST_DIR));
await context.IntegrateFromFile( "vfs_cache.bin".RelativeTo(VFS_TEST_DIR));
}
@ -58,17 +56,17 @@ namespace Wabbajack.VirtualFileSystem.Test
ZipUpFolder("archive", "test.zip");
await AddTestRoot();
var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip");
var file = context.Index.ByFullPath[abs_path];
var abs_path = "test.zip".RelativeTo(VFS_TEST_DIR);
var file = context.Index.ByRootPath[abs_path];
Assert.IsNotNull(file);
Assert.AreEqual(128, file.Size);
Assert.AreEqual(abs_path.FileHash(), file.Hash);
Assert.IsTrue(file.IsArchive);
var inner_file = file.Children.First();
Assert.AreEqual(14, inner_file.Size);
Assert.AreEqual("qX0GZvIaTKM=", inner_file.Hash);
var innerFile = file.Children.First();
Assert.AreEqual(14, innerFile.Size);
Assert.AreEqual("qX0GZvIaTKM=", innerFile.Hash);
Assert.AreSame(file, file.Children.First().Parent);
}

116
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -134,68 +134,63 @@ namespace Wabbajack.VirtualFileSystem
});
}
public async Task WriteToFile(string filename)
public async Task WriteToFile(AbsolutePath filename)
{
using (var fs = File.Open(filename, FileMode.Create))
using (var bw = new BinaryWriter(fs, Encoding.UTF8, true))
{
fs.SetLength(0);
await using var fs = filename.Create();
await using var bw = new BinaryWriter(fs, Encoding.UTF8, true);
fs.SetLength(0);
bw.Write(Encoding.ASCII.GetBytes(Magic));
bw.Write(FileVersion);
bw.Write((ulong) Index.AllFiles.Count);
bw.Write(Encoding.ASCII.GetBytes(Magic));
bw.Write(FileVersion);
bw.Write((ulong) Index.AllFiles.Count);
(await Index.AllFiles
(await Index.AllFiles
.PMap(Queue, f =>
{
var ms = new MemoryStream();
f.Write(ms);
return ms;
}))
.Do(ms =>
{
var size = ms.Position;
ms.Position = 0;
bw.Write((ulong) size);
ms.CopyTo(fs);
});
Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}");
}
.Do(ms =>
{
var size = ms.Position;
ms.Position = 0;
bw.Write((ulong) size);
ms.CopyTo(fs);
});
Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}");
}
public async Task IntegrateFromFile(string filename)
public async Task IntegrateFromFile(AbsolutePath filename)
{
try
{
using (var fs = File.OpenRead(filename))
using (var br = new BinaryReader(fs, Encoding.UTF8, true))
{
var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length));
var fileVersion = br.ReadUInt64();
if (fileVersion != FileVersion || magic != Magic)
throw new InvalidDataException("Bad Data Format");
await using var fs = filename.OpenRead();
using var br = new BinaryReader(fs, Encoding.UTF8, true);
var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length));
var fileVersion = br.ReadUInt64();
if (fileVersion != FileVersion || magic != Magic)
throw new InvalidDataException("Bad Data Format");
var numFiles = br.ReadUInt64();
var numFiles = br.ReadUInt64();
var files = Enumerable.Range(0, (int) numFiles)
.Select(idx =>
{
var size = br.ReadUInt64();
var bytes = new byte[size];
br.BaseStream.Read(bytes, 0, (int) size);
return VirtualFile.Read(this, bytes);
}).ToList();
var newIndex = await Index.Integrate(files);
lock (this)
var files = Enumerable.Range(0, (int) numFiles)
.Select(idx =>
{
Index = newIndex;
}
var size = br.ReadUInt64();
var bytes = new byte[size];
br.BaseStream.Read(bytes, 0, (int) size);
return VirtualFile.Read(this, bytes);
}).ToList();
var newIndex = await Index.Integrate(files);
lock (this)
{
Index = newIndex;
}
}
catch (IOException)
{
if (File.Exists(filename))
File.Delete(filename);
filename.Delete();
}
}
@ -208,23 +203,22 @@ namespace Wabbajack.VirtualFileSystem
.OrderBy(f => f.Key?.NestingFactor ?? 0)
.ToList();
var paths = new List<string>();
var paths = new List<AbsolutePath>();
foreach (var group in grouped)
{
var tmpPath = Path.Combine(StagingFolder, Guid.NewGuid().ToString());
var tmpPath = ((RelativePath)Guid.NewGuid().ToString()).RelativeTo(StagingFolder);
await FileExtractor.ExtractAll(Queue, group.Key.StagedPath, tmpPath);
paths.Add(tmpPath);
foreach (var file in group)
file.StagedPath = Path.Combine(tmpPath, file.Name);
file.StagedPath = file.RelativeName.RelativeTo(tmpPath);
}
return () =>
{
paths.Do(p =>
{
if (Directory.Exists(p))
Utils.DeleteDirectory(p);
p.DeleteDirectory();
});
};
}
@ -242,7 +236,7 @@ namespace Wabbajack.VirtualFileSystem
}).ToList();
}
public async Task IntegrateFromPortable(List<PortableFile> state, Dictionary<Hash, string> links)
public async Task IntegrateFromPortable(List<PortableFile> state, Dictionary<Hash, AbsolutePath> links)
{
var indexedState = state.GroupBy(f => f.ParentHash)
.ToDictionary(f => f.Key, f => (IEnumerable<PortableFile>) f);
@ -264,28 +258,28 @@ namespace Wabbajack.VirtualFileSystem
#region KnownFiles
private List<KnownFile> _knownFiles = new List<KnownFile>();
public void AddKnown(IEnumerable<KnownFile> known)
private List<HashRelativePath> _knownFiles = new List<HashRelativePath>();
public void AddKnown(IEnumerable<HashRelativePath> known)
{
_knownFiles.AddRange(known);
}
public async Task BackfillMissing()
{
var newFiles = _knownFiles.Where(f => f.Paths.Length == 1)
.GroupBy(f => f.Hash)
var newFiles = _knownFiles.Where(f => f.Paths.Length == 0)
.GroupBy(f => f.BaseHash)
.ToDictionary(f => f.Key, s => new VirtualFile()
{
Name = s.First().Paths[0],
Hash = s.First().Hash,
Hash = s.First().BaseHash,
Context = this
});
var parentchild = new Dictionary<(VirtualFile, string), VirtualFile>();
var parentchild = new Dictionary<(VirtualFile, RelativePath), VirtualFile>();
void BackFillOne(KnownFile file)
void BackFillOne(HashRelativePath file)
{
var parent = newFiles[Hash.FromBase64(file.Paths[0])];
var parent = newFiles[file.BaseHash];
foreach (var path in file.Paths.Skip(1))
{
if (parentchild.TryGetValue((parent, path), out var foundParent))
@ -294,9 +288,7 @@ namespace Wabbajack.VirtualFileSystem
continue;
}
var nf = new VirtualFile();
nf.Name = path;
nf.Parent = parent;
var nf = new VirtualFile {Name = path, Parent = parent};
parent.Children = parent.Children.Add(nf);
parentchild.Add((parent, path), nf);
parent = nf;
@ -309,19 +301,13 @@ namespace Wabbajack.VirtualFileSystem
lock (this)
Index = newIndex;
_knownFiles = new List<KnownFile>();
_knownFiles = new List<HashRelativePath>();
}
#endregion
}
public class KnownFile
{
public string[] Paths { get; set; }
public Hash Hash { get; set; }
}
public class DisposableList<T> : List<T>, IDisposable
{
private Action _unstage;

17
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -22,6 +22,9 @@ namespace Wabbajack.VirtualFileSystem
private AbsolutePath _stagedPath;
public AbstractPath Name { get; internal set; }
public RelativePath RelativeName => Name as RelativePath;
public AbsolutePath AbsoluteName => Name as AbsolutePath;
public FullPath FullPath
{
@ -235,7 +238,7 @@ namespace Wabbajack.VirtualFileSystem
}
private void Write(Stream stream)
public void Write(Stream stream)
{
stream.WriteAsMessagePack(this);
}
@ -289,16 +292,16 @@ namespace Wabbajack.VirtualFileSystem
return vf;
}
public string[] MakeRelativePaths()
public HashRelativePath MakeRelativePaths()
{
var path = new string[NestingFactor];
path[0] = FilesInFullPath.First().Hash.ToBase64();
var idx = 1;
var path = new HashRelativePath();
path.BaseHash = FilesInFullPath.First().Hash;
path.Paths = new RelativePath[FilesInFullPath.Count() - 1];
var idx = 0;
foreach (var itm in FilesInFullPath.Skip(1))
{
path[idx] = itm.Name;
path.Paths[idx] = itm.Name as RelativePath;
idx += 1;
}
return path;