diff --git a/Compression.BSA.Test/Compression.BSA.Test.csproj b/Compression.BSA.Test/Compression.BSA.Test.csproj
index 191e765e..288dc349 100644
--- a/Compression.BSA.Test/Compression.BSA.Test.csproj
+++ b/Compression.BSA.Test/Compression.BSA.Test.csproj
@@ -89,10 +89,10 @@
2.2.6
- 1.3.2
+ 2.0.0
- 1.3.2
+ 2.0.0
diff --git a/VirtualFileSystem/VirtualFileSystem.cs b/VirtualFileSystem/VirtualFileSystem.cs
index 6c441a64..acbcfdcf 100644
--- a/VirtualFileSystem/VirtualFileSystem.cs
+++ b/VirtualFileSystem/VirtualFileSystem.cs
@@ -44,7 +44,10 @@ namespace VFS
public static void Reconfigure(string root)
{
RootFolder = root;
- _stagedRoot = Path.Combine(RootFolder, "vfs_staged_files");
+ if (RootFolder != null)
+ _stagedRoot = Path.Combine(RootFolder, "vfs_staged_files");
+ else
+ _stagedRoot = "vfs_staged_files";
}
public static void Clean()
diff --git a/VirtualFileSystem/VirtualFileSystem.csproj b/VirtualFileSystem/VirtualFileSystem.csproj
index b365484b..facaa800 100644
--- a/VirtualFileSystem/VirtualFileSystem.csproj
+++ b/VirtualFileSystem/VirtualFileSystem.csproj
@@ -112,7 +112,7 @@
1.2.0
- 1.5.0
+ 1.6.0
diff --git a/Wabbajack.Common.CSP/ManyToManyChannel.cs b/Wabbajack.Common.CSP/ManyToManyChannel.cs
index 464d2840..25239c94 100644
--- a/Wabbajack.Common.CSP/ManyToManyChannel.cs
+++ b/Wabbajack.Common.CSP/ManyToManyChannel.cs
@@ -122,7 +122,7 @@ namespace Wabbajack.Common.CSP
Monitor.Exit(this);
throw new TooManyHanldersException();
}
- _puts.Unshift((handler, val));
+ _puts.UnboundedUnshift((handler, val));
}
Monitor.Exit(this);
return (AsyncResult.Enqueued, true);
@@ -191,7 +191,7 @@ namespace Wabbajack.Common.CSP
throw new TooManyHanldersException();
}
- _takes.Unshift(handler);
+ _takes.UnboundedUnshift(handler);
}
Monitor.Exit(this);
return (AsyncResult.Enqueued, default);
diff --git a/Wabbajack.Common.CSP/PIpelines.cs b/Wabbajack.Common.CSP/PIpelines.cs
index 86fd826c..01c3b795 100644
--- a/Wabbajack.Common.CSP/PIpelines.cs
+++ b/Wabbajack.Common.CSP/PIpelines.cs
@@ -121,6 +121,59 @@ namespace Wabbajack.Common.CSP
}
+ public static IReadPort UnorderedPipelineRx(
+ this IReadPort from,
+ Func, IObservable> f,
+ bool propagateClose = true)
+ {
+ var parallelism = Environment.ProcessorCount;
+ var to = Channel.Create(parallelism * 2);
+ var pipeline = from.UnorderedPipeline(parallelism, to, f);
+ return to;
+
+ }
+
+ public static IReadPort UnorderedPipelineSync(
+ this IReadPort from,
+ Func f,
+ bool propagateClose = true)
+ {
+ var parallelism = Environment.ProcessorCount;
+ var to = Channel.Create(parallelism * 2);
+
+ async Task Pump()
+ {
+ while (true)
+ {
+ var (is_open, job) = await from.Take();
+ if (!is_open) break;
+ try
+ {
+ var putIsOpen = await to.Put(f(job));
+ if (!putIsOpen) return;
+ }
+ catch (Exception ex)
+ {
+
+ }
+ }
+ }
+
+ Task.Run(async () =>
+ {
+ await Task.WhenAll(Enumerable.Range(0, parallelism)
+ .Select(idx => Task.Run(Pump)));
+
+ if (propagateClose)
+ {
+ from.Close();
+ to.Close();
+ }
+ });
+
+ return to;
+ }
+
public static async Task UnorderedThreadedPipeline(
this IReadPort from,
int parallelism,
diff --git a/Wabbajack.Common.CSP/RingBuffer.cs b/Wabbajack.Common.CSP/RingBuffer.cs
index 24344d2c..5b59b522 100644
--- a/Wabbajack.Common.CSP/RingBuffer.cs
+++ b/Wabbajack.Common.CSP/RingBuffer.cs
@@ -1,6 +1,8 @@
using System;
using System.Collections;
using System.Collections.Generic;
+using System.ComponentModel;
+using System.IO;
namespace Wabbajack.Common.CSP
{
@@ -23,7 +25,8 @@ namespace Wabbajack.Common.CSP
public T Pop()
{
- if (_length == 0) return default;
+ if (_length == 0)
+ throw new InvalidDataException("Pop on empty buffer");
var val = _arr[_tail];
_arr[_tail] = default;
_tail = (_tail + 1) % _size;
@@ -45,7 +48,7 @@ namespace Wabbajack.Common.CSP
public void UnboundedUnshift(T x)
{
- if (_length == _size)
+ if (_length + 1 == _size)
Resize();
Unshift(x);
}
@@ -67,8 +70,8 @@ namespace Wabbajack.Common.CSP
}
else if (_tail > _head)
{
- Array.Copy(_arr, _tail, new_arr, 0, _length - _tail);
- Array.Copy(_arr, 0, new_arr, (_length - _tail), _head);
+ Array.Copy(_arr, _tail, new_arr, 0, _size - _tail);
+ Array.Copy(_arr, 0, new_arr, (_size - _tail), _head);
_tail = 0;
_head = _length;
_arr = new_arr;
diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs
index ed87903d..183230fd 100644
--- a/Wabbajack.Common/Utils.cs
+++ b/Wabbajack.Common/Utils.cs
@@ -125,6 +125,28 @@ namespace Wabbajack.Common
}
}
+ public static async Task FileHashAsync(this string file, bool nullOnIOError = false)
+ {
+ try
+ {
+ var hash = new xxHashConfig();
+ hash.HashSizeInBits = 64;
+ hash.Seed = 0x42;
+ using (var fs = File.OpenRead(file))
+ {
+ var config = new xxHashConfig();
+ config.HashSizeInBits = 64;
+ var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(fs);
+ return value.AsBase64String();
+ }
+ }
+ catch (IOException ex)
+ {
+ if (nullOnIOError) return null;
+ throw ex;
+ }
+ }
+
public static void CopyToWithStatus(this Stream istream, long maxSize, Stream ostream, string status)
{
var buffer = new byte[1024 * 64];
diff --git a/Wabbajack.Test.ListValidation/Wabbajack.Test.ListValidation.csproj b/Wabbajack.Test.ListValidation/Wabbajack.Test.ListValidation.csproj
index 55a57841..bb5da801 100644
--- a/Wabbajack.Test.ListValidation/Wabbajack.Test.ListValidation.csproj
+++ b/Wabbajack.Test.ListValidation/Wabbajack.Test.ListValidation.csproj
@@ -76,10 +76,10 @@
- 1.3.2
+ 2.0.0
- 1.3.2
+ 2.0.0
12.0.2
diff --git a/Wabbajack.Test/Wabbajack.Test.csproj b/Wabbajack.Test/Wabbajack.Test.csproj
index b934ad1e..e4d751b3 100644
--- a/Wabbajack.Test/Wabbajack.Test.csproj
+++ b/Wabbajack.Test/Wabbajack.Test.csproj
@@ -84,6 +84,7 @@
+
@@ -137,10 +138,10 @@
2.2.6
- 1.3.2
+ 2.0.0
- 1.3.2
+ 2.0.0
12.0.2
diff --git a/Wabbajack.VirtualFileSystem.Test/Properties/AssemblyInfo.cs b/Wabbajack.VirtualFileSystem.Test/Properties/AssemblyInfo.cs
new file mode 100644
index 00000000..6e7a9eaf
--- /dev/null
+++ b/Wabbajack.VirtualFileSystem.Test/Properties/AssemblyInfo.cs
@@ -0,0 +1,19 @@
+using System.Reflection;
+using System.Runtime.InteropServices;
+
+[assembly: AssemblyTitle("Wabbajack.VirtualFileSystem.Test")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("Wabbajack.VirtualFileSystem.Test")]
+[assembly: AssemblyCopyright("Copyright © 2019")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+[assembly: ComVisible(false)]
+
+[assembly: Guid("51ceb604-985a-45b9-af0d-c5ba8cfa1bf0")]
+
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
\ No newline at end of file
diff --git a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
new file mode 100644
index 00000000..940cd21e
--- /dev/null
+++ b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
@@ -0,0 +1,209 @@
+using System;
+using System.Collections.Generic;
+using System.IO.Compression;
+using System.Linq;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem.Test
+{
+ [TestClass]
+ public class VFSTests
+ {
+ private const string VFS_TEST_DIR = "vfs_test_dir";
+ private static readonly string VFS_TEST_DIR_FULL = Path.Combine(Directory.GetCurrentDirectory(), VFS_TEST_DIR);
+ private Context context;
+
+ public TestContext TestContext { get; set; }
+
+ [TestInitialize]
+ public void Setup()
+ {
+ Utils.LogMessages.Subscribe(f => TestContext.WriteLine(f));
+ if (Directory.Exists(VFS_TEST_DIR))
+ Directory.Delete(VFS_TEST_DIR, true);
+ Directory.CreateDirectory(VFS_TEST_DIR);
+ context = new Context();
+ }
+
+ [TestMethod]
+ public async Task FilesAreIndexed()
+ {
+ AddFile("test.txt", "This is a test");
+ await AddTestRoot();
+
+ var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
+ Assert.IsNotNull(file);
+
+ Assert.AreEqual(file.Size, 14);
+ Assert.AreEqual(file.Hash, "qX0GZvIaTKM=");
+ }
+
+ private async Task AddTestRoot()
+ {
+ await context.AddRoot(VFS_TEST_DIR_FULL);
+ await context.WriteToFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
+ await context.IntegrateFromFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
+ }
+
+
+ [TestMethod]
+ public async Task ArchiveContentsAreIndexed()
+ {
+ AddFile("archive/test.txt", "This is a test");
+ ZipUpFolder("archive", "test.zip");
+ await AddTestRoot();
+
+ var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip");
+ var file = context.Index.ByFullPath[abs_path];
+ Assert.IsNotNull(file);
+
+ Assert.AreEqual(128, file.Size);
+ Assert.AreEqual(abs_path.FileHash(), file.Hash);
+
+ Assert.IsTrue(file.IsArchive);
+ var inner_file = file.Children.First();
+ Assert.AreEqual(14, inner_file.Size);
+ Assert.AreEqual("qX0GZvIaTKM=", inner_file.Hash);
+ Assert.AreSame(file, file.Children.First().Parent);
+ }
+
+ [TestMethod]
+ public async Task DuplicateFileHashes()
+ {
+ AddFile("archive/test.txt", "This is a test");
+ ZipUpFolder("archive", "test.zip");
+
+ AddFile("test.txt", "This is a test");
+ await AddTestRoot();
+
+
+ var files = context.Index.ByHash["qX0GZvIaTKM="];
+ Assert.AreEqual(files.Count(), 2);
+ }
+
+ [TestMethod]
+ public async Task DeletedFilesAreRemoved()
+ {
+ AddFile("test.txt", "This is a test");
+ await AddTestRoot();
+
+ var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
+ Assert.IsNotNull(file);
+
+ Assert.AreEqual(file.Size, 14);
+ Assert.AreEqual(file.Hash, "qX0GZvIaTKM=");
+
+ File.Delete(Path.Combine(VFS_TEST_DIR_FULL, "test.txt"));
+
+ await AddTestRoot();
+
+ CollectionAssert.DoesNotContain(context.Index.ByFullPath, Path.Combine(VFS_TEST_DIR_FULL, "test.txt"));
+ }
+
+ [TestMethod]
+ public async Task UnmodifiedFilesAreNotReIndexed()
+ {
+ AddFile("test.txt", "This is a test");
+ await AddTestRoot();
+
+ var old_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
+ var old_time = old_file.LastAnalyzed;
+
+ await AddTestRoot();
+
+ var new_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
+
+ Assert.AreEqual(old_time, new_file.LastAnalyzed);
+ }
+
+ [TestMethod]
+ public async Task CanStageSimpleArchives()
+ {
+ AddFile("archive/test.txt", "This is a test");
+ ZipUpFolder("archive", "test.zip");
+ await AddTestRoot();
+
+ var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip");
+ var file = context.Index.ByFullPath[abs_path + "|test.txt"];
+
+ var cleanup = context.Stage(new List {file});
+ Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
+
+ cleanup();
+ }
+
+ [TestMethod]
+ public async Task CanStageNestedArchives()
+ {
+ AddFile("archive/test.txt", "This is a test");
+ ZipUpFolder("archive", "test.zip");
+
+ Directory.CreateDirectory(Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir"));
+ File.Move(Path.Combine(VFS_TEST_DIR_FULL, "test.zip"),
+ Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip"));
+ ZipUpFolder("archive", "test.zip");
+
+ await AddTestRoot();
+
+ var files = context.Index.ByHash["qX0GZvIaTKM="];
+
+ var cleanup = context.Stage(files);
+
+ foreach (var file in files)
+ Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
+
+ cleanup();
+ }
+
+ [TestMethod]
+ public async Task CanRequestPortableFileTrees()
+ {
+ AddFile("archive/test.txt", "This is a test");
+ ZipUpFolder("archive", "test.zip");
+
+ Directory.CreateDirectory(Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir"));
+ File.Move(Path.Combine(VFS_TEST_DIR_FULL, "test.zip"),
+ Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip"));
+ ZipUpFolder("archive", "test.zip");
+
+ await AddTestRoot();
+
+ var files = context.Index.ByHash["qX0GZvIaTKM="];
+ var archive = context.Index.ByRootPath[Path.Combine(VFS_TEST_DIR_FULL, "test.zip")];
+
+ var state = context.GetPortableState(files);
+
+ var new_context = new Context();
+
+ await new_context.IntegrateFromPortable(state,
+ new Dictionary {{archive.Hash, archive.FullPath}});
+
+ var new_files = new_context.Index.ByHash["qX0GZvIaTKM="];
+
+ var close = new_context.Stage(new_files);
+
+ foreach (var file in new_files)
+ Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
+
+ close();
+ }
+
+ private static void AddFile(string filename, string thisIsATest)
+ {
+ var fullpath = Path.Combine(VFS_TEST_DIR, filename);
+ if (!Directory.Exists(Path.GetDirectoryName(fullpath)))
+ Directory.CreateDirectory(Path.GetDirectoryName(fullpath));
+ File.WriteAllText(fullpath, thisIsATest);
+ }
+
+ private static void ZipUpFolder(string folder, string output)
+ {
+ var path = Path.Combine(VFS_TEST_DIR, folder);
+ ZipFile.CreateFromDirectory(path, Path.Combine(VFS_TEST_DIR, output));
+ Directory.Delete(path, true);
+ }
+ }
+}
\ No newline at end of file
diff --git a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj
new file mode 100644
index 00000000..b014ded3
--- /dev/null
+++ b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj
@@ -0,0 +1,99 @@
+
+
+
+
+ Debug
+ AnyCPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}
+ Library
+ Properties
+ Wabbajack.VirtualFileSystem.Test
+ Wabbajack.VirtualFileSystem.Test
+ v4.7.2
+ 512
+ {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}
+ 15.0
+ $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)
+ $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages
+ False
+ UnitTest
+
+
+
+
+ true
+ full
+ false
+ bin\Debug\
+ DEBUG;TRACE
+ prompt
+ 4
+
+
+ pdbonly
+ true
+ bin\Release\
+ TRACE
+ prompt
+ 4
+
+
+ true
+ bin\x64\Debug\
+ DEBUG;TRACE
+ full
+ x64
+ 7.3
+ prompt
+ MinimumRecommendedRules.ruleset
+
+
+ bin\x64\Release\
+ TRACE
+ true
+ pdbonly
+ x64
+ 7.3
+ prompt
+ MinimumRecommendedRules.ruleset
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}
+ Wabbajack.Common
+
+
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}
+ Wabbajack.VirtualFileSystem
+
+
+
+
+ 2.2.6
+
+
+ 2.0.0
+
+
+ 2.0.0
+
+
+ 4.2.0
+
+
+
+
+
\ No newline at end of file
diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs
new file mode 100644
index 00000000..9935ace5
--- /dev/null
+++ b/Wabbajack.VirtualFileSystem/Context.cs
@@ -0,0 +1,267 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using Wabbajack.Common;
+using Wabbajack.Common.CSP;
+using Directory = Alphaleonis.Win32.Filesystem.Directory;
+using File = System.IO.File;
+using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
+using Path = Alphaleonis.Win32.Filesystem.Path;
+
+namespace Wabbajack.VirtualFileSystem
+{
+ public class Context
+ {
+ public const ulong FileVersion = 0x02;
+ public const string Magic = "WABBAJACK VFS FILE";
+
+ private readonly string _stagingFolder = "vfs_staging";
+ public IndexRoot Index { get; private set; } = IndexRoot.Empty;
+
+ public TemporaryDirectory GetTemporaryFolder()
+ {
+ return new TemporaryDirectory(Path.Combine(_stagingFolder, Guid.NewGuid().ToString()));
+ }
+
+ public async Task AddRoot(string root)
+ {
+ if (!Path.IsPathRooted(root))
+ throw new InvalidDataException($"Path is not absolute: {root}");
+
+ var filtered = await Index.AllFiles
+ .ToChannel()
+ .UnorderedPipelineRx(o => o.Where(file => File.Exists(file.Name)))
+ .TakeAll();
+
+ var byPath = filtered.ToImmutableDictionary(f => f.Name);
+
+ var results = Channel.Create(1024);
+ var pipeline = Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive)
+ .ToChannel()
+ .UnorderedPipeline(results, async f =>
+ {
+ if (byPath.TryGetValue(f, out var found))
+ {
+ var fi = new FileInfo(f);
+ if (found.LastModified == fi.LastWriteTimeUtc.Ticks && found.Size == fi.Length)
+ return found;
+ }
+
+ return await VirtualFile.Analyze(this, null, f, f);
+ });
+
+ var allFiles = await results.TakeAll();
+
+ // Should already be done but let's make the async tracker happy
+ await pipeline;
+
+ var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
+
+ lock (this)
+ {
+ Index = newIndex;
+ }
+
+ return newIndex;
+ }
+
+ public async Task WriteToFile(string filename)
+ {
+ using (var fs = File.OpenWrite(filename))
+ using (var bw = new BinaryWriter(fs, Encoding.UTF8, true))
+ {
+ fs.SetLength(0);
+
+ bw.Write(Encoding.ASCII.GetBytes(Magic));
+ bw.Write(FileVersion);
+ bw.Write((ulong) Index.AllFiles.Count);
+
+ var sizes = await Index.AllFiles
+ .ToChannel()
+ .UnorderedPipelineSync(f =>
+ {
+ var ms = new MemoryStream();
+ f.Write(ms);
+ return ms;
+ })
+ .Select(async ms =>
+ {
+ var size = ms.Position;
+ ms.Position = 0;
+ bw.Write((ulong) size);
+ await ms.CopyToAsync(fs);
+ return ms.Position;
+ })
+ .TakeAll();
+ Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}");
+ }
+ }
+
+ public async Task IntegrateFromFile(string filename)
+ {
+ using (var fs = File.OpenRead(filename))
+ using (var br = new BinaryReader(fs, Encoding.UTF8, true))
+ {
+ var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length));
+ var fileVersion = br.ReadUInt64();
+ if (fileVersion != FileVersion || magic != magic)
+ throw new InvalidDataException("Bad Data Format");
+
+ var numFiles = br.ReadUInt64();
+
+ var input = Channel.Create(1024);
+ var pipeline = input.UnorderedPipelineSync(
+ data => VirtualFile.Read(this, data))
+ .TakeAll();
+
+ Utils.Log($"Loading {numFiles} files from {filename}");
+
+ for (ulong idx = 0; idx < numFiles; idx++)
+ {
+ var size = br.ReadUInt64();
+ var bytes = new byte[size];
+ await br.BaseStream.ReadAsync(bytes, 0, (int) size);
+ await input.Put(bytes);
+ }
+
+ input.Close();
+
+ var files = await pipeline;
+ var newIndex = await Index.Integrate(files);
+ lock (this)
+ {
+ Index = newIndex;
+ }
+ }
+ }
+
+ public Action Stage(IEnumerable files)
+ {
+ var grouped = files.SelectMany(f => f.FilesInFullPath)
+ .Distinct()
+ .Where(f => f.Parent != null)
+ .GroupBy(f => f.Parent)
+ .OrderBy(f => f.Key?.NestingFactor ?? 0)
+ .ToList();
+
+ var paths = new List();
+
+ foreach (var group in grouped)
+ {
+ var tmpPath = Path.Combine(_stagingFolder, Guid.NewGuid().ToString());
+ FileExtractor.ExtractAll(group.Key.StagedPath, tmpPath).Wait();
+ paths.Add(tmpPath);
+ foreach (var file in group)
+ file.StagedPath = Path.Combine(tmpPath, file.Name);
+ }
+
+ return () =>
+ {
+ paths.Do(p =>
+ {
+ if (Directory.Exists(p))
+ Directory.Delete(p, true, true);
+ });
+ };
+ }
+
+ public List GetPortableState(IEnumerable files)
+ {
+ return files.SelectMany(f => f.FilesInFullPath)
+ .Distinct()
+ .Select(f => new PortableFile
+ {
+ Name = f.Parent != null ? f.Name : null,
+ Hash = f.Hash,
+ ParentHash = f.Parent?.Hash,
+ Size = f.Size
+ }).ToList();
+ }
+
+ public async Task IntegrateFromPortable(List state, Dictionary links)
+ {
+ var indexedState = state.GroupBy(f => f.ParentHash)
+ .ToDictionary(f => f.Key ?? "", f => (IEnumerable) f);
+ var parents = await indexedState[""]
+ .ToChannel()
+ .UnorderedPipelineSync(f => VirtualFile.CreateFromPortable(this, indexedState, links, f))
+ .TakeAll();
+
+ var newIndex = await Index.Integrate(parents);
+ lock (this)
+ {
+ Index = newIndex;
+ }
+ }
+ }
+
+ public class IndexRoot
+ {
+ public static IndexRoot Empty = new IndexRoot();
+
+ public IndexRoot(ImmutableList aFiles,
+ ImmutableDictionary byFullPath,
+ ImmutableDictionary> byHash,
+ ImmutableDictionary byRoot)
+ {
+ AllFiles = aFiles;
+ ByFullPath = byFullPath;
+ ByHash = byHash;
+ ByRootPath = byRoot;
+ }
+
+ public IndexRoot()
+ {
+ AllFiles = ImmutableList.Empty;
+ ByFullPath = ImmutableDictionary.Empty;
+ ByHash = ImmutableDictionary>.Empty;
+ ByRootPath = ImmutableDictionary.Empty;
+ }
+
+ public ImmutableList AllFiles { get; }
+ public ImmutableDictionary ByFullPath { get; }
+ public ImmutableDictionary> ByHash { get; }
+ public ImmutableDictionary ByRootPath { get; }
+
+ public async Task Integrate(List files)
+ {
+ var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList();
+
+ var byFullPath = Task.Run(() =>
+ allFiles.SelectMany(f => f.ThisAndAllChildren)
+ .ToImmutableDictionary(f => f.FullPath));
+
+ var byHash = Task.Run(() =>
+ allFiles.SelectMany(f => f.ThisAndAllChildren)
+ .ToGroupedImmutableDictionary(f => f.Hash));
+
+ var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.Name));
+
+ return new IndexRoot(allFiles,
+ await byFullPath,
+ await byHash,
+ await byRootPath);
+ }
+ }
+
+ public class TemporaryDirectory : IDisposable
+ {
+ public TemporaryDirectory(string name)
+ {
+ FullName = name;
+ }
+
+ public string FullName { get; }
+
+ public void Dispose()
+ {
+ Directory.Delete(FullName, true, true);
+ }
+ }
+}
\ No newline at end of file
diff --git a/Wabbajack.VirtualFileSystem/Extensions.cs b/Wabbajack.VirtualFileSystem/Extensions.cs
new file mode 100644
index 00000000..92021027
--- /dev/null
+++ b/Wabbajack.VirtualFileSystem/Extensions.cs
@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+
+namespace Wabbajack.VirtualFileSystem
+{
+ public static class Extensions
+ {
+ public static ImmutableDictionary ToImmutableDictionary(this IEnumerable coll,
+ Func keyFunc)
+ {
+ var builder = ImmutableDictionary.Empty.ToBuilder();
+ foreach (var itm in coll)
+ builder.Add(keyFunc(itm), itm);
+ return builder.ToImmutable();
+ }
+
+ public static ImmutableDictionary> ToGroupedImmutableDictionary(
+ this IEnumerable coll, Func keyFunc)
+ {
+ var builder = ImmutableDictionary>.Empty.ToBuilder();
+ foreach (var itm in coll)
+ {
+ var key = keyFunc(itm);
+ if (builder.TryGetValue(key, out var prev))
+ builder[key] = prev.Push(itm);
+ else
+ builder[key] = ImmutableStack.Empty.Push(itm);
+ }
+
+ return builder.ToImmutable();
+ }
+ }
+}
\ No newline at end of file
diff --git a/Wabbajack.VirtualFileSystem/PortableFile.cs b/Wabbajack.VirtualFileSystem/PortableFile.cs
new file mode 100644
index 00000000..8b3a7e78
--- /dev/null
+++ b/Wabbajack.VirtualFileSystem/PortableFile.cs
@@ -0,0 +1,10 @@
+namespace Wabbajack.VirtualFileSystem
+{
+ public class PortableFile
+ {
+ public string Name { get; set; }
+ public string Hash { get; set; }
+ public string ParentHash { get; set; }
+ public long Size { get; set; }
+ }
+}
\ No newline at end of file
diff --git a/Wabbajack.VirtualFileSystem/Properties/AssemblyInfo.cs b/Wabbajack.VirtualFileSystem/Properties/AssemblyInfo.cs
new file mode 100644
index 00000000..27b88b86
--- /dev/null
+++ b/Wabbajack.VirtualFileSystem/Properties/AssemblyInfo.cs
@@ -0,0 +1,35 @@
+using System.Reflection;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("Wabbajack.VirtualFileSystem")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("Wabbajack.VirtualFileSystem")]
+[assembly: AssemblyCopyright("Copyright © 2019")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components. If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("5d6a2eaf-6604-4c51-8ae2-a746b4bc5e3e")]
+
+// Version information for an assembly consists of the following four values:
+//
+// Major Version
+// Minor Version
+// Build Number
+// Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
\ No newline at end of file
diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs
new file mode 100644
index 00000000..02729671
--- /dev/null
+++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs
@@ -0,0 +1,251 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Common.CSP;
+using Directory = Alphaleonis.Win32.Filesystem.Directory;
+using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
+using Path = Alphaleonis.Win32.Filesystem.Path;
+
+namespace Wabbajack.VirtualFileSystem
+{
+ public class VirtualFile
+ {
+ private string _fullPath;
+
+ private string _stagedPath;
+ public string Name { get; internal set; }
+
+ public string FullPath
+ {
+ get
+ {
+ if (_fullPath != null) return _fullPath;
+ var cur = this;
+ var acc = new LinkedList();
+ while (cur != null)
+ {
+ acc.AddFirst(cur.Name);
+ cur = cur.Parent;
+ }
+
+ _fullPath = string.Join("|", acc);
+
+ return _fullPath;
+ }
+ }
+
+ public string Hash { get; internal set; }
+ public long Size { get; internal set; }
+
+ public long LastModified { get; internal set; }
+
+ public long LastAnalyzed { get; internal set; }
+
+ public VirtualFile Parent { get; internal set; }
+
+ public Context Context { get; set; }
+
+ public string StagedPath
+ {
+ get
+ {
+ if (IsNative)
+ return Name;
+ if (_stagedPath == null)
+ throw new UnstagedFileException(FullPath);
+ return _stagedPath;
+ }
+ internal set
+ {
+ if (IsNative)
+ throw new CannotStageNativeFile("Cannot stage a native file");
+ _stagedPath = value;
+ }
+ }
+
+ ///
+ /// Returns the nesting factor for this file. Native files will have a nesting of 1, the factor
+ /// goes up for each nesting of a file in an archive.
+ ///
+ public int NestingFactor
+ {
+ get
+ {
+ var cnt = 0;
+ var cur = this;
+ while (cur != null)
+ {
+ cnt += 1;
+ cur = cur.Parent;
+ }
+
+ return cnt;
+ }
+ }
+
+ public ImmutableList Children { get; internal set; } = ImmutableList.Empty;
+
+ public bool IsArchive => Children != null && Children.Count > 0;
+
+ public bool IsNative => Parent == null;
+
+ public IEnumerable ThisAndAllChildren =>
+ Children.SelectMany(child => child.ThisAndAllChildren).Append(this);
+
+
+ ///
+ /// Returns all the virtual files in the path to this file, starting from the root file.
+ ///
+ public IEnumerable FilesInFullPath
+ {
+ get
+ {
+ var stack = ImmutableStack.Empty;
+ var cur = this;
+ while (cur != null)
+ {
+ stack = stack.Push(cur);
+ cur = cur.Parent;
+ }
+
+ return stack;
+ }
+ }
+
+ public static async Task Analyze(Context context, VirtualFile parent, string abs_path,
+ string rel_path)
+ {
+ var hasher = abs_path.FileHashAsync();
+ var fi = new FileInfo(abs_path);
+ var self = new VirtualFile
+ {
+ Context = context,
+ Name = rel_path,
+ Parent = parent,
+ Size = fi.Length,
+ LastModified = fi.LastWriteTimeUtc.Ticks,
+ LastAnalyzed = DateTime.Now.Ticks
+ };
+
+ if (FileExtractor.CanExtract(Path.GetExtension(abs_path)))
+ using (var tempFolder = context.GetTemporaryFolder())
+ {
+ await FileExtractor.ExtractAll(abs_path, tempFolder.FullName);
+
+ var results = Channel.Create(1024);
+ var files = Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories)
+ .ToChannel()
+ .UnorderedPipeline(results,
+ async abs_src => await Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName)));
+ self.Children = (await results.TakeAll()).ToImmutableList();
+ }
+
+ self.Hash = await hasher;
+ return self;
+ }
+
+ public void Write(MemoryStream ms)
+ {
+ using (var bw = new BinaryWriter(ms, Encoding.UTF8, true))
+ {
+ Write(bw);
+ }
+ }
+
+ private void Write(BinaryWriter bw)
+ {
+ bw.Write(Name);
+ bw.Write(Hash);
+ bw.Write(Size);
+ bw.Write(LastModified);
+ bw.Write(LastAnalyzed);
+ bw.Write(Children.Count);
+ foreach (var child in Children)
+ child.Write(bw);
+ }
+
+ public static VirtualFile Read(Context context, byte[] data)
+ {
+ using (var ms = new MemoryStream(data))
+ using (var br = new BinaryReader(ms))
+ {
+ return Read(context, null, br);
+ }
+ }
+
+ private static VirtualFile Read(Context context, VirtualFile parent, BinaryReader br)
+ {
+ var vf = new VirtualFile
+ {
+ Context = context,
+ Parent = parent,
+ Name = br.ReadString(),
+ Hash = br.ReadString(),
+ Size = br.ReadInt64(),
+ LastModified = br.ReadInt64(),
+ LastAnalyzed = br.ReadInt64(),
+ Children = ImmutableList.Empty
+ };
+
+ var childrenCount = br.ReadInt32();
+ for (var idx = 0; idx < childrenCount; idx += 1) vf.Children = vf.Children.Add(Read(context, vf, br));
+
+ return vf;
+ }
+
+ public static VirtualFile CreateFromPortable(Context context,
+ Dictionary> state, Dictionary links,
+ PortableFile portableFile)
+ {
+ var vf = new VirtualFile
+ {
+ Parent = null,
+ Context = context,
+ Name = links[portableFile.Hash],
+ Hash = portableFile.Hash,
+ Size = portableFile.Size
+ };
+ if (state.TryGetValue(portableFile.Hash, out var children))
+ vf.Children = children.Select(child => CreateFromPortable(context, vf, state, child)).ToImmutableList();
+ return vf;
+ }
+
+ public static VirtualFile CreateFromPortable(Context context, VirtualFile parent,
+ Dictionary> state, PortableFile portableFile)
+ {
+ var vf = new VirtualFile
+ {
+ Parent = parent,
+ Context = context,
+ Name = portableFile.Name,
+ Hash = portableFile.Hash,
+ Size = portableFile.Size
+ };
+ if (state.TryGetValue(portableFile.Hash, out var children))
+ vf.Children = children.Select(child => CreateFromPortable(context, vf, state, child)).ToImmutableList();
+ return vf;
+ }
+ }
+
+ public class CannotStageNativeFile : Exception
+ {
+ public CannotStageNativeFile(string cannotStageANativeFile) : base(cannotStageANativeFile)
+ {
+ }
+ }
+
+ public class UnstagedFileException : Exception
+ {
+ private readonly string _fullPath;
+
+ public UnstagedFileException(string fullPath) : base($"File {fullPath} is unstaged, cannot get staged name")
+ {
+ _fullPath = fullPath;
+ }
+ }
+}
\ No newline at end of file
diff --git a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj
new file mode 100644
index 00000000..34328510
--- /dev/null
+++ b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj
@@ -0,0 +1,91 @@
+
+
+
+
+ Debug
+ AnyCPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}
+ Library
+ Properties
+ Wabbajack.VirtualFileSystem
+ Wabbajack.VirtualFileSystem
+ v4.7.2
+ 512
+ true
+
+
+ true
+ full
+ false
+ bin\Debug\
+ DEBUG;TRACE
+ prompt
+ 4
+
+
+ pdbonly
+ true
+ bin\Release\
+ TRACE
+ prompt
+ 4
+
+
+ true
+ bin\x64\Debug\
+ DEBUG;TRACE
+ full
+ x64
+ 7.3
+ prompt
+ MinimumRecommendedRules.ruleset
+
+
+ bin\x64\Release\
+ TRACE
+ true
+ pdbonly
+ x64
+ 7.3
+ prompt
+ MinimumRecommendedRules.ruleset
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {9e69bc98-1512-4977-b683-6e7e5292c0b8}
+ Wabbajack.Common.CSP
+
+
+ {b3f3fb6e-b9eb-4f49-9875-d78578bc7ae5}
+ Wabbajack.Common
+
+
+
+
+ 2.2.6
+
+
+ 1.6.0
+
+
+
+
\ No newline at end of file
diff --git a/Wabbajack.sln b/Wabbajack.sln
index c368a714..745a2991 100644
--- a/Wabbajack.sln
+++ b/Wabbajack.sln
@@ -32,6 +32,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Compression.BSA.Test", "Com
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Common.CSP", "Wabbajack.Common.CSP\Wabbajack.Common.CSP.csproj", "{9E69BC98-1512-4977-B683-6E7E5292C0B8}"
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VirtualFileSystem", "Wabbajack.VirtualFileSystem\Wabbajack.VirtualFileSystem.csproj", "{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VirtualFileSystem.Test", "Wabbajack.VirtualFileSystem.Test\Wabbajack.VirtualFileSystem.Test.csproj", "{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug (no commandargs)|Any CPU = Debug (no commandargs)|Any CPU
@@ -225,6 +229,42 @@ Global
{9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x64.Build.0 = Release|Any CPU
{9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x86.ActiveCfg = Release|Any CPU
{9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x86.Build.0 = Release|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|x86.ActiveCfg = Debug|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|x86.Build.0 = Debug|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|x64.ActiveCfg = Debug|x64
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|x64.Build.0 = Debug|x64
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|x86.Build.0 = Debug|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|Any CPU.Build.0 = Release|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x64.ActiveCfg = Release|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x64.Build.0 = Release|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x86.ActiveCfg = Release|Any CPU
+ {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x86.Build.0 = Release|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|x86.ActiveCfg = Debug|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|x86.Build.0 = Debug|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|x64.ActiveCfg = Debug|x64
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|x64.Build.0 = Debug|x64
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|x86.Build.0 = Debug|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|Any CPU.Build.0 = Release|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x64.ActiveCfg = Release|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x64.Build.0 = Release|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x86.ActiveCfg = Release|Any CPU
+ {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE