mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
Async VFS Implementation (#171)
* New VFS implementation can load/save/analyze files. All in a immutable, lock free (mostly) async parallel manner. * VFS indexing is complete * Can stage files * Can extract VirtualFiles into PortableFiles and create contexts from PortableFiles * Code cleanup
This commit is contained in:
Timothy Baldridge
GitHub
parent
9617ca4982
commit
133fa2febd
@ -89,10 +89,10 @@
|
||||
<Version>2.2.6</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="MSTest.TestAdapter">
|
||||
<Version>1.3.2</Version>
|
||||
<Version>2.0.0</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="MSTest.TestFramework">
|
||||
<Version>1.3.2</Version>
|
||||
<Version>2.0.0</Version>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
<Import Project="$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets" Condition="Exists('$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets')" />
|
||||
|
||||
@ -44,7 +44,10 @@ namespace VFS
|
||||
public static void Reconfigure(string root)
|
||||
{
|
||||
RootFolder = root;
|
||||
_stagedRoot = Path.Combine(RootFolder, "vfs_staged_files");
|
||||
if (RootFolder != null)
|
||||
_stagedRoot = Path.Combine(RootFolder, "vfs_staged_files");
|
||||
else
|
||||
_stagedRoot = "vfs_staged_files";
|
||||
}
|
||||
|
||||
public static void Clean()
|
||||
|
||||
@ -112,7 +112,7 @@
|
||||
<Version>1.2.0</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="System.Collections.Immutable">
|
||||
<Version>1.5.0</Version>
|
||||
<Version>1.6.0</Version>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
|
||||
@ -122,7 +122,7 @@ namespace Wabbajack.Common.CSP
|
||||
Monitor.Exit(this);
|
||||
throw new TooManyHanldersException();
|
||||
}
|
||||
_puts.Unshift((handler, val));
|
||||
_puts.UnboundedUnshift((handler, val));
|
||||
}
|
||||
Monitor.Exit(this);
|
||||
return (AsyncResult.Enqueued, true);
|
||||
@ -191,7 +191,7 @@ namespace Wabbajack.Common.CSP
|
||||
throw new TooManyHanldersException();
|
||||
}
|
||||
|
||||
_takes.Unshift(handler);
|
||||
_takes.UnboundedUnshift(handler);
|
||||
}
|
||||
Monitor.Exit(this);
|
||||
return (AsyncResult.Enqueued, default);
|
||||
|
||||
@ -121,6 +121,59 @@ namespace Wabbajack.Common.CSP
|
||||
|
||||
}
|
||||
|
||||
public static IReadPort<TOut> UnorderedPipelineRx<TIn, TOut>(
|
||||
this IReadPort<TIn> from,
|
||||
Func<IObservable<TIn>, IObservable<TOut>> f,
|
||||
bool propagateClose = true)
|
||||
{
|
||||
var parallelism = Environment.ProcessorCount;
|
||||
var to = Channel.Create<TOut>(parallelism * 2);
|
||||
var pipeline = from.UnorderedPipeline(parallelism, to, f);
|
||||
return to;
|
||||
|
||||
}
|
||||
|
||||
public static IReadPort<TOut> UnorderedPipelineSync<TIn, TOut>(
|
||||
this IReadPort<TIn> from,
|
||||
Func<TIn, TOut> f,
|
||||
bool propagateClose = true)
|
||||
{
|
||||
var parallelism = Environment.ProcessorCount;
|
||||
var to = Channel.Create<TOut>(parallelism * 2);
|
||||
|
||||
async Task Pump()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var (is_open, job) = await from.Take();
|
||||
if (!is_open) break;
|
||||
try
|
||||
{
|
||||
var putIsOpen = await to.Put(f(job));
|
||||
if (!putIsOpen) return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Task.Run(async () =>
|
||||
{
|
||||
await Task.WhenAll(Enumerable.Range(0, parallelism)
|
||||
.Select(idx => Task.Run(Pump)));
|
||||
|
||||
if (propagateClose)
|
||||
{
|
||||
from.Close();
|
||||
to.Close();
|
||||
}
|
||||
});
|
||||
|
||||
return to;
|
||||
}
|
||||
|
||||
public static async Task UnorderedThreadedPipeline<TIn, TOut>(
|
||||
this IReadPort<TIn> from,
|
||||
int parallelism,
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.ComponentModel;
|
||||
using System.IO;
|
||||
|
||||
namespace Wabbajack.Common.CSP
|
||||
{
|
||||
@ -23,7 +25,8 @@ namespace Wabbajack.Common.CSP
|
||||
|
||||
public T Pop()
|
||||
{
|
||||
if (_length == 0) return default;
|
||||
if (_length == 0)
|
||||
throw new InvalidDataException("Pop on empty buffer");
|
||||
var val = _arr[_tail];
|
||||
_arr[_tail] = default;
|
||||
_tail = (_tail + 1) % _size;
|
||||
@ -45,7 +48,7 @@ namespace Wabbajack.Common.CSP
|
||||
|
||||
public void UnboundedUnshift(T x)
|
||||
{
|
||||
if (_length == _size)
|
||||
if (_length + 1 == _size)
|
||||
Resize();
|
||||
Unshift(x);
|
||||
}
|
||||
@ -67,8 +70,8 @@ namespace Wabbajack.Common.CSP
|
||||
}
|
||||
else if (_tail > _head)
|
||||
{
|
||||
Array.Copy(_arr, _tail, new_arr, 0, _length - _tail);
|
||||
Array.Copy(_arr, 0, new_arr, (_length - _tail), _head);
|
||||
Array.Copy(_arr, _tail, new_arr, 0, _size - _tail);
|
||||
Array.Copy(_arr, 0, new_arr, (_size - _tail), _head);
|
||||
_tail = 0;
|
||||
_head = _length;
|
||||
_arr = new_arr;
|
||||
|
||||
@ -125,6 +125,28 @@ namespace Wabbajack.Common
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<string> FileHashAsync(this string file, bool nullOnIOError = false)
|
||||
{
|
||||
try
|
||||
{
|
||||
var hash = new xxHashConfig();
|
||||
hash.HashSizeInBits = 64;
|
||||
hash.Seed = 0x42;
|
||||
using (var fs = File.OpenRead(file))
|
||||
{
|
||||
var config = new xxHashConfig();
|
||||
config.HashSizeInBits = 64;
|
||||
var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(fs);
|
||||
return value.AsBase64String();
|
||||
}
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
if (nullOnIOError) return null;
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
public static void CopyToWithStatus(this Stream istream, long maxSize, Stream ostream, string status)
|
||||
{
|
||||
var buffer = new byte[1024 * 64];
|
||||
|
||||
@ -76,10 +76,10 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="MSTest.TestAdapter">
|
||||
<Version>1.3.2</Version>
|
||||
<Version>2.0.0</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="MSTest.TestFramework">
|
||||
<Version>1.3.2</Version>
|
||||
<Version>2.0.0</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Newtonsoft.Json">
|
||||
<Version>12.0.2</Version>
|
||||
|
||||
@ -84,6 +84,7 @@
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Drawing" />
|
||||
<Reference Include="System.IO.Compression" />
|
||||
<Reference Include="System.IO.Compression.FileSystem" />
|
||||
<Reference Include="System.Transactions" />
|
||||
<Reference Include="System.Windows" />
|
||||
<Reference Include="System.Windows.Forms" />
|
||||
@ -137,10 +138,10 @@
|
||||
<Version>2.2.6</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="MSTest.TestAdapter">
|
||||
<Version>1.3.2</Version>
|
||||
<Version>2.0.0</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="MSTest.TestFramework">
|
||||
<Version>1.3.2</Version>
|
||||
<Version>2.0.0</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Newtonsoft.Json">
|
||||
<Version>12.0.2</Version>
|
||||
|
||||
19
Wabbajack.VirtualFileSystem.Test/Properties/AssemblyInfo.cs
Normal file
19
Wabbajack.VirtualFileSystem.Test/Properties/AssemblyInfo.cs
Normal file
@ -0,0 +1,19 @@
|
||||
using System.Reflection;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
[assembly: AssemblyTitle("Wabbajack.VirtualFileSystem.Test")]
|
||||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("")]
|
||||
[assembly: AssemblyProduct("Wabbajack.VirtualFileSystem.Test")]
|
||||
[assembly: AssemblyCopyright("Copyright © 2019")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
[assembly: ComVisible(false)]
|
||||
|
||||
[assembly: Guid("51ceb604-985a-45b9-af0d-c5ba8cfa1bf0")]
|
||||
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("1.0.0.0")]
|
||||
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||
209
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
Normal file
209
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
Normal file
@ -0,0 +1,209 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Alphaleonis.Win32.Filesystem;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using Wabbajack.Common;
|
||||
|
||||
namespace Wabbajack.VirtualFileSystem.Test
|
||||
{
|
||||
[TestClass]
|
||||
public class VFSTests
|
||||
{
|
||||
private const string VFS_TEST_DIR = "vfs_test_dir";
|
||||
private static readonly string VFS_TEST_DIR_FULL = Path.Combine(Directory.GetCurrentDirectory(), VFS_TEST_DIR);
|
||||
private Context context;
|
||||
|
||||
public TestContext TestContext { get; set; }
|
||||
|
||||
[TestInitialize]
|
||||
public void Setup()
|
||||
{
|
||||
Utils.LogMessages.Subscribe(f => TestContext.WriteLine(f));
|
||||
if (Directory.Exists(VFS_TEST_DIR))
|
||||
Directory.Delete(VFS_TEST_DIR, true);
|
||||
Directory.CreateDirectory(VFS_TEST_DIR);
|
||||
context = new Context();
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task FilesAreIndexed()
|
||||
{
|
||||
AddFile("test.txt", "This is a test");
|
||||
await AddTestRoot();
|
||||
|
||||
var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
|
||||
Assert.IsNotNull(file);
|
||||
|
||||
Assert.AreEqual(file.Size, 14);
|
||||
Assert.AreEqual(file.Hash, "qX0GZvIaTKM=");
|
||||
}
|
||||
|
||||
private async Task AddTestRoot()
|
||||
{
|
||||
await context.AddRoot(VFS_TEST_DIR_FULL);
|
||||
await context.WriteToFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
|
||||
await context.IntegrateFromFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
|
||||
}
|
||||
|
||||
|
||||
[TestMethod]
|
||||
public async Task ArchiveContentsAreIndexed()
|
||||
{
|
||||
AddFile("archive/test.txt", "This is a test");
|
||||
ZipUpFolder("archive", "test.zip");
|
||||
await AddTestRoot();
|
||||
|
||||
var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip");
|
||||
var file = context.Index.ByFullPath[abs_path];
|
||||
Assert.IsNotNull(file);
|
||||
|
||||
Assert.AreEqual(128, file.Size);
|
||||
Assert.AreEqual(abs_path.FileHash(), file.Hash);
|
||||
|
||||
Assert.IsTrue(file.IsArchive);
|
||||
var inner_file = file.Children.First();
|
||||
Assert.AreEqual(14, inner_file.Size);
|
||||
Assert.AreEqual("qX0GZvIaTKM=", inner_file.Hash);
|
||||
Assert.AreSame(file, file.Children.First().Parent);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task DuplicateFileHashes()
|
||||
{
|
||||
AddFile("archive/test.txt", "This is a test");
|
||||
ZipUpFolder("archive", "test.zip");
|
||||
|
||||
AddFile("test.txt", "This is a test");
|
||||
await AddTestRoot();
|
||||
|
||||
|
||||
var files = context.Index.ByHash["qX0GZvIaTKM="];
|
||||
Assert.AreEqual(files.Count(), 2);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task DeletedFilesAreRemoved()
|
||||
{
|
||||
AddFile("test.txt", "This is a test");
|
||||
await AddTestRoot();
|
||||
|
||||
var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
|
||||
Assert.IsNotNull(file);
|
||||
|
||||
Assert.AreEqual(file.Size, 14);
|
||||
Assert.AreEqual(file.Hash, "qX0GZvIaTKM=");
|
||||
|
||||
File.Delete(Path.Combine(VFS_TEST_DIR_FULL, "test.txt"));
|
||||
|
||||
await AddTestRoot();
|
||||
|
||||
CollectionAssert.DoesNotContain(context.Index.ByFullPath, Path.Combine(VFS_TEST_DIR_FULL, "test.txt"));
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task UnmodifiedFilesAreNotReIndexed()
|
||||
{
|
||||
AddFile("test.txt", "This is a test");
|
||||
await AddTestRoot();
|
||||
|
||||
var old_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
|
||||
var old_time = old_file.LastAnalyzed;
|
||||
|
||||
await AddTestRoot();
|
||||
|
||||
var new_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
|
||||
|
||||
Assert.AreEqual(old_time, new_file.LastAnalyzed);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task CanStageSimpleArchives()
|
||||
{
|
||||
AddFile("archive/test.txt", "This is a test");
|
||||
ZipUpFolder("archive", "test.zip");
|
||||
await AddTestRoot();
|
||||
|
||||
var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip");
|
||||
var file = context.Index.ByFullPath[abs_path + "|test.txt"];
|
||||
|
||||
var cleanup = context.Stage(new List<VirtualFile> {file});
|
||||
Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
|
||||
|
||||
cleanup();
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task CanStageNestedArchives()
|
||||
{
|
||||
AddFile("archive/test.txt", "This is a test");
|
||||
ZipUpFolder("archive", "test.zip");
|
||||
|
||||
Directory.CreateDirectory(Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir"));
|
||||
File.Move(Path.Combine(VFS_TEST_DIR_FULL, "test.zip"),
|
||||
Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip"));
|
||||
ZipUpFolder("archive", "test.zip");
|
||||
|
||||
await AddTestRoot();
|
||||
|
||||
var files = context.Index.ByHash["qX0GZvIaTKM="];
|
||||
|
||||
var cleanup = context.Stage(files);
|
||||
|
||||
foreach (var file in files)
|
||||
Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
|
||||
|
||||
cleanup();
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task CanRequestPortableFileTrees()
|
||||
{
|
||||
AddFile("archive/test.txt", "This is a test");
|
||||
ZipUpFolder("archive", "test.zip");
|
||||
|
||||
Directory.CreateDirectory(Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir"));
|
||||
File.Move(Path.Combine(VFS_TEST_DIR_FULL, "test.zip"),
|
||||
Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip"));
|
||||
ZipUpFolder("archive", "test.zip");
|
||||
|
||||
await AddTestRoot();
|
||||
|
||||
var files = context.Index.ByHash["qX0GZvIaTKM="];
|
||||
var archive = context.Index.ByRootPath[Path.Combine(VFS_TEST_DIR_FULL, "test.zip")];
|
||||
|
||||
var state = context.GetPortableState(files);
|
||||
|
||||
var new_context = new Context();
|
||||
|
||||
await new_context.IntegrateFromPortable(state,
|
||||
new Dictionary<string, string> {{archive.Hash, archive.FullPath}});
|
||||
|
||||
var new_files = new_context.Index.ByHash["qX0GZvIaTKM="];
|
||||
|
||||
var close = new_context.Stage(new_files);
|
||||
|
||||
foreach (var file in new_files)
|
||||
Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
|
||||
|
||||
close();
|
||||
}
|
||||
|
||||
private static void AddFile(string filename, string thisIsATest)
|
||||
{
|
||||
var fullpath = Path.Combine(VFS_TEST_DIR, filename);
|
||||
if (!Directory.Exists(Path.GetDirectoryName(fullpath)))
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(fullpath));
|
||||
File.WriteAllText(fullpath, thisIsATest);
|
||||
}
|
||||
|
||||
private static void ZipUpFolder(string folder, string output)
|
||||
{
|
||||
var path = Path.Combine(VFS_TEST_DIR, folder);
|
||||
ZipFile.CreateFromDirectory(path, Path.Combine(VFS_TEST_DIR, output));
|
||||
Directory.Delete(path, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,99 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="15.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProjectGuid>{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}</ProjectGuid>
|
||||
<OutputType>Library</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>Wabbajack.VirtualFileSystem.Test</RootNamespace>
|
||||
<AssemblyName>Wabbajack.VirtualFileSystem.Test</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<ProjectTypeGuids>{3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
|
||||
<VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">15.0</VisualStudioVersion>
|
||||
<VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
|
||||
<ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages</ReferencePath>
|
||||
<IsCodedUITest>False</IsCodedUITest>
|
||||
<TestProjectType>UnitTest</TestProjectType>
|
||||
<NuGetPackageImportStamp>
|
||||
</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<OutputPath>bin\x64\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<DebugType>full</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<LangVersion>7.3</LangVersion>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
|
||||
<OutputPath>bin\x64\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<Optimize>true</Optimize>
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<LangVersion>7.3</LangVersion>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.IO.Compression.FileSystem" />
|
||||
<Reference Include="System.Transactions" />
|
||||
<Reference Include="System.Windows" />
|
||||
<Reference Include="System.Windows.Forms" />
|
||||
<Reference Include="WindowsBase" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="VirtualFileSystemTests.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
|
||||
<Project>{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}</Project>
|
||||
<Name>Wabbajack.Common</Name>
|
||||
</ProjectReference>
|
||||
<ProjectReference Include="..\Wabbajack.VirtualFileSystem\Wabbajack.VirtualFileSystem.csproj">
|
||||
<Project>{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}</Project>
|
||||
<Name>Wabbajack.VirtualFileSystem</Name>
|
||||
</ProjectReference>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AlphaFS">
|
||||
<Version>2.2.6</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="MSTest.TestAdapter">
|
||||
<Version>2.0.0</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="MSTest.TestFramework">
|
||||
<Version>2.0.0</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="System.Reactive">
|
||||
<Version>4.2.0</Version>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
<Import Project="$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets" Condition="Exists('$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets')" />
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
</Project>
|
||||
267
Wabbajack.VirtualFileSystem/Context.cs
Normal file
267
Wabbajack.VirtualFileSystem/Context.cs
Normal file
@ -0,0 +1,267 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Reactive.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Alphaleonis.Win32.Filesystem;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Common.CSP;
|
||||
using Directory = Alphaleonis.Win32.Filesystem.Directory;
|
||||
using File = System.IO.File;
|
||||
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
|
||||
using Path = Alphaleonis.Win32.Filesystem.Path;
|
||||
|
||||
namespace Wabbajack.VirtualFileSystem
|
||||
{
|
||||
public class Context
|
||||
{
|
||||
public const ulong FileVersion = 0x02;
|
||||
public const string Magic = "WABBAJACK VFS FILE";
|
||||
|
||||
private readonly string _stagingFolder = "vfs_staging";
|
||||
public IndexRoot Index { get; private set; } = IndexRoot.Empty;
|
||||
|
||||
public TemporaryDirectory GetTemporaryFolder()
|
||||
{
|
||||
return new TemporaryDirectory(Path.Combine(_stagingFolder, Guid.NewGuid().ToString()));
|
||||
}
|
||||
|
||||
public async Task<IndexRoot> AddRoot(string root)
|
||||
{
|
||||
if (!Path.IsPathRooted(root))
|
||||
throw new InvalidDataException($"Path is not absolute: {root}");
|
||||
|
||||
var filtered = await Index.AllFiles
|
||||
.ToChannel()
|
||||
.UnorderedPipelineRx(o => o.Where(file => File.Exists(file.Name)))
|
||||
.TakeAll();
|
||||
|
||||
var byPath = filtered.ToImmutableDictionary(f => f.Name);
|
||||
|
||||
var results = Channel.Create<VirtualFile>(1024);
|
||||
var pipeline = Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive)
|
||||
.ToChannel()
|
||||
.UnorderedPipeline(results, async f =>
|
||||
{
|
||||
if (byPath.TryGetValue(f, out var found))
|
||||
{
|
||||
var fi = new FileInfo(f);
|
||||
if (found.LastModified == fi.LastWriteTimeUtc.Ticks && found.Size == fi.Length)
|
||||
return found;
|
||||
}
|
||||
|
||||
return await VirtualFile.Analyze(this, null, f, f);
|
||||
});
|
||||
|
||||
var allFiles = await results.TakeAll();
|
||||
|
||||
// Should already be done but let's make the async tracker happy
|
||||
await pipeline;
|
||||
|
||||
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
|
||||
|
||||
lock (this)
|
||||
{
|
||||
Index = newIndex;
|
||||
}
|
||||
|
||||
return newIndex;
|
||||
}
|
||||
|
||||
public async Task WriteToFile(string filename)
|
||||
{
|
||||
using (var fs = File.OpenWrite(filename))
|
||||
using (var bw = new BinaryWriter(fs, Encoding.UTF8, true))
|
||||
{
|
||||
fs.SetLength(0);
|
||||
|
||||
bw.Write(Encoding.ASCII.GetBytes(Magic));
|
||||
bw.Write(FileVersion);
|
||||
bw.Write((ulong) Index.AllFiles.Count);
|
||||
|
||||
var sizes = await Index.AllFiles
|
||||
.ToChannel()
|
||||
.UnorderedPipelineSync(f =>
|
||||
{
|
||||
var ms = new MemoryStream();
|
||||
f.Write(ms);
|
||||
return ms;
|
||||
})
|
||||
.Select(async ms =>
|
||||
{
|
||||
var size = ms.Position;
|
||||
ms.Position = 0;
|
||||
bw.Write((ulong) size);
|
||||
await ms.CopyToAsync(fs);
|
||||
return ms.Position;
|
||||
})
|
||||
.TakeAll();
|
||||
Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}");
|
||||
}
|
||||
}
|
||||
|
||||
public async Task IntegrateFromFile(string filename)
|
||||
{
|
||||
using (var fs = File.OpenRead(filename))
|
||||
using (var br = new BinaryReader(fs, Encoding.UTF8, true))
|
||||
{
|
||||
var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length));
|
||||
var fileVersion = br.ReadUInt64();
|
||||
if (fileVersion != FileVersion || magic != magic)
|
||||
throw new InvalidDataException("Bad Data Format");
|
||||
|
||||
var numFiles = br.ReadUInt64();
|
||||
|
||||
var input = Channel.Create<byte[]>(1024);
|
||||
var pipeline = input.UnorderedPipelineSync(
|
||||
data => VirtualFile.Read(this, data))
|
||||
.TakeAll();
|
||||
|
||||
Utils.Log($"Loading {numFiles} files from {filename}");
|
||||
|
||||
for (ulong idx = 0; idx < numFiles; idx++)
|
||||
{
|
||||
var size = br.ReadUInt64();
|
||||
var bytes = new byte[size];
|
||||
await br.BaseStream.ReadAsync(bytes, 0, (int) size);
|
||||
await input.Put(bytes);
|
||||
}
|
||||
|
||||
input.Close();
|
||||
|
||||
var files = await pipeline;
|
||||
var newIndex = await Index.Integrate(files);
|
||||
lock (this)
|
||||
{
|
||||
Index = newIndex;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Action Stage(IEnumerable<VirtualFile> files)
|
||||
{
|
||||
var grouped = files.SelectMany(f => f.FilesInFullPath)
|
||||
.Distinct()
|
||||
.Where(f => f.Parent != null)
|
||||
.GroupBy(f => f.Parent)
|
||||
.OrderBy(f => f.Key?.NestingFactor ?? 0)
|
||||
.ToList();
|
||||
|
||||
var paths = new List<string>();
|
||||
|
||||
foreach (var group in grouped)
|
||||
{
|
||||
var tmpPath = Path.Combine(_stagingFolder, Guid.NewGuid().ToString());
|
||||
FileExtractor.ExtractAll(group.Key.StagedPath, tmpPath).Wait();
|
||||
paths.Add(tmpPath);
|
||||
foreach (var file in group)
|
||||
file.StagedPath = Path.Combine(tmpPath, file.Name);
|
||||
}
|
||||
|
||||
return () =>
|
||||
{
|
||||
paths.Do(p =>
|
||||
{
|
||||
if (Directory.Exists(p))
|
||||
Directory.Delete(p, true, true);
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
public List<PortableFile> GetPortableState(IEnumerable<VirtualFile> files)
|
||||
{
|
||||
return files.SelectMany(f => f.FilesInFullPath)
|
||||
.Distinct()
|
||||
.Select(f => new PortableFile
|
||||
{
|
||||
Name = f.Parent != null ? f.Name : null,
|
||||
Hash = f.Hash,
|
||||
ParentHash = f.Parent?.Hash,
|
||||
Size = f.Size
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
public async Task IntegrateFromPortable(List<PortableFile> state, Dictionary<string, string> links)
|
||||
{
|
||||
var indexedState = state.GroupBy(f => f.ParentHash)
|
||||
.ToDictionary(f => f.Key ?? "", f => (IEnumerable<PortableFile>) f);
|
||||
var parents = await indexedState[""]
|
||||
.ToChannel()
|
||||
.UnorderedPipelineSync(f => VirtualFile.CreateFromPortable(this, indexedState, links, f))
|
||||
.TakeAll();
|
||||
|
||||
var newIndex = await Index.Integrate(parents);
|
||||
lock (this)
|
||||
{
|
||||
Index = newIndex;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class IndexRoot
|
||||
{
|
||||
public static IndexRoot Empty = new IndexRoot();
|
||||
|
||||
public IndexRoot(ImmutableList<VirtualFile> aFiles,
|
||||
ImmutableDictionary<string, VirtualFile> byFullPath,
|
||||
ImmutableDictionary<string, ImmutableStack<VirtualFile>> byHash,
|
||||
ImmutableDictionary<string, VirtualFile> byRoot)
|
||||
{
|
||||
AllFiles = aFiles;
|
||||
ByFullPath = byFullPath;
|
||||
ByHash = byHash;
|
||||
ByRootPath = byRoot;
|
||||
}
|
||||
|
||||
public IndexRoot()
|
||||
{
|
||||
AllFiles = ImmutableList<VirtualFile>.Empty;
|
||||
ByFullPath = ImmutableDictionary<string, VirtualFile>.Empty;
|
||||
ByHash = ImmutableDictionary<string, ImmutableStack<VirtualFile>>.Empty;
|
||||
ByRootPath = ImmutableDictionary<string, VirtualFile>.Empty;
|
||||
}
|
||||
|
||||
public ImmutableList<VirtualFile> AllFiles { get; }
|
||||
public ImmutableDictionary<string, VirtualFile> ByFullPath { get; }
|
||||
public ImmutableDictionary<string, ImmutableStack<VirtualFile>> ByHash { get; }
|
||||
public ImmutableDictionary<string, VirtualFile> ByRootPath { get; }
|
||||
|
||||
public async Task<IndexRoot> Integrate(List<VirtualFile> files)
|
||||
{
|
||||
var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList();
|
||||
|
||||
var byFullPath = Task.Run(() =>
|
||||
allFiles.SelectMany(f => f.ThisAndAllChildren)
|
||||
.ToImmutableDictionary(f => f.FullPath));
|
||||
|
||||
var byHash = Task.Run(() =>
|
||||
allFiles.SelectMany(f => f.ThisAndAllChildren)
|
||||
.ToGroupedImmutableDictionary(f => f.Hash));
|
||||
|
||||
var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.Name));
|
||||
|
||||
return new IndexRoot(allFiles,
|
||||
await byFullPath,
|
||||
await byHash,
|
||||
await byRootPath);
|
||||
}
|
||||
}
|
||||
|
||||
public class TemporaryDirectory : IDisposable
|
||||
{
|
||||
public TemporaryDirectory(string name)
|
||||
{
|
||||
FullName = name;
|
||||
}
|
||||
|
||||
public string FullName { get; }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Directory.Delete(FullName, true, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
34
Wabbajack.VirtualFileSystem/Extensions.cs
Normal file
34
Wabbajack.VirtualFileSystem/Extensions.cs
Normal file
@ -0,0 +1,34 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace Wabbajack.VirtualFileSystem
|
||||
{
|
||||
public static class Extensions
|
||||
{
|
||||
public static ImmutableDictionary<TK, TI> ToImmutableDictionary<TI, TK>(this IEnumerable<TI> coll,
|
||||
Func<TI, TK> keyFunc)
|
||||
{
|
||||
var builder = ImmutableDictionary<TK, TI>.Empty.ToBuilder();
|
||||
foreach (var itm in coll)
|
||||
builder.Add(keyFunc(itm), itm);
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
public static ImmutableDictionary<TK, ImmutableStack<TI>> ToGroupedImmutableDictionary<TI, TK>(
|
||||
this IEnumerable<TI> coll, Func<TI, TK> keyFunc)
|
||||
{
|
||||
var builder = ImmutableDictionary<TK, ImmutableStack<TI>>.Empty.ToBuilder();
|
||||
foreach (var itm in coll)
|
||||
{
|
||||
var key = keyFunc(itm);
|
||||
if (builder.TryGetValue(key, out var prev))
|
||||
builder[key] = prev.Push(itm);
|
||||
else
|
||||
builder[key] = ImmutableStack<TI>.Empty.Push(itm);
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
}
|
||||
}
|
||||
10
Wabbajack.VirtualFileSystem/PortableFile.cs
Normal file
10
Wabbajack.VirtualFileSystem/PortableFile.cs
Normal file
@ -0,0 +1,10 @@
|
||||
namespace Wabbajack.VirtualFileSystem
|
||||
{
|
||||
public class PortableFile
|
||||
{
|
||||
public string Name { get; set; }
|
||||
public string Hash { get; set; }
|
||||
public string ParentHash { get; set; }
|
||||
public long Size { get; set; }
|
||||
}
|
||||
}
|
||||
35
Wabbajack.VirtualFileSystem/Properties/AssemblyInfo.cs
Normal file
35
Wabbajack.VirtualFileSystem/Properties/AssemblyInfo.cs
Normal file
@ -0,0 +1,35 @@
|
||||
using System.Reflection;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTitle("Wabbajack.VirtualFileSystem")]
|
||||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("")]
|
||||
[assembly: AssemblyProduct("Wabbajack.VirtualFileSystem")]
|
||||
[assembly: AssemblyCopyright("Copyright © 2019")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
// Setting ComVisible to false makes the types in this assembly not visible
|
||||
// to COM components. If you need to access a type in this assembly from
|
||||
// COM, set the ComVisible attribute to true on that type.
|
||||
[assembly: ComVisible(false)]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("5d6a2eaf-6604-4c51-8ae2-a746b4bc5e3e")]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
// Major Version
|
||||
// Minor Version
|
||||
// Build Number
|
||||
// Revision
|
||||
//
|
||||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("1.0.0.0")]
|
||||
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||
251
Wabbajack.VirtualFileSystem/VirtualFile.cs
Normal file
251
Wabbajack.VirtualFileSystem/VirtualFile.cs
Normal file
@ -0,0 +1,251 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Common.CSP;
|
||||
using Directory = Alphaleonis.Win32.Filesystem.Directory;
|
||||
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
|
||||
using Path = Alphaleonis.Win32.Filesystem.Path;
|
||||
|
||||
namespace Wabbajack.VirtualFileSystem
|
||||
{
|
||||
public class VirtualFile
|
||||
{
|
||||
private string _fullPath;
|
||||
|
||||
private string _stagedPath;
|
||||
public string Name { get; internal set; }
|
||||
|
||||
public string FullPath
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_fullPath != null) return _fullPath;
|
||||
var cur = this;
|
||||
var acc = new LinkedList<string>();
|
||||
while (cur != null)
|
||||
{
|
||||
acc.AddFirst(cur.Name);
|
||||
cur = cur.Parent;
|
||||
}
|
||||
|
||||
_fullPath = string.Join("|", acc);
|
||||
|
||||
return _fullPath;
|
||||
}
|
||||
}
|
||||
|
||||
public string Hash { get; internal set; }
|
||||
public long Size { get; internal set; }
|
||||
|
||||
public long LastModified { get; internal set; }
|
||||
|
||||
public long LastAnalyzed { get; internal set; }
|
||||
|
||||
public VirtualFile Parent { get; internal set; }
|
||||
|
||||
public Context Context { get; set; }
|
||||
|
||||
public string StagedPath
|
||||
{
|
||||
get
|
||||
{
|
||||
if (IsNative)
|
||||
return Name;
|
||||
if (_stagedPath == null)
|
||||
throw new UnstagedFileException(FullPath);
|
||||
return _stagedPath;
|
||||
}
|
||||
internal set
|
||||
{
|
||||
if (IsNative)
|
||||
throw new CannotStageNativeFile("Cannot stage a native file");
|
||||
_stagedPath = value;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the nesting factor for this file. Native files will have a nesting of 1, the factor
|
||||
/// goes up for each nesting of a file in an archive.
|
||||
/// </summary>
|
||||
public int NestingFactor
|
||||
{
|
||||
get
|
||||
{
|
||||
var cnt = 0;
|
||||
var cur = this;
|
||||
while (cur != null)
|
||||
{
|
||||
cnt += 1;
|
||||
cur = cur.Parent;
|
||||
}
|
||||
|
||||
return cnt;
|
||||
}
|
||||
}
|
||||
|
||||
public ImmutableList<VirtualFile> Children { get; internal set; } = ImmutableList<VirtualFile>.Empty;
|
||||
|
||||
public bool IsArchive => Children != null && Children.Count > 0;
|
||||
|
||||
public bool IsNative => Parent == null;
|
||||
|
||||
public IEnumerable<VirtualFile> ThisAndAllChildren =>
|
||||
Children.SelectMany(child => child.ThisAndAllChildren).Append(this);
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Returns all the virtual files in the path to this file, starting from the root file.
|
||||
/// </summary>
|
||||
public IEnumerable<VirtualFile> FilesInFullPath
|
||||
{
|
||||
get
|
||||
{
|
||||
var stack = ImmutableStack<VirtualFile>.Empty;
|
||||
var cur = this;
|
||||
while (cur != null)
|
||||
{
|
||||
stack = stack.Push(cur);
|
||||
cur = cur.Parent;
|
||||
}
|
||||
|
||||
return stack;
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, string abs_path,
|
||||
string rel_path)
|
||||
{
|
||||
var hasher = abs_path.FileHashAsync();
|
||||
var fi = new FileInfo(abs_path);
|
||||
var self = new VirtualFile
|
||||
{
|
||||
Context = context,
|
||||
Name = rel_path,
|
||||
Parent = parent,
|
||||
Size = fi.Length,
|
||||
LastModified = fi.LastWriteTimeUtc.Ticks,
|
||||
LastAnalyzed = DateTime.Now.Ticks
|
||||
};
|
||||
|
||||
if (FileExtractor.CanExtract(Path.GetExtension(abs_path)))
|
||||
using (var tempFolder = context.GetTemporaryFolder())
|
||||
{
|
||||
await FileExtractor.ExtractAll(abs_path, tempFolder.FullName);
|
||||
|
||||
var results = Channel.Create<VirtualFile>(1024);
|
||||
var files = Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories)
|
||||
.ToChannel()
|
||||
.UnorderedPipeline(results,
|
||||
async abs_src => await Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName)));
|
||||
self.Children = (await results.TakeAll()).ToImmutableList();
|
||||
}
|
||||
|
||||
self.Hash = await hasher;
|
||||
return self;
|
||||
}
|
||||
|
||||
public void Write(MemoryStream ms)
|
||||
{
|
||||
using (var bw = new BinaryWriter(ms, Encoding.UTF8, true))
|
||||
{
|
||||
Write(bw);
|
||||
}
|
||||
}
|
||||
|
||||
private void Write(BinaryWriter bw)
|
||||
{
|
||||
bw.Write(Name);
|
||||
bw.Write(Hash);
|
||||
bw.Write(Size);
|
||||
bw.Write(LastModified);
|
||||
bw.Write(LastAnalyzed);
|
||||
bw.Write(Children.Count);
|
||||
foreach (var child in Children)
|
||||
child.Write(bw);
|
||||
}
|
||||
|
||||
public static VirtualFile Read(Context context, byte[] data)
|
||||
{
|
||||
using (var ms = new MemoryStream(data))
|
||||
using (var br = new BinaryReader(ms))
|
||||
{
|
||||
return Read(context, null, br);
|
||||
}
|
||||
}
|
||||
|
||||
private static VirtualFile Read(Context context, VirtualFile parent, BinaryReader br)
|
||||
{
|
||||
var vf = new VirtualFile
|
||||
{
|
||||
Context = context,
|
||||
Parent = parent,
|
||||
Name = br.ReadString(),
|
||||
Hash = br.ReadString(),
|
||||
Size = br.ReadInt64(),
|
||||
LastModified = br.ReadInt64(),
|
||||
LastAnalyzed = br.ReadInt64(),
|
||||
Children = ImmutableList<VirtualFile>.Empty
|
||||
};
|
||||
|
||||
var childrenCount = br.ReadInt32();
|
||||
for (var idx = 0; idx < childrenCount; idx += 1) vf.Children = vf.Children.Add(Read(context, vf, br));
|
||||
|
||||
return vf;
|
||||
}
|
||||
|
||||
public static VirtualFile CreateFromPortable(Context context,
|
||||
Dictionary<string, IEnumerable<PortableFile>> state, Dictionary<string, string> links,
|
||||
PortableFile portableFile)
|
||||
{
|
||||
var vf = new VirtualFile
|
||||
{
|
||||
Parent = null,
|
||||
Context = context,
|
||||
Name = links[portableFile.Hash],
|
||||
Hash = portableFile.Hash,
|
||||
Size = portableFile.Size
|
||||
};
|
||||
if (state.TryGetValue(portableFile.Hash, out var children))
|
||||
vf.Children = children.Select(child => CreateFromPortable(context, vf, state, child)).ToImmutableList();
|
||||
return vf;
|
||||
}
|
||||
|
||||
public static VirtualFile CreateFromPortable(Context context, VirtualFile parent,
|
||||
Dictionary<string, IEnumerable<PortableFile>> state, PortableFile portableFile)
|
||||
{
|
||||
var vf = new VirtualFile
|
||||
{
|
||||
Parent = parent,
|
||||
Context = context,
|
||||
Name = portableFile.Name,
|
||||
Hash = portableFile.Hash,
|
||||
Size = portableFile.Size
|
||||
};
|
||||
if (state.TryGetValue(portableFile.Hash, out var children))
|
||||
vf.Children = children.Select(child => CreateFromPortable(context, vf, state, child)).ToImmutableList();
|
||||
return vf;
|
||||
}
|
||||
}
|
||||
|
||||
public class CannotStageNativeFile : Exception
|
||||
{
|
||||
public CannotStageNativeFile(string cannotStageANativeFile) : base(cannotStageANativeFile)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
public class UnstagedFileException : Exception
|
||||
{
|
||||
private readonly string _fullPath;
|
||||
|
||||
public UnstagedFileException(string fullPath) : base($"File {fullPath} is unstaged, cannot get staged name")
|
||||
{
|
||||
_fullPath = fullPath;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,91 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProjectGuid>{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}</ProjectGuid>
|
||||
<OutputType>Library</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>Wabbajack.VirtualFileSystem</RootNamespace>
|
||||
<AssemblyName>Wabbajack.VirtualFileSystem</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<Deterministic>true</Deterministic>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<OutputPath>bin\x64\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<DebugType>full</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<LangVersion>7.3</LangVersion>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
|
||||
<OutputPath>bin\x64\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<Optimize>true</Optimize>
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<LangVersion>7.3</LangVersion>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Numerics" />
|
||||
<Reference Include="System.Transactions" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
<Reference Include="System.Data.DataSetExtensions" />
|
||||
<Reference Include="Microsoft.CSharp" />
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="Context.cs" />
|
||||
<Compile Include="Extensions.cs" />
|
||||
<Compile Include="PortableFile.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
<Compile Include="VirtualFile.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Wabbajack.Common.CSP\Wabbajack.Common.CSP.csproj">
|
||||
<Project>{9e69bc98-1512-4977-b683-6e7e5292c0b8}</Project>
|
||||
<Name>Wabbajack.Common.CSP</Name>
|
||||
</ProjectReference>
|
||||
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
|
||||
<Project>{b3f3fb6e-b9eb-4f49-9875-d78578bc7ae5}</Project>
|
||||
<Name>Wabbajack.Common</Name>
|
||||
</ProjectReference>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AlphaFS">
|
||||
<Version>2.2.6</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="System.Collections.Immutable">
|
||||
<Version>1.6.0</Version>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
</Project>
|
||||
@ -32,6 +32,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Compression.BSA.Test", "Com
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Common.CSP", "Wabbajack.Common.CSP\Wabbajack.Common.CSP.csproj", "{9E69BC98-1512-4977-B683-6E7E5292C0B8}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VirtualFileSystem", "Wabbajack.VirtualFileSystem\Wabbajack.VirtualFileSystem.csproj", "{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VirtualFileSystem.Test", "Wabbajack.VirtualFileSystem.Test\Wabbajack.VirtualFileSystem.Test.csproj", "{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug (no commandargs)|Any CPU = Debug (no commandargs)|Any CPU
|
||||
@ -225,6 +229,42 @@ Global
|
||||
{9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x64.Build.0 = Release|Any CPU
|
||||
{9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x86.Build.0 = Release|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|x86.ActiveCfg = Debug|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|x86.Build.0 = Debug|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|x64.Build.0 = Debug|x64
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x64.Build.0 = Release|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x86.Build.0 = Release|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|x86.ActiveCfg = Debug|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|x86.Build.0 = Debug|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|x64.Build.0 = Debug|x64
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x64.Build.0 = Release|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x86.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
||||
Loading…
Reference in New Issue
Block a user