Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #176 from wabbajack-tools/vfs-integration-redux

Browse Source 
Vfs integration
This commit is contained in:
Timothy Baldridge 2019-11-15 12:02:39 -07:00 committed by GitHub
commit 815295b7d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 197 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
VirtualFileSystem.Test/VirtualFileSystem.Test.csproj
View File

@ -92,10 +92,6 @@
<None Include="App.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VirtualFileSystem\VirtualFileSystem.csproj">
<Project>{5128b489-bc28-4f66-9f0b-b4565af36cbc}</Project>
<Name>VirtualFileSystem</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
<Project>{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}</Project>
<Name>Wabbajack.Common</Name>

5
Wabbajack.Lib/ACompiler.cs
View File

@ -3,14 +3,15 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using VFS;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.Lib
{
public abstract class ACompiler
{
public Context VFS { get; internal set; } = new Context();
public ModManager ModManager;
public string GamePath;
@ -22,7 +23,7 @@ namespace Wabbajack.Lib
public List<Directive> InstallDirectives = new List<Directive>();
public List<RawSourceFile> AllFiles = new List<RawSourceFile>();
public ModList ModList = new ModList();
public VirtualFileSystem VFS = VirtualFileSystem.VFS;
public List<IndexedArchive> IndexedArchives = new List<IndexedArchive>();
public Dictionary<string, IEnumerable<VirtualFile>> IndexedFiles = new Dictionary<string, IEnumerable<VirtualFile>>();

2
Wabbajack.Lib/CerasConfig.cs
View File

@ -1,8 +1,8 @@
using Ceras;
using Compression.BSA;
using VFS;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.Lib
{

4
Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs
View File

@ -57,7 +57,7 @@ namespace Wabbajack.Lib.CompilationSteps
if (_include_directly.Any(path => source.Path.StartsWith(path)))
defaultInclude = true;
var source_files = source.File.FileInArchive;
var source_files = source.File.Children;
var stack = defaultInclude ? _microstackWithInclude : _microstack;
@ -65,7 +65,7 @@ namespace Wabbajack.Lib.CompilationSteps
var matches = source_files.PMap(e => _mo2Compiler.RunStack(stack, new RawSourceFile(e)
{
Path = Path.Combine(Consts.BSACreationDir, id, e.Paths.Last())
Path = Path.Combine(Consts.BSACreationDir, id, e.Name)
}));

7
Wabbajack.Lib/CompilationSteps/DirectMatch.cs
View File

@ -15,11 +15,10 @@ namespace Wabbajack.Lib.CompilationSteps
if (!_compiler.IndexedFiles.TryGetValue(source.Hash, out var found)) return null;
var result = source.EvolveTo<FromArchive>();
var match = found.Where(f =>
Path.GetFileName(f.Paths[f.Paths.Length - 1]) == Path.GetFileName(source.Path))
.OrderBy(f => f.Paths.Length)
var match = found.Where(f => Path.GetFileName(f.Name) == Path.GetFileName(source.Path))
.OrderBy(f => f.NestingFactor)
.FirstOrDefault()
?? found.OrderBy(f => f.Paths.Length).FirstOrDefault();
?? found.OrderBy(f => f.NestingFactor).FirstOrDefault();
result.ArchiveHashPath = match.MakeRelativePaths();

8
Wabbajack.Lib/CompilationSteps/IncludePatches.cs
View File

@ -2,8 +2,8 @@
using System.Linq;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
using VFS;
using Wabbajack.Common;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.Lib.CompilationSteps
{
@ -15,16 +15,16 @@ namespace Wabbajack.Lib.CompilationSteps
{
_indexed = _compiler.IndexedFiles.Values
.SelectMany(f => f)
.GroupBy(f => Path.GetFileName(Enumerable.Last(f.Paths)).ToLower())
.GroupBy(f => Path.GetFileName(f.Name).ToLower())
.ToDictionary(f => f.Key);
}
public override Directive Run(RawSourceFile source)
{
if (!_indexed.TryGetValue(Path.GetFileName(Enumerable.Last(source.File.Paths).ToLower()), out var value))
if (!_indexed.TryGetValue(Path.GetFileName(source.File.Name.ToLower()), out var value))
return null;
var found = value.OrderByDescending(f => (f.TopLevelArchive ?? f).LastModified).First();
var found = value.OrderByDescending(f => (f.FilesInFullPath.First() ?? f).LastModified).First();
var e = source.EvolveTo<PatchedFromArchive>();
e.ArchiveHashPath = found.MakeRelativePaths();

2
Wabbajack.Lib/CompilationSteps/PatchStockESMs.cs
View File

@ -25,7 +25,7 @@ namespace Wabbajack.Lib.CompilationSteps
Utils.Log(
$"A ESM named {filename} was found in a mod that shares a name with a core game ESMs, it is assumed this is a cleaned ESM and it will be binary patched.");
var result = source.EvolveTo<CleanedESM>();
result.SourceESMHash = _compiler.VFS.Lookup(gameFile).Hash;
result.SourceESMHash = _compiler.VFS.Index.ByRootPath[gameFile].Hash;
Utils.Status($"Generating patch of {filename}");
using (var ms = new MemoryStream())

39
Wabbajack.Lib/Compiler.cs
View File

@ -12,13 +12,13 @@ using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using VFS;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Wabbajack.VirtualFileSystem;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
@ -31,8 +31,7 @@ namespace Wabbajack.Lib
private string _mo2DownloadsFolder;
public Dictionary<string, IEnumerable<IndexedFileMatch>> DirectMatchIndex;
public string MO2Folder;
@ -112,7 +111,6 @@ namespace Wabbajack.Lib
public override bool Compile()
{
VirtualFileSystem.Clean();
Info("Looking for other profiles");
var other_profiles_path = Path.Combine(MO2ProfileDir, "otherprofiles.txt");
SelectedProfiles = new HashSet<string>();
@ -122,12 +120,12 @@ namespace Wabbajack.Lib
Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p)));
Info($"Indexing {MO2Folder}");
VFS.AddRoot(MO2Folder);
VFS.AddRoot(MO2Folder).Wait();
Info($"Indexing {GamePath}");
VFS.AddRoot(GamePath);
VFS.AddRoot(GamePath).Wait();
Info($"Indexing {MO2DownloadsFolder}");
VFS.AddRoot(MO2DownloadsFolder);
VFS.AddRoot(MO2DownloadsFolder).Wait();
Info("Cleaning output folder");
if (Directory.Exists(ModListOutputFolder))
@ -137,11 +135,11 @@ namespace Wabbajack.Lib
var mo2_files = Directory.EnumerateFiles(MO2Folder, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Lookup(p)) { Path = p.RelativeTo(MO2Folder) });
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p]) { Path = p.RelativeTo(MO2Folder) });
var game_files = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Lookup(p))
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{ Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) });
var loot_path = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
@ -152,11 +150,11 @@ namespace Wabbajack.Lib
if (Directory.Exists(loot_path))
{
Info($"Indexing {loot_path}");
VFS.AddRoot(loot_path);
VFS.AddRoot(loot_path).Wait();
loot_files = Directory.EnumerateFiles(loot_path, "userlist.yaml", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Lookup(p))
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{ Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(loot_path)) });
}
@ -168,7 +166,7 @@ namespace Wabbajack.Lib
.Where(f => File.Exists(f + ".meta"))
.Select(f => new IndexedArchive
{
File = VFS.Lookup(f),
File = VFS.Index.ByRootPath[f],
Name = Path.GetFileName(f),
IniData = (f + ".meta").LoadIniFile(),
Meta = File.ReadAllText(f + ".meta")
@ -176,16 +174,8 @@ namespace Wabbajack.Lib
.ToList();
Info("Indexing Files");
var grouped = VFS.GroupedByArchive();
IndexedFiles = IndexedArchives.Select(f =>
{
if (grouped.TryGetValue(f.File, out var result))
return result;
return new List<VirtualFile>();
})
.SelectMany(fs => fs)
.Concat(IndexedArchives.Select(f => f.File))
.OrderByDescending(f => f.TopLevelArchive.LastModified)
IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
.OrderBy(f => f.NestingFactor)
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, f => f.AsEnumerable());
@ -429,10 +419,9 @@ namespace Wabbajack.Lib
private void BuildArchivePatches(string archive_sha, IEnumerable<PatchedFromArchive> group,
Dictionary<string, string> absolute_paths)
{
var archive = VFS.HashIndex[archive_sha];
using (var files = VFS.StageWith(group.Select(g => VFS.FileForArchiveHashPath(g.ArchiveHashPath))))
using (var files = VFS.StageWith(group.Select(g => VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath))).Result)
{
var by_path = files.GroupBy(f => string.Join("|", f.Paths.Skip(1)))
var by_path = files.GroupBy(f => string.Join("|", f.FilesInFullPath.Skip(1).Select(i => i.Name)))
.ToDictionary(f => f.Key, f => f.First());
// Now Create the patches
group.PMap(entry =>

2
Wabbajack.Lib/Data.cs
View File

@ -2,9 +2,9 @@
using System.Collections.Generic;
using Ceras;
using Compression.BSA;
using VFS;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.Lib
{

32
Wabbajack.Lib/Installer.cs
View File

@ -4,12 +4,13 @@ using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows;
using VFS;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Wabbajack.VirtualFileSystem;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
@ -28,7 +29,7 @@ namespace Wabbajack.Lib
ModList = mod_list;
}
public VirtualFileSystem VFS => VirtualFileSystem.VFS;
public Context VFS { get; } = new Context();
public string Outputfolder { get; }
@ -116,7 +117,6 @@ namespace Wabbajack.Lib
ValidateGameESMs();
ValidateModlist.RunValidation(ModList);
VirtualFileSystem.Clean();
Directory.CreateDirectory(Outputfolder);
Directory.CreateDirectory(DownloadFolder);
@ -150,7 +150,7 @@ namespace Wabbajack.Lib
Error("Cannot continue, was unable to download one or more archives");
}
PrimeVFS();
PrimeVFS().Wait();
BuildFolderStructure();
InstallArchives();
@ -233,27 +233,21 @@ namespace Wabbajack.Lib
/// We don't want to make the installer index all the archives, that's just a waste of time, so instead
/// we'll pass just enough information to VFS to let it know about the files we have.
/// </summary>
private void PrimeVFS()
private async Task PrimeVFS()
{
HashedArchives.Do(a => VFS.AddKnown(new VirtualFile
VFS.AddKnown(HashedArchives.Select(a => new KnownFile
{
Paths = new[] { a.Value },
Hash = a.Key
}));
VFS.RefreshIndexes();
VFS.AddKnown(
ModList.Directives
.OfType<FromArchive>()
.Do(f =>
{
var updated_path = new string[f.ArchiveHashPath.Length];
f.ArchiveHashPath.CopyTo(updated_path, 0);
updated_path[0] = VFS.HashIndex[updated_path[0]].Where(e => e.IsConcrete).First().FullPath;
VFS.AddKnown(new VirtualFile { Paths = updated_path });
});
.Select(f => new KnownFile { Paths = f.ArchiveHashPath}));
VFS.BackfillMissing();
await VFS.BackfillMissing();
}
private void BuildBSAs()
@ -287,7 +281,7 @@ namespace Wabbajack.Lib
if (Directory.Exists(bsa_dir))
{
Info($"Removing temp folder {Consts.BSACreationDir}");
VirtualFileSystem.DeleteDirectory(bsa_dir);
Directory.Delete(bsa_dir, true, true);
}
}
@ -387,12 +381,12 @@ namespace Wabbajack.Lib
var vfiles = grouping.Select(g =>
{
var file = VFS.FileForArchiveHashPath(g.ArchiveHashPath);
var file = VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath);
g.FromFile = file;
return g;
}).ToList();
var on_finish = VFS.Stage(vfiles.Select(f => f.FromFile).Distinct());
var on_finish = VFS.Stage(vfiles.Select(f => f.FromFile).Distinct()).Result;
Status($"Copying files for {archive.Name}");

30
Wabbajack.Lib/VortexCompiler.cs
View File

@ -6,12 +6,12 @@ using System.Linq;
using System.Security.Cryptography;
using System.Text;
using Microsoft.WindowsAPICodePack.Shell;
using VFS;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
using Wabbajack.VirtualFileSystem;
using File = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.Lib
@ -89,20 +89,19 @@ namespace Wabbajack.Lib
public override bool Compile()
{
VirtualFileSystem.Clean();
Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}.");
Info("Starting pre-compilation steps");
CreateMetaFiles();
Info($"Indexing {StagingFolder}");
VFS.AddRoot(StagingFolder);
VFS.AddRoot(StagingFolder).Wait();
Info($"Indexing {GamePath}");
VFS.AddRoot(GamePath);
VFS.AddRoot(GamePath).Wait();
Info($"Indexing {DownloadsFolder}");
VFS.AddRoot(DownloadsFolder);
VFS.AddRoot(DownloadsFolder).Wait();
AddExternalFolder();
@ -112,37 +111,34 @@ namespace Wabbajack.Lib
IEnumerable<RawSourceFile> vortexStagingFiles = Directory.EnumerateFiles(StagingFolder, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists() && p != "__vortex_staging_folder")
.Select(p => new RawSourceFile(VFS.Lookup(p))
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{Path = p.RelativeTo(StagingFolder)});
IEnumerable<RawSourceFile> vortexDownloads = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Lookup(p))
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{Path = p.RelativeTo(DownloadsFolder)});
IEnumerable<RawSourceFile> gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Lookup(p))
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{ Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) });
Info("Indexing Archives");
IndexedArchives = Directory.EnumerateFiles(DownloadsFolder)
.Where(f => File.Exists(f+".meta"))
.Where(f => File.Exists(f + ".meta"))
.Select(f => new IndexedArchive
{
File = VFS.Lookup(f),
File = VFS.Index.ByRootPath[f],
Name = Path.GetFileName(f),
IniData = (f+".meta").LoadIniFile(),
Meta = File.ReadAllText(f+".meta")
IniData = (f + ".meta").LoadIniFile(),
Meta = File.ReadAllText(f + ".meta")
})
.ToList();
Info("Indexing Files");
IDictionary<VirtualFile, IEnumerable<VirtualFile>> grouped = VFS.GroupedByArchive();
IndexedFiles = IndexedArchives.Select(f => grouped.TryGetValue(f.File, out var result) ? result : new List<VirtualFile>())
.SelectMany(fs => fs)
.Concat(IndexedArchives.Select(f => f.File))
.OrderByDescending(f => f.TopLevelArchive.LastModified)
IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
.OrderBy(f => f.NestingFactor)
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, f => f.AsEnumerable());

23
Wabbajack.Lib/VortexInstaller.cs
View File

@ -3,9 +3,9 @@ using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using VFS;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
@ -24,7 +24,7 @@ namespace Wabbajack.Lib
public string StagingFolder { get; set; }
public string DownloadFolder { get; set; }
public VirtualFileSystem VFS => VirtualFileSystem.VFS;
public Context VFS { get; } = new Context();
public bool IgnoreMissingFiles { get; internal set; }
@ -89,8 +89,6 @@ namespace Wabbajack.Lib
{
Directory.CreateDirectory(DownloadFolder);
VirtualFileSystem.Clean();
HashArchives();
DownloadArchives();
HashArchives();
@ -153,12 +151,12 @@ namespace Wabbajack.Lib
var vFiles = grouping.Select(g =>
{
var file = VFS.FileForArchiveHashPath(g.ArchiveHashPath);
var file = VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath);
g.FromFile = file;
return g;
}).ToList();
var onFinish = VFS.Stage(vFiles.Select(f => f.FromFile).Distinct());
var onFinish = VFS.Stage(vFiles.Select(f => f.FromFile).Distinct()).Result;
Status($"Copying files for {archive.Name}");
@ -203,24 +201,27 @@ namespace Wabbajack.Lib
});
}
/// <summary>
/// We don't want to make the installer index all the archives, that's just a waste of time, so instead
/// we'll pass just enough information to VFS to let it know about the files we have.
/// </summary>
private void PrimeVFS()
{
HashedArchives.Do(a => VFS.AddKnown(new VirtualFile
VFS.AddKnown(HashedArchives.Select(a => new KnownFile
{
Paths = new[] { a.Value },
Hash = a.Key
}));
VFS.RefreshIndexes();
ModList.Directives
.OfType<FromArchive>()
.Do(f =>
.Select(f =>
{
var updated_path = new string[f.ArchiveHashPath.Length];
f.ArchiveHashPath.CopyTo(updated_path, 0);
updated_path[0] = VFS.HashIndex[updated_path[0]].Where(e => e.IsConcrete).First().FullPath;
VFS.AddKnown(new VirtualFile { Paths = updated_path });
updated_path[0] = VFS.Index.ByHash[updated_path[0]].First(e => e.IsNative).FullPath;
return new KnownFile { Paths = updated_path };
});
VFS.BackfillMissing();

8
Wabbajack.Lib/Wabbajack.Lib.csproj
View File

@ -151,14 +151,14 @@
<Project>{ff5d892f-8ff4-44fc-8f7f-cd58f307ad1b}</Project>
<Name>Compression.BSA</Name>
</ProjectReference>
<ProjectReference Include="..\VirtualFileSystem\VirtualFileSystem.csproj">
<Project>{5128b489-bc28-4f66-9f0b-b4565af36cbc}</Project>
<Name>VirtualFileSystem</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
<Project>{b3f3fb6e-b9eb-4f49-9875-d78578bc7ae5}</Project>
<Name>Wabbajack.Common</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.VirtualFileSystem\Wabbajack.VirtualFileSystem.csproj">
<Project>{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}</Project>
<Name>Wabbajack.VirtualFileSystem</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="css-min.css" />

2
Wabbajack.Lib/zEditIntegration.cs
View File

@ -93,7 +93,7 @@ namespace Wabbajack.Lib
return new SourcePatch
{
RelativePath = abs_path.RelativeTo(_mo2Compiler.MO2Folder),
Hash = _compiler.VFS[abs_path].Hash
Hash = _compiler.VFS.Index.ByFullPath[abs_path].Hash
};
}).ToList();

3
Wabbajack.Test/ACompilerTest.cs
View File

@ -4,7 +4,6 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using VFS;
using Wabbajack.Common;
using Wabbajack.Lib;
@ -36,7 +35,6 @@ namespace Wabbajack.Test
protected Compiler ConfigureAndRunCompiler(string profile)
{
var compiler = MakeCompiler();
compiler.VFS.Reset();
compiler.MO2Profile = profile;
compiler.ShowReportWhenFinished = false;
Assert.IsTrue(compiler.Compile());
@ -45,7 +43,6 @@ namespace Wabbajack.Test
protected Compiler MakeCompiler()
{
VirtualFileSystem.Reconfigure(utils.TestFolder);
var compiler = new Compiler(utils.MO2Folder);
return compiler;
}

3
Wabbajack.Test/AVortexCompilerTest.cs
View File

@ -1,7 +1,6 @@
using System;
using System.IO;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using VFS;
using Wabbajack.Common;
using Wabbajack.Lib;
@ -35,7 +34,6 @@ namespace Wabbajack.Test
protected VortexCompiler ConfigureAndRunCompiler()
{
var vortexCompiler = MakeCompiler();
vortexCompiler.VFS.Reset();
vortexCompiler.DownloadsFolder = utils.DownloadsFolder;
vortexCompiler.StagingFolder = utils.InstallFolder;
Directory.CreateDirectory(utils.InstallFolder);
@ -45,7 +43,6 @@ namespace Wabbajack.Test
protected VortexCompiler MakeCompiler()
{
VirtualFileSystem.Reconfigure(utils.TestFolder);
var vortexCompiler = new VortexCompiler(utils.GameName, utils.GameFolder);
return vortexCompiler;
}

5
Wabbajack.Test/EndToEndTests.cs
View File

@ -5,7 +5,6 @@ using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using VFS;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
@ -66,10 +65,8 @@ namespace Wabbajack.Test
if (Directory.Exists(loot_folder))
Directory.Delete(loot_folder, true);
VirtualFileSystem.Reconfigure(utils.TestFolder);
var compiler = new Compiler(utils.InstallFolder);
compiler.MO2DownloadsFolder = Path.Combine(utils.DownloadsFolder);
compiler.VFS.Reset();
compiler.MO2Profile = profile;
compiler.ShowReportWhenFinished = false;
Assert.IsTrue(compiler.Compile());
@ -153,9 +150,7 @@ namespace Wabbajack.Test
private Compiler ConfigureAndRunCompiler(string profile)
{
VirtualFileSystem.Reconfigure(utils.TestFolder);
var compiler = new Compiler(utils.MO2Folder);
compiler.VFS.Reset();
compiler.MO2Profile = profile;
compiler.ShowReportWhenFinished = false;
Assert.IsTrue(compiler.Compile());

1
Wabbajack.Test/SanityTests.cs
View File

@ -5,7 +5,6 @@ using System.Linq;
using System.Runtime;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Microsoft.VisualStudio.TestTools.UnitTesting.Logging;
using VFS;
using Wabbajack.Common;
using Wabbajack.Lib;
using File = Alphaleonis.Win32.Filesystem.File;

4
Wabbajack.Test/Wabbajack.Test.csproj
View File

@ -114,10 +114,6 @@
<None Include="app.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VirtualFileSystem\VirtualFileSystem.csproj">
<Project>{5128b489-bc28-4f66-9f0b-b4565af36cbc}</Project>
<Name>VirtualFileSystem</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.Common.CSP\Wabbajack.Common.CSP.csproj">
<Project>{9e69bc98-1512-4977-b683-6e7e5292c0b8}</Project>
<Name>Wabbajack.Common.CSP</Name>

6
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -129,7 +129,7 @@ namespace Wabbajack.VirtualFileSystem.Test
var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip");
var file = context.Index.ByFullPath[abs_path + "|test.txt"];
var cleanup = context.Stage(new List<VirtualFile> {file});
var cleanup = await context.Stage(new List<VirtualFile> {file});
Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
cleanup();
@ -150,7 +150,7 @@ namespace Wabbajack.VirtualFileSystem.Test
var files = context.Index.ByHash["qX0GZvIaTKM="];
var cleanup = context.Stage(files);
var cleanup = await context.Stage(files);
foreach (var file in files)
Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
@ -183,7 +183,7 @@ namespace Wabbajack.VirtualFileSystem.Test
var new_files = new_context.Index.ByHash["qX0GZvIaTKM="];
var close = new_context.Stage(new_files);
var close = await new_context.Stage(new_files);
foreach (var file in new_files)
Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));

106
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -141,7 +141,7 @@ namespace Wabbajack.VirtualFileSystem
}
}
public Action Stage(IEnumerable<VirtualFile> files)
public async Task<Action> Stage(IEnumerable<VirtualFile> files)
{
var grouped = files.SelectMany(f => f.FilesInFullPath)
.Distinct()
@ -155,7 +155,7 @@ namespace Wabbajack.VirtualFileSystem
foreach (var group in grouped)
{
var tmpPath = Path.Combine(_stagingFolder, Guid.NewGuid().ToString());
FileExtractor.ExtractAll(group.Key.StagedPath, tmpPath).Wait();
await FileExtractor.ExtractAll(group.Key.StagedPath, tmpPath);
paths.Add(tmpPath);
foreach (var file in group)
file.StagedPath = Path.Combine(tmpPath, file.Name);
@ -199,6 +199,87 @@ namespace Wabbajack.VirtualFileSystem
Index = newIndex;
}
}
public async Task<DisposableList<VirtualFile>> StageWith(IEnumerable<VirtualFile> files)
{
return new DisposableList<VirtualFile>(await Stage(files), files);
}
#region KnownFiles
private List<KnownFile> _knownFiles = new List<KnownFile>();
public void AddKnown(IEnumerable<KnownFile> known)
{
_knownFiles.AddRange(known);
}
public async Task BackfillMissing()
{
var newFiles = _knownFiles.Where(f => f.Paths.Length == 1)
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, s => new VirtualFile()
{
Name = s.First().Paths[0],
Hash = s.First().Hash,
Context = this
});
var parentchild = new Dictionary<(VirtualFile, string), VirtualFile>();
void BackFillOne(KnownFile file)
{
var parent = newFiles[file.Paths[0]];
foreach (var path in file.Paths.Skip(1))
{
if (parentchild.TryGetValue((parent, path), out var foundParent))
{
parent = foundParent;
continue;
}
var nf = new VirtualFile();
nf.Name = path;
nf.Parent = parent;
parent.Children = parent.Children.Add(nf);
parentchild.Add((parent, path), nf);
parent = nf;
}
}
_knownFiles.Where(f => f.Paths.Length > 1).Do(BackFillOne);
var newIndex = await Index.Integrate(newFiles.Values.ToList());
lock (this)
Index = newIndex;
_knownFiles = new List<KnownFile>();
}
#endregion
}
public class KnownFile
{
public string[] Paths { get; set; }
public string Hash { get; set; }
}
public class DisposableList<T> : List<T>, IDisposable
{
private Action _unstage;
public DisposableList(Action unstage, IEnumerable<T> files) : base(files)
{
_unstage = unstage;
}
public void Dispose()
{
_unstage();
}
}
public class IndexRoot
@ -208,12 +289,14 @@ namespace Wabbajack.VirtualFileSystem
public IndexRoot(ImmutableList<VirtualFile> aFiles,
ImmutableDictionary<string, VirtualFile> byFullPath,
ImmutableDictionary<string, ImmutableStack<VirtualFile>> byHash,
ImmutableDictionary<string, VirtualFile> byRoot)
ImmutableDictionary<string, VirtualFile> byRoot,
ImmutableDictionary<string, ImmutableStack<VirtualFile>> byName)
{
AllFiles = aFiles;
ByFullPath = byFullPath;
ByHash = byHash;
ByRootPath = byRoot;
ByName = byName;
}
public IndexRoot()
@ -222,11 +305,14 @@ namespace Wabbajack.VirtualFileSystem
ByFullPath = ImmutableDictionary<string, VirtualFile>.Empty;
ByHash = ImmutableDictionary<string, ImmutableStack<VirtualFile>>.Empty;
ByRootPath = ImmutableDictionary<string, VirtualFile>.Empty;
ByName = ImmutableDictionary<string, ImmutableStack<VirtualFile>>.Empty;
}
public ImmutableList<VirtualFile> AllFiles { get; }
public ImmutableDictionary<string, VirtualFile> ByFullPath { get; }
public ImmutableDictionary<string, ImmutableStack<VirtualFile>> ByHash { get; }
public ImmutableDictionary<string, ImmutableStack<VirtualFile>> ByName { get; set; }
public ImmutableDictionary<string, VirtualFile> ByRootPath { get; }
public async Task<IndexRoot> Integrate(List<VirtualFile> files)
@ -239,14 +325,26 @@ namespace Wabbajack.VirtualFileSystem
var byHash = Task.Run(() =>
allFiles.SelectMany(f => f.ThisAndAllChildren)
.Where(f => f.Hash != null)
.ToGroupedImmutableDictionary(f => f.Hash));
var byName = Task.Run(() =>
allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToGroupedImmutableDictionary(f => f.Name));
var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.Name));
return new IndexRoot(allFiles,
await byFullPath,
await byHash,
await byRootPath);
await byRootPath,
await byName);
}
public VirtualFile FileForArchiveHashPath(string[] argArchiveHashPath)
{
var cur = ByHash[argArchiveHashPath[0]].First(f => f.Parent == null);
return argArchiveHashPath.Skip(1).Aggregate(cur, (current, itm) => ByName[itm].First(f => f.Parent == current));
}
}

20
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -230,6 +230,26 @@ namespace Wabbajack.VirtualFileSystem
vf.Children = children.Select(child => CreateFromPortable(context, vf, state, child)).ToImmutableList();
return vf;
}
public string[] MakeRelativePaths()
{
var path = new string[NestingFactor];
path[0] = FilesInFullPath.First().Hash;
var idx = 1;
foreach (var itm in FilesInFullPath.Skip(1))
{
path[idx] = itm.Name;
idx += 1;
}
return path;
}
public Stream OpenRead()
{
return File.OpenRead(StagedPath);
}
}
public class CannotStageNativeFile : Exception

40
Wabbajack.sln
View File

@ -18,10 +18,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
RECIPES.md = RECIPES.md
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VirtualFileSystem", "VirtualFileSystem\VirtualFileSystem.csproj", "{5128B489-BC28-4F66-9F0B-B4565AF36CBC}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VirtualFileSystem.Test", "VirtualFileSystem.Test\VirtualFileSystem.Test.csproj", "{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Test", "Wabbajack.Test\Wabbajack.Test.csproj", "{A47FFF32-782B-4D9F-8704-C98FB32FA8CC}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Lib", "Wabbajack.Lib\Wabbajack.Lib.csproj", "{0A820830-A298-497D-85E0-E9A89EFEF5FE}"
@ -103,42 +99,6 @@ Global
{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|x64.Build.0 = Release|x64
{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|x86.ActiveCfg = Release|x86
{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|x86.Build.0 = Release|x86
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|x64.ActiveCfg = Debug|x64
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|x64.Build.0 = Debug|x64
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|x86.ActiveCfg = Debug|x86
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|x86.Build.0 = Debug|x86
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|x64.ActiveCfg = Debug|x64
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|x64.Build.0 = Debug|x64
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|x86.ActiveCfg = Debug|x86
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|x86.Build.0 = Debug|x86
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|Any CPU.Build.0 = Release|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|x64.ActiveCfg = Release|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|x64.Build.0 = Release|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|x86.ActiveCfg = Release|x86
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|x86.Build.0 = Release|x86
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|x64.ActiveCfg = Debug|x64
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|x64.Build.0 = Debug|x64
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|x86.ActiveCfg = Debug|x86
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|x86.Build.0 = Debug|x86
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|x64.ActiveCfg = Debug|x64
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|x64.Build.0 = Debug|x64
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|x86.ActiveCfg = Debug|x86
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|x86.Build.0 = Debug|x86
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|Any CPU.Build.0 = Release|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|x64.ActiveCfg = Release|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|x64.Build.0 = Release|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|x86.ActiveCfg = Release|x86
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|x86.Build.0 = Release|x86
{A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU
{A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU
{A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Debug (no commandargs)|x64.ActiveCfg = Debug|x64

4
Wabbajack/Wabbajack.csproj
View File

@ -332,10 +332,6 @@
<Project>{ff5d892f-8ff4-44fc-8f7f-cd58f307ad1b}</Project>
<Name>Compression.BSA</Name>
</ProjectReference>
<ProjectReference Include="..\VirtualFileSystem\VirtualFileSystem.csproj">
<Project>{5128b489-bc28-4f66-9f0b-b4565af36cbc}</Project>
<Name>VirtualFileSystem</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
<Project>{b3f3fb6e-b9eb-4f49-9875-d78578bc7ae5}</Project>
<Name>Wabbajack.Common</Name>