Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

WIP, massive rework of file extraction

Browse Source
This commit is contained in:
Timothy Baldridge 2020-09-04 15:00:29 -06:00
parent 27f3571951
commit 8a365ac445
24 changed files with 802 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Compression.BSA/BSA/Reader/BSAReader.cs
View File

@ -76,6 +76,7 @@ namespace Compression.BSA
return bsa;
}
public static BSAReader Load(AbsolutePath filename)
{
var bsa = new BSAReader { _fileName = filename };

2
Compression.BSA/BSADispatch.cs
View File

@ -19,7 +19,7 @@ namespace Compression.BSA
_ => throw new InvalidDataException("Filename is not a .bsa or .ba2")
};
}
private static SignatureChecker BSASignatures = new SignatureChecker(Definitions.FileType.BSA, Definitions.FileType.BA2, Definitions.FileType.TES3);
public static async ValueTask<bool> MightBeBSA(AbsolutePath filename)
{

10
Wabbajack.Common/FileSignatures/SignatureChecker.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
@ -21,9 +22,14 @@ namespace Wabbajack.Common.FileSignatures
public async Task<Definitions.FileType?> MatchesAsync(AbsolutePath path)
{
await using var fs = await path.OpenShared();
return await MatchesAsync(fs);
}
public async Task<Definitions.FileType?> MatchesAsync(Stream stream)
{
var buffer = new byte[_maxLength];
fs.Position = 0;
await fs.ReadAsync(buffer);
stream.Position = 0;
await stream.ReadAsync(buffer);
foreach (var (fileType, signature) in _signatures)
{

6
Wabbajack.Lib/AInstaller.cs
View File

@ -113,6 +113,10 @@ namespace Wabbajack.Lib
public async Task InstallArchives()
{
await VFS.CopyTo(Queue, ModList.Directives
.OfType<FromArchive>()
.Select(a => (VFS.Index.FileForArchiveHashPath(a.ArchiveHashPath), a.To.RelativeTo(OutputFolder))));
/*
Info("Installing Archives");
Info("Grouping Install Files");
var grouped = ModList.Directives
@ -125,7 +129,7 @@ namespace Wabbajack.Lib
.ToList();
Info("Installing Archives");
await archives.PMap(Queue, UpdateTracker,a => InstallArchive(Queue, a.Archive, a.AbsolutePath, grouped[a.Archive.Hash]));
await archives.PMap(Queue, UpdateTracker,a => InstallArchive(Queue, a.Archive, a.AbsolutePath, grouped[a.Archive.Hash]));*/
}
private async Task InstallArchive(WorkQueue queue, Archive archive, AbsolutePath absolutePath, IGrouping<Hash, FromArchive> grouping)

63
Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs Normal file
View File

@ -0,0 +1,63 @@
using System;
using System.IO.Compression;
using System.Threading.Tasks;
using Wabbajack.Common;
using Xunit;
namespace Wabbajack.VirtualFileSystem.Test
{
public class FileExtractorTests
{
[Fact]
public async Task CanGatherDataFromZipFiles()
{
await using var temp = await TempFolder.Create();
await using var archive = new TempFile();
for (int i = 0; i < 10; i ++)
{
await WriteRandomData(temp.Dir.Combine($"{i}.bin"), _rng.Next(10, 1024));
}
await ZipUpFolder(temp.Dir, archive.Path, false);
var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path),
_ => true,
async (path, sfn) =>
{
await using var s = await sfn.GetStream();
return await s.xxHashAsync();
});
Assert.Equal(10, results.Count);
foreach (var (path, hash) in results)
{
Assert.Equal(await temp.Dir.Combine(path).FileHashAsync(), hash);
}
}
private static readonly Random _rng = new Random();
private static async Task WriteRandomData(AbsolutePath path, int size)
{
var buff = new byte[size];
_rng.NextBytes(buff);
await path.WriteAllBytesAsync(buff);
}
private static async Task AddFile(AbsolutePath filename, string text)
{
filename.Parent.CreateDirectory();
await filename.WriteAllTextAsync(text);
}
private static async Task ZipUpFolder(AbsolutePath folder, AbsolutePath output, bool deleteSource = true)
{
ZipFile.CreateFromDirectory((string)folder, (string)output);
if (deleteSource)
await folder.DeleteDirectory();
}
}
}

27
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -139,15 +139,15 @@ namespace Wabbajack.VirtualFileSystem.Test
await AddTestRoot();
var res = new FullPath(TEST_ZIP, new[] {(RelativePath)"test.txt"});
var file = context.Index.ByFullPath[res];
var files = new [] {context.Index.ByFullPath[res]};
var cleanup = await context.Stage(new List<VirtualFile> {file});
await using var stream = await file.StagedFile.OpenRead();
Assert.Equal("This is a test", await stream.ReadAllTextAsync());
var queue = new WorkQueue();
await context.Extract(queue, files.ToHashSet(), async (file, factory) =>
{
await using var s = await factory.GetStream();
Assert.Equal("This is a test", await s.ReadAllTextAsync());
});
await cleanup();
}
[Fact]
@ -165,16 +165,13 @@ namespace Wabbajack.VirtualFileSystem.Test
var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")];
var cleanup = await context.Stage(files);
foreach (var file in files)
var queue = new WorkQueue();
await context.Extract(queue, files.ToHashSet(), async (file, factory) =>
{
await using var stream = await file.StagedFile.OpenRead();
Assert.Equal("This is a test", await stream.ReadAllTextAsync());
}
await using var s = await factory.GetStream();
Assert.Equal("This is a test", await s.ReadAllTextAsync());
});
await cleanup();
}
private static async Task AddFile(AbsolutePath filename, string text)

44
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -71,7 +71,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return await VirtualFile.Analyze(this, null, new RootDiskFile(f), f, 0);
return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, 0);
});
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@ -103,7 +103,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return await VirtualFile.Analyze(this, null, new RootDiskFile(f), f, 0);
return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, 0);
});
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@ -196,6 +196,40 @@ namespace Wabbajack.VirtualFileSystem
}
}
/// <summary>
/// Extract the set of files and call the callback for each, handing it a stream factory and the virtual file,
/// top level archives (native archives) will be processed in parallel. Duplicate files will not be
/// </summary>
/// <param name="files"></param>
/// <param name="callback"></param>
/// <returns></returns>
public async Task Extract(WorkQueue queue, HashSet<VirtualFile> files, Func<VirtualFile, IStreamFactory, ValueTask> callback)
{
var top = new VirtualFile();
var filesByParent = files.SelectMany(f => f.FilesInFullPath)
.Distinct()
.GroupBy(f => f.Parent ?? top)
.ToDictionary(f => f.Key);
async Task HandleFile(VirtualFile file, IStreamFactory sfn)
{
if (files.Contains(file)) await callback(file, sfn);
if (filesByParent.TryGetValue(file, out var children))
{
var fileNames = children.ToDictionary(c => c.RelativeName);
await FileExtractor2.GatheringExtract(sfn,
r => fileNames.ContainsKey(r),
async (rel, csf) =>
{
await HandleFile(fileNames[rel], sfn);
return 0;
});
}
}
await filesByParent[top].PMap(queue, async file => await HandleFile(file, new NativeFileStreamFactory(file.AbsoluteName)));
}
public async Task<Func<Task>> Stage(IEnumerable<VirtualFile> files)
{
await _cleanupTask;
@ -227,6 +261,12 @@ namespace Wabbajack.VirtualFileSystem
};
}
public async Task CopyTo(WorkQueue queue, IEnumerable<(VirtualFile src, AbsolutePath dest)> directives)
{
var plans = StagingPlan.StagingPlan.CreatePlan(directives).ToArray();
await StagingPlan.StagingPlan.ExecutePlans(queue, plans);
}
public async Task<AsyncDisposableList<VirtualFile>> StageWith(IEnumerable<VirtualFile> files)
{
return new AsyncDisposableList<VirtualFile>(await Stage(files), files);

71
Wabbajack.VirtualFileSystem/FileExtractor.cs
View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reactive.Linq;
using System.Threading.Tasks;
@ -19,22 +20,21 @@ namespace Wabbajack.VirtualFileSystem
{
public class FileExtractor
{
private static SignatureChecker archiveSigs = new SignatureChecker(Definitions.FileType.TES3,
public static readonly SignatureChecker ArchiveSigs = new SignatureChecker(Definitions.FileType.TES3,
Definitions.FileType.BSA,
Definitions.FileType.BA2,
Definitions.FileType.ZIP,
Definitions.FileType.EXE,
Definitions.FileType.RAR,
Definitions.FileType._7Z);
public static async Task<ExtractedFiles> ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable<RelativePath> OnlyFiles = null, bool throwOnError = true)
{
OnlyFiles ??= new RelativePath[0];
try
{
var sig = await archiveSigs.MatchesAsync(source);
var sig = await ArchiveSigs.MatchesAsync(source);
if (source.Extension == Consts.OMOD)
return await ExtractAllWithOMOD(source);
@ -71,7 +71,7 @@ namespace Wabbajack.VirtualFileSystem
if (isArchive)
{
return await ExtractAllWith7Zip(source, null);
return await ExtractAllWith7Zip(source, (IEnumerable<RelativePath>) null);
}
var dest = await TempFolder.Create();
@ -183,7 +183,7 @@ namespace Wabbajack.VirtualFileSystem
/// <returns></returns>
public static async Task<bool> CanExtract(AbsolutePath v)
{
var found = await archiveSigs.MatchesAsync(v);
var found = await ArchiveSigs.MatchesAsync(v);
switch (found)
{
case null:
@ -220,5 +220,64 @@ namespace Wabbajack.VirtualFileSystem
{
return ext == _exeExtension || Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext);
}
/// <summary>
/// Extract the specific files to the specific locations
/// </summary>
/// <param name="queue"></param>
/// <param name="archive"></param>
/// <param name="indexed"></param>
/// <exception cref="NotImplementedException"></exception>
public static async Task ExtractTo(WorkQueue queue, AbsolutePath source, Dictionary<RelativePath,AbsolutePath> indexed)
{
try
{
var sig = await ArchiveSigs.MatchesAsync(source);
/*if (source.Extension == Consts.OMOD)
return await ExtractAllWithOMOD(source);*/
switch (sig)
{
case Definitions.FileType.BSA:
case Definitions.FileType.TES3:
case Definitions.FileType.BA2:
await ExtractAllWithBSA(queue, source, indexed);
return;
case Definitions.FileType.EXE:
await ExtractAllExe(source, indexed);
return;
case Definitions.FileType._7Z:
case Definitions.FileType.ZIP:
case Definitions.FileType.RAR:
await ExtractAllWith7Zip(source, indexed);
return;
}
throw new Exception("Invalid archive format");
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
Utils.ErrorThrow(ex, $"Error while extracting {source}");
throw new Exception();
}
}
private static async Task ExtractAllWith7Zip(AbsolutePath source, Dictionary<RelativePath,AbsolutePath> onlyFiles)
{
using var archive = await ArchiveFile.Open(source);
await archive.Extract(f => onlyFiles.TryGetValue(f, out var dest) ? dest : default);
}
private static async Task ExtractAllExe(AbsolutePath source, Dictionary<RelativePath,AbsolutePath> indexed)
{
throw new NotImplementedException();
}
private static async Task ExtractAllWithBSA(WorkQueue queue, AbsolutePath source, Dictionary<RelativePath,AbsolutePath> indexed)
{
throw new NotImplementedException();
}
}
}

139
Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs Normal file
View File

@ -0,0 +1,139 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.FileSignatures;
using Wabbajack.VirtualFileSystem.SevenZipExtractor;
namespace Wabbajack.VirtualFileSystem
{
public class GatheringExtractor<T> : IArchiveExtractCallback
{
private ArchiveFile _archive;
private Predicate<RelativePath> _shouldExtract;
private Func<RelativePath, IStreamFactory, ValueTask<T>> _mapFn;
private Dictionary<RelativePath, T> _results;
private Dictionary<uint, RelativePath> _indexes;
private Stream _stream;
private Definitions.FileType _sig;
public GatheringExtractor(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory, ValueTask<T>> mapfn)
{
_shouldExtract = shouldExtract;
_mapFn = mapfn;
_results = new Dictionary<RelativePath, T>();
_stream = stream;
_sig = sig;
}
public async Task<Dictionary<RelativePath, T>> Extract()
{
var source = new TaskCompletionSource<bool>();
var th = new Thread(() =>
{
try
{
_archive = ArchiveFile.Open(_stream, _sig).Result;
_indexes = _archive.Entries
.Where(f => !f.IsFolder)
.Select((entry, idx) => ((RelativePath)entry.FileName, (uint)idx))
.Where(t => _shouldExtract(t.Item1))
.ToDictionary(t => t.Item2, t => t.Item1);
_archive._archive.Extract(null, 0xFFFFFFFF, 0, this);
_archive.Dispose();
source.SetResult(true);
}
catch (Exception ex)
{
source.SetException(ex);
}
}) {Priority = ThreadPriority.BelowNormal, Name = "7Zip Extraction Worker Thread"};
th.Start();
await source.Task;
return _results;
}
public void SetTotal(ulong total)
{
}
public void SetCompleted(ref ulong completeValue)
{
}
public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode)
{
if (_indexes.ContainsKey(index))
{
outStream = new GatheringExtractorStream<T>(this, index);
return 0;
}
outStream = null;
return 0;
}
public void PrepareOperation(AskMode askExtractMode)
{
}
public void SetOperationResult(OperationResult resultEOperationResult)
{
}
private class GatheringExtractorStream<T> : ISequentialOutStream, IOutStream
{
private GatheringExtractor<T> _extractor;
private uint _index;
public GatheringExtractorStream(GatheringExtractor<T> extractor, uint index)
{
_extractor = extractor;
_index = index;
}
public int Write(IntPtr data, uint size, IntPtr processedSize)
{
unsafe
{
var result = _extractor._mapFn(_extractor._indexes[_index], new UnmanagedStreamFactory((byte*)data, size)).AsTask().Result;
_extractor._results[_extractor._indexes[_index]] = result;
if (processedSize != IntPtr.Zero)
{
Marshal.WriteInt32(processedSize, (int) size);
}
return 0;
}
}
public void Seek(long offset, uint seekOrigin, IntPtr newPosition)
{
}
public int SetSize(long newSize)
{
return 0;
}
}
}
}

52
Wabbajack.VirtualFileSystem/FileExtractor2/IStreamFactory.cs Normal file
View File

@ -0,0 +1,52 @@
using System;
using System.IO;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public interface IStreamFactory
{
Task<Stream> GetStream();
DateTime LastModifiedUtc { get; }
}
public class UnmanagedStreamFactory : IStreamFactory
{
private readonly unsafe byte* _data;
private readonly long _size;
public unsafe UnmanagedStreamFactory(byte* data, long size)
{
_data = data;
_size = size;
}
public async Task<Stream> GetStream()
{
unsafe
{
return new UnmanagedMemoryStream(_data, _size);
}
}
public DateTime LastModifiedUtc => DateTime.UtcNow;
}
public class NativeFileStreamFactory : IStreamFactory
{
private AbsolutePath _file;
public NativeFileStreamFactory(AbsolutePath file)
{
_file = file;
}
public async Task<Stream> GetStream()
{
return await _file.OpenRead();
}
public DateTime LastModifiedUtc => _file.LastModifiedUtc;
}
}

6
Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs
View File

@ -12,7 +12,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
public class ArchiveFile : IDisposable
{
private SevenZipHandle _sevenZipHandle;
private IInArchive _archive;
internal IInArchive _archive;
private InStreamWrapper _archiveStream;
private IList<Entry> _entries;
@ -42,11 +42,11 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
return self;
}
public static async Task<ArchiveFile> Open(Stream archiveStream, SevenZipFormat? format = null, string libraryFilePath = null)
public static async Task<ArchiveFile> Open(Stream archiveStream, Definitions.FileType format)
{
var self = new ArchiveFile();
self.InitializeAndValidateLibrary();
self._archive = self._sevenZipHandle.CreateInArchive(Formats.FormatGuidMapping[SevenZipFormat.SevenZip]);
self._archive = self._sevenZipHandle.CreateInArchive(Formats.FileTypeGuidMapping[format]);
self._archiveStream = new InStreamWrapper(archiveStream);
return self;
}

18
Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs
View File

@ -136,14 +136,14 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
//string CryptoGetTextPassword();
}
internal enum AskMode : int
public enum AskMode : int
{
kExtract = 0,
kTest,
kSkip
}
internal enum OperationResult : int
public enum OperationResult : int
{
kOK = 0,
kUnSupportedMethod,
@ -203,11 +203,11 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
[ComImport]
[Guid("23170F69-40C1-278A-0000-000300020000")]
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
internal interface ISequentialOutStream
public interface ISequentialOutStream
{
[PreserveSig]
int Write(
[In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
IntPtr data,
uint size,
IntPtr processedSize); // ref uint processedSize
/*
@ -246,7 +246,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
{
[PreserveSig]
int Write(
[In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
IntPtr data,
uint size,
IntPtr processedSize); // ref uint processedSize
@ -444,16 +444,18 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
return 0;
}
public int Write(byte[] data, uint size, IntPtr processedSize)
public int Write(IntPtr data, uint size, IntPtr processedSize)
{
throw new NotImplementedException();
/*
this.BaseStream.Write(data, 0, (int) size);
if (processedSize != IntPtr.Zero)
{
Marshal.WriteInt32(processedSize, (int) size);
}
*/
return 0;
}
}
}
}

64
Wabbajack.VirtualFileSystem/SevenZipExtractor/TypedExtractor.cs Normal file
View File

@ -0,0 +1,64 @@
using System;
using System.Collections.Generic;
using System.IO;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
{
internal class TypedExtractor<T> : IArchiveExtractCallback
{
private Dictionary<RelativePath, T> _mappings;
private Action<RelativePath, T, Func<Stream>> _callback;
private Dictionary<uint, RelativePath> _indexToFile;
public TypedExtractor(Dictionary<RelativePath, T> mappings, Action<RelativePath, T, Func<Stream>> callback)
{
_mappings = mappings;
_callback = callback;
}
public void Extract(ArchiveFile file)
{
_indexToFile = new Dictionary<uint, RelativePath>();
uint idx = 0;
foreach (var entry in file.Entries)
{
var rel = (RelativePath)entry.FileName;
if (_mappings.ContainsKey(rel)) ;
{
_indexToFile.Add(idx, rel);
}
idx += 1;
}
file._archive.Extract(null, 0xFFFFFFFF, 0, this);
}
public void SetTotal(ulong total)
{
throw new System.NotImplementedException();
}
public void SetCompleted(ref ulong completeValue)
{
throw new System.NotImplementedException();
}
public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode)
{
outStream = null;
throw new System.NotImplementedException();
}
public void PrepareOperation(AskMode askExtractMode)
{
throw new System.NotImplementedException();
}
public void SetOperationResult(OperationResult resultEOperationResult)
{
throw new System.NotImplementedException();
}
}
}

25
Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs Normal file
View File

@ -0,0 +1,25 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public abstract class ASubStage : ISubStage
{
private IEnumerable<IStagingPlan> _plans;
public ASubStage(IEnumerable<IStagingPlan> plans)
{
_plans = plans;
}
public abstract ValueTask DisposeAsync();
public abstract AbsolutePath Destination { get; }
public abstract IPath Source { get; }
public async Task Execute(WorkQueue queue)
{
await StagingPlan.ExecutePlan(queue, async () => await Destination.OpenWrite(), _plans);
}
}
}

23
Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs Normal file
View File

@ -0,0 +1,23 @@
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public class CopyTo : IStagingSrc
{
public CopyTo(IPath src, AbsolutePath destination)
{
Destination = destination;
Source = src;
}
public AbsolutePath Destination { get; }
public async ValueTask DisposeAsync()
{
}
public IPath Source { get; }
}
}

26
Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs Normal file
View File

@ -0,0 +1,26 @@
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public class DuplicateTo : IStagingPlan
{
private readonly IStagingPlan _src;
public DuplicateTo(IStagingPlan src, AbsolutePath destination)
{
_src = src;
Destination = destination;
}
public async ValueTask DisposeAsync()
{
}
public async Task Execute()
{
await _src.Destination.CopyToAsync(Destination);
}
public AbsolutePath Destination { get; }
}
}

156
Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs Normal file
View File

@ -0,0 +1,156 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.FileSignatures;
using Wabbajack.VirtualFileSystem.SevenZipExtractor;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public interface IStagingPlan : IAsyncDisposable
{
AbsolutePath Destination { get; }
}
public static class StagingPlan
{
public static IEnumerable<IStagingPlan> CreatePlan(IEnumerable<(VirtualFile src, AbsolutePath dest)> directives)
{
var top = new VirtualFile();
var original = directives.ToHashSet();
var files = directives.GroupBy(f => f.src).ToDictionary(f => f.Key);
var childrenForParent = files.Keys.GroupBy(f => f.Parent ?? top).ToDictionary(f => f.Key);
var allFilesForParent = files.Select(f => f.Key)
.SelectMany(f => f.FilesInFullPath)
.Distinct()
.GroupBy(f => f.Parent ?? top)
.ToDictionary(f => f.Key, f => f.ToArray());
var baseDirectives = new List<IStagingPlan>();
// Recursive internal function to get the plans for a given parent, we'll then start at the
// null parent (the top) and recurse our way down into all the children files.
IEnumerable<IStagingPlan> GetPlans(VirtualFile parent)
{
foreach (var forParent in allFilesForParent[parent])
{
// Do we need files inside this file?
if (childrenForParent.TryGetValue(forParent, out var children))
{
if (files.TryGetValue(forParent, out var copies))
{
ASubStage subStage;
if (parent == top)
{
subStage = new NativeArchive(forParent.AbsoluteName, GetPlans(forParent));
}
else
{
subStage = new SubStage(forParent.RelativeName, copies.First().dest, GetPlans(forParent));
}
yield return subStage;
foreach (var copy in copies)
{
yield return new DuplicateTo(subStage, copy.dest);
}
}
else
{
if (parent == top)
{
yield return new NativeArchive(forParent.AbsoluteName, GetPlans(forParent));
}
else
{
yield return new TempSubStage(forParent.RelativeName, GetPlans(forParent));
}
}
}
else
{
// If not then we need to copy this file around
var copies = files[forParent];
var firstCopy = new CopyTo(copies.Key.Name, copies.First().dest);
yield return firstCopy;
foreach (var duplicate in copies.Skip(1))
{
yield return new DuplicateTo(firstCopy, duplicate.dest);
}
}
}
}
return GetPlans(top);
}
public static async ValueTask ExecutePlan(WorkQueue queue, Func<ValueTask<Stream>> src, IEnumerable<IStagingPlan> plans)
{
// Extract these files
await using var stream = await src();
var sig = await FileExtractor.ArchiveSigs.MatchesAsync(stream);
stream.Position = 0;
switch (sig)
{
case Definitions.FileType.ZIP:
await ExtractWith7Zip((Definitions.FileType)sig, stream, plans);
break;
default:
throw new Exception($"Invalid archive for extraction");
}
// Copy around the duplicates
foreach (var file in plans.OfType<DuplicateTo>())
{
await file.Execute();
}
// Execute the sub-stages
foreach (var subStage in plans.OfType<ISubStage>())
{
await subStage.Execute(queue);
}
// Dispose of all plans
foreach (var file in plans)
{
await file.DisposeAsync();
}
}
private static async ValueTask ExtractWith7Zip(Definitions.FileType sig, Stream stream, IEnumerable<IStagingPlan> plans)
{
using var archive = await ArchiveFile.Open(stream, sig);
void HandleFile(RelativePath path, IStagingPlan src, Func<Stream> sf)
{
}
var extractor = new TypedExtractor<IStagingPlan>(plans.OfType<IStagingSrc>().ToDictionary(s => s.Source.FileName, s => (IStagingPlan)s),
HandleFile);
extractor.Extract(archive);
}
public static async Task ExecutePlans(WorkQueue queue, IEnumerable<IStagingPlan> plans)
{
foreach (var file in plans.OfType<NativeArchive>())
{
await file.Execute(queue);
}
}
}
}

9
Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs Normal file
View File

@ -0,0 +1,9 @@
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public interface IStagingSrc : IStagingPlan
{
public IPath Source { get; }
}
}

10
Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs Normal file
View File

@ -0,0 +1,10 @@
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public interface ISubStage : IStagingSrc
{
Task Execute(WorkQueue queue);
}
}

22
Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs Normal file
View File

@ -0,0 +1,22 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public class NativeArchive : ASubStage
{
public NativeArchive(AbsolutePath src, IEnumerable<IStagingPlan> plans) : base(plans)
{
Source = src;
Destination = src;
}
public override async ValueTask DisposeAsync()
{
}
public override AbsolutePath Destination { get; }
public override IPath Source { get; }
}
}

21
Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs Normal file
View File

@ -0,0 +1,21 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public class SubStage : ASubStage
{
public SubStage(RelativePath src, AbsolutePath destination, IEnumerable<IStagingPlan> plans) : base(plans)
{
Source = src;
Destination = destination;
}
public override async ValueTask DisposeAsync()
{
}
public override AbsolutePath Destination { get; }
public override IPath Source { get; }
}
}

25
Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs Normal file
View File

@ -0,0 +1,25 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public class TempSubStage : ASubStage
{
private readonly TempFile _temp;
public TempSubStage(RelativePath src, IEnumerable<IStagingPlan> plans) : base(plans)
{
Source = src;
_temp = new TempFile();
}
public override async ValueTask DisposeAsync()
{
await _temp.DisposeAsync();
}
public override AbsolutePath Destination => _temp.Path;
public override IPath Source { get; }
}
}

47
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -190,34 +190,26 @@ namespace Wabbajack.VirtualFileSystem
}
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile,
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile,
IPath relPath, int depth = 0)
{
var hash = await extractedFile.HashAsync();
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension))
{
// Disabled because it isn't enabled on the server
IndexedVirtualFile result = null; //await TryGetContentsFromServer(hash);
if (result != null)
{
Utils.Log($"Downloaded VFS data for {relPath.FileName}");
return ConvertFromIndexedFile(context, result, relPath, parent, extractedFile);
}
}
await using var stream = await extractedFile.GetStream();
var hash = await stream.xxHashAsync();
stream.Position = 0;
var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);
/* TODO
if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
return vself;
*/
var self = new VirtualFile
{
Context = context,
Name = relPath,
Parent = parent,
Size = extractedFile.Size,
Size = stream.Length,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = hash
@ -226,20 +218,19 @@ namespace Wabbajack.VirtualFileSystem
self.FillFullPath(depth);
if (context.UseExtendedHashes)
self.ExtendedHashes = await ExtendedHashes.FromFile(extractedFile);
self.ExtendedHashes = await ExtendedHashes.FromStream(stream);
if (!await extractedFile.CanExtract()) return self;
// Can't extract, so return
if (!sig.HasValue) return self;
try
{
await using var extracted = await extractedFile.ExtractAll(context.Queue, throwOnError:false);
var list = await extracted
.PMap(context.Queue,
file => Analyze(context, self, file.Value, file.Key, depth + 1));
self.Children = list.ToImmutableList();
var list = await FileExtractor2.GatheringExtract(extractedFile,
_ => true,
async (path, sfactory) => await Analyze(context, self, sfactory, path, depth + 1));
self.Children = list.Values.ToImmutableList();
}
catch (Exception ex)
{
@ -405,10 +396,10 @@ namespace Wabbajack.VirtualFileSystem
public string MD5 { get; set; }
public string CRC { get; set; }
public static async ValueTask<ExtendedHashes> FromFile(IExtractedFile file)
public static async ValueTask<ExtendedHashes> FromStream(Stream stream)
{
var hashes = new ExtendedHashes();
await using var stream = await file.OpenRead();
stream.Position = 0;
hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex();
stream.Position = 0;
hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex();

1
Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj
View File

@ -4,6 +4,7 @@
<TargetFramework>netstandard2.1</TargetFramework>
<Platforms>x64</Platforms>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<DocumentationFile>Wabbajack.VirtualFileSystem.xml</DocumentationFile>