diff --git a/Compression.BSA/BA2Reader.cs b/Compression.BSA/BA2Reader.cs index d6c6137b..e3abe392 100644 --- a/Compression.BSA/BA2Reader.cs +++ b/Compression.BSA/BA2Reader.cs @@ -27,7 +27,6 @@ namespace Compression.BSA public class BA2Reader : IBSAReader { - internal AbsolutePath _filename; private Stream _stream; internal BinaryReader _rdr; internal uint _version; @@ -35,15 +34,16 @@ namespace Compression.BSA internal EntryType _type; internal uint _numFiles; internal ulong _nameTableOffset; + public IStreamFactory _streamFactory; public bool UseATIFourCC { get; set; } = false; public bool HasNameTable => _nameTableOffset > 0; - public static async Task Load(AbsolutePath filename) + public static async Task Load(IStreamFactory streamFactory) { - var rdr = new BA2Reader(await filename.OpenShared()) {_filename = filename}; + var rdr = new BA2Reader(await streamFactory.GetStream()) {_streamFactory = streamFactory}; await rdr.LoadHeaders(); return rdr; } @@ -206,7 +206,7 @@ namespace Compression.BSA WriteHeader(bw); - await using var fs = await _bsa._filename.OpenRead(); + await using var fs = await _bsa._streamFactory.GetStream(); using var br = new BinaryReader(fs); foreach (var chunk in _chunks) { @@ -344,6 +344,14 @@ namespace Compression.BSA break; } } + + public async ValueTask GetStreamFactory() + { + var ms = new MemoryStream(); + await CopyDataTo(ms); + ms.Position = 0; + return new MemoryStreamFactory(ms); + } } [JsonName("BA2DX10Entry")] @@ -483,7 +491,7 @@ namespace Compression.BSA public async ValueTask CopyDataTo(Stream output) { - await using var fs = await _bsa._filename.OpenRead(); + await using var fs = await _bsa._streamFactory.GetStream(); fs.Seek((long) _offset, SeekOrigin.Begin); uint len = Compressed ? _size : _realSize; @@ -503,6 +511,14 @@ namespace Compression.BSA await output.WriteAsync(uncompressed, 0, uncompressed.Length); } } + + public async ValueTask GetStreamFactory() + { + var ms = new MemoryStream(); + await CopyDataTo(ms); + ms.Position = 0; + return new MemoryStreamFactory(ms); + } } [JsonName("BA2FileEntryState")] diff --git a/Compression.BSA/BSA/Reader/BSAReader.cs b/Compression.BSA/BSA/Reader/BSAReader.cs index e67d1b1d..cd85723d 100644 --- a/Compression.BSA/BSA/Reader/BSAReader.cs +++ b/Compression.BSA/BSA/Reader/BSAReader.cs @@ -16,7 +16,6 @@ namespace Compression.BSA public const int HeaderLength = 0x24; internal uint _fileCount; - internal AbsolutePath _fileName; internal uint _folderCount; internal uint _folderRecordOffset; private Lazy _folders = null!; @@ -24,6 +23,7 @@ namespace Compression.BSA internal string _magic = string.Empty; internal uint _totalFileNameLength; internal uint _totalFolderNameLength; + public IStreamFactory _streamFactory = new NativeFileStreamFactory(default); public VersionType HeaderType { get; private set; } @@ -56,7 +56,7 @@ namespace Compression.BSA public void Dump(Action print) { - print($"File Name: {_fileName}"); + print($"File Name: {_streamFactory.Name}"); print($"File Count: {_fileCount}"); print($"Magic: {_magic}"); @@ -67,11 +67,11 @@ namespace Compression.BSA } } - public static async ValueTask LoadAsync(AbsolutePath filename) + public static async ValueTask LoadAsync(IStreamFactory factory) { - using var stream = await filename.OpenRead().ConfigureAwait(false); + await using var stream = await factory.GetStream().ConfigureAwait(false); using var br = new BinaryReader(stream); - var bsa = new BSAReader { _fileName = filename }; + var bsa = new BSAReader { _streamFactory = factory }; bsa.LoadHeaders(br); return bsa; } @@ -79,7 +79,7 @@ namespace Compression.BSA public static BSAReader Load(AbsolutePath filename) { - var bsa = new BSAReader { _fileName = filename }; + var bsa = new BSAReader { _streamFactory = new NativeFileStreamFactory(filename)}; using var rdr = bsa.GetStream(); bsa.LoadHeaders(rdr); return bsa; @@ -87,7 +87,7 @@ namespace Compression.BSA internal BinaryReader GetStream() { - return new BinaryReader(File.Open(_fileName.ToString(), FileMode.Open, FileAccess.Read, FileShare.Read)); + return new BinaryReader(_streamFactory.GetStream().Result); } private void LoadHeaders(BinaryReader rdr) diff --git a/Compression.BSA/BSA/Reader/FileRecord.cs b/Compression.BSA/BSA/Reader/FileRecord.cs index fcfc2dab..c7a0fa54 100644 --- a/Compression.BSA/BSA/Reader/FileRecord.cs +++ b/Compression.BSA/BSA/Reader/FileRecord.cs @@ -74,7 +74,7 @@ namespace Compression.BSA public async ValueTask CopyDataTo(Stream output) { - await using var in_file = await BSA._fileName.OpenRead().ConfigureAwait(false); + await using var in_file = await BSA._streamFactory.GetStream().ConfigureAwait(false); using var rdr = new BinaryReader(in_file); rdr.BaseStream.Position = Offset; @@ -165,5 +165,13 @@ namespace Compression.BSA print($"Raw Size: {RawSize}"); print($"Index: {_index}"); } + + public async ValueTask GetStreamFactory() + { + var ms = new MemoryStream(); + await CopyDataTo(ms); + ms.Position = 0; + return new MemoryStreamFactory(ms); + } } } diff --git a/Compression.BSA/BSADispatch.cs b/Compression.BSA/BSADispatch.cs index c3dcc60c..8213a055 100644 --- a/Compression.BSA/BSADispatch.cs +++ b/Compression.BSA/BSADispatch.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Text; using System.Threading.Tasks; @@ -13,9 +15,9 @@ namespace Compression.BSA { return await BSASignatures.MatchesAsync(filename) switch { - Definitions.FileType.TES3 => await TES3Reader.Load(filename), - Definitions.FileType.BSA => await BSAReader.LoadAsync(filename), - Definitions.FileType.BA2 => await BA2Reader.Load(filename), + Definitions.FileType.TES3 => await TES3Reader.Load(new NativeFileStreamFactory(filename)), + Definitions.FileType.BSA => await BSAReader.LoadAsync(new NativeFileStreamFactory(filename)), + Definitions.FileType.BA2 => await BA2Reader.Load(new NativeFileStreamFactory(filename)), _ => throw new InvalidDataException("Filename is not a .bsa or .ba2") }; } @@ -25,5 +27,21 @@ namespace Compression.BSA { return await BSASignatures.MatchesAsync(filename) != null; } + + public static async ValueTask OpenRead(IStreamFactory sFn, Definitions.FileType sig) + { + switch(sig) + { + case Definitions.FileType.TES3: + return await TES3Reader.Load(sFn); + case Definitions.FileType.BSA: + return await BSAReader.LoadAsync(sFn); + case Definitions.FileType.BA2: + return await BA2Reader.Load(sFn); + default: + throw new Exception($"Bad archive format for {sFn.Name}"); + + } + } } } diff --git a/Compression.BSA/Interfaces/IFile.cs b/Compression.BSA/Interfaces/IFile.cs index fe8054b5..c9aca0f5 100644 --- a/Compression.BSA/Interfaces/IFile.cs +++ b/Compression.BSA/Interfaces/IFile.cs @@ -32,5 +32,6 @@ namespace Compression.BSA ValueTask CopyDataTo(Stream output); void Dump(Action print); + ValueTask GetStreamFactory(); } } diff --git a/Compression.BSA/MemoryStreamFactory.cs b/Compression.BSA/MemoryStreamFactory.cs new file mode 100644 index 00000000..c428ed9e --- /dev/null +++ b/Compression.BSA/MemoryStreamFactory.cs @@ -0,0 +1,24 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Compression.BSA +{ + public class MemoryStreamFactory : IStreamFactory + { + private readonly MemoryStream _data; + + public MemoryStreamFactory(MemoryStream data) + { + _data = data; + } + public async ValueTask GetStream() + { + return new MemoryStream(_data.GetBuffer(), 0, (int)_data.Length); + } + + public DateTime LastModifiedUtc => DateTime.UtcNow; + public IPath Name => (RelativePath)"BSA Memory Stream"; + } +} diff --git a/Compression.BSA/StreamView.cs b/Compression.BSA/StreamView.cs new file mode 100644 index 00000000..24742231 --- /dev/null +++ b/Compression.BSA/StreamView.cs @@ -0,0 +1,82 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Compression.BSA +{ + public class StreamView : Stream + { + private Stream _base; + private long _startPos; + private long _length; + + public StreamView(Stream baseStream, long startPos, long length) + { + _base = baseStream; + _startPos = startPos; + _length = length; + } + + public override void Flush() + { + throw new System.NotImplementedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + var realCount = Math.Min(count, Length - Position); + return _base.Read(buffer, offset, (int)realCount); + } + + public override long Seek(long offset, SeekOrigin origin) + { + switch (origin) + { + case SeekOrigin.Begin: + Position = offset; + return Position; + case SeekOrigin.End: + Position = _length - offset; + return Position; + case SeekOrigin.Current: + Position += offset; + return Position; + default: + throw new ArgumentOutOfRangeException(nameof(origin), origin, null); + } + } + + public override void SetLength(long value) + { + throw new System.NotImplementedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new System.NotImplementedException(); + } + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + public override long Length => _length; + + public override long Position + { + get + { + return _base.Position - _startPos; + } + set + { + _base.Position = _startPos + value; + } + } + + public override async ValueTask DisposeAsync() + { + await _base.DisposeAsync(); + } + } +} diff --git a/Compression.BSA/TES3Reader.cs b/Compression.BSA/TES3Reader.cs index 0ebe34f6..f9fd7afb 100644 --- a/Compression.BSA/TES3Reader.cs +++ b/Compression.BSA/TES3Reader.cs @@ -17,15 +17,15 @@ namespace Compression.BSA private uint _fileCount; private TES3FileEntry[] _files; internal long _dataOffset; - internal AbsolutePath _filename; + public IStreamFactory _streamFactory; - public static async ValueTask Load(AbsolutePath filename) + public static async ValueTask Load(IStreamFactory factory) { - await using var fs = await filename.OpenRead(); + await using var fs = await factory.GetStream(); using var br = new BinaryReader(fs); var rdr = new TES3Reader { - _filename = filename, + _streamFactory = factory, _versionNumber = br.ReadUInt32(), _hashTableOffset = br.ReadUInt32(), _fileCount = br.ReadUInt32() @@ -125,16 +125,27 @@ namespace Compression.BSA public async ValueTask CopyDataTo(Stream output) { - await using var fs = await Archive._filename.OpenRead(); + await using var fs = await Archive._streamFactory.GetStream(); fs.Position = Archive._dataOffset + Offset; await fs.CopyToLimitAsync(output, (int)Size); } + public async ValueTask GetStreamFactory() + { + var ms = new MemoryStream(); + await CopyDataTo(ms); + ms.Position = 0; + return new MemoryStreamFactory(ms); + } + + + public void Dump(Action print) { throw new NotImplementedException(); } + public uint Offset { get; set; } public uint NameOffset { get; set; } public uint Hash1 { get; set; } diff --git a/Wabbajack.Common/IStreamFactory.cs b/Wabbajack.Common/IStreamFactory.cs new file mode 100644 index 00000000..783c425e --- /dev/null +++ b/Wabbajack.Common/IStreamFactory.cs @@ -0,0 +1,34 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.Common +{ + public interface IStreamFactory + { + ValueTask GetStream(); + + DateTime LastModifiedUtc { get; } + + IPath Name { get; } + + } + public class NativeFileStreamFactory : IStreamFactory + { + private AbsolutePath _file; + + public NativeFileStreamFactory(AbsolutePath file) + { + _file = file; + } + public async ValueTask GetStream() + { + return await _file.OpenRead(); + } + + public DateTime LastModifiedUtc => _file.LastModifiedUtc; + public IPath Name => _file; + } + +} diff --git a/Wabbajack.Common/Patches.cs b/Wabbajack.Common/Patches.cs index 6e9f800f..2791e0dd 100644 --- a/Wabbajack.Common/Patches.cs +++ b/Wabbajack.Common/Patches.cs @@ -50,7 +50,7 @@ namespace Wabbajack.Common await patch.CopyToAsync(output); } - public static async Task CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash, + public static async Task CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, Stream? patchOutStream = null) { var key = PatchKey(srcHash, destHash); @@ -128,7 +128,7 @@ namespace Wabbajack.Common public static Task CreatePatchCached(byte[] a, byte[] b, Stream output) => PatchCache.CreatePatchCached(a, b, output); - public static Task CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash, Stream? patchOutStream = null) => + public static Task CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, Stream? patchOutStream = null) => PatchCache.CreatePatchCached(srcStream, srcHash, destStream, destHash, patchOutStream); public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch) => diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index df948df3..ed41a416 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -9,6 +9,7 @@ using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; using Wabbajack.VirtualFileSystem; +using Wabbajack.VirtualFileSystem.SevenZipExtractor; using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; @@ -35,6 +36,8 @@ namespace Wabbajack.Lib public bool UseCompression { get; set; } + public TempFolder? ExtractedModlistFolder { get; set; } = null; + public AInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters? parameters, int steps, Game game) : base(steps) @@ -45,12 +48,23 @@ namespace Wabbajack.Lib DownloadFolder = downloadFolder; SystemParameters = parameters; Game = game.MetaData(); + } + private ExtractedFiles? ExtractedModListFiles { get; set; } = null; public async Task ExtractModlist() { - ExtractedModListFiles = await FileExtractor.ExtractAll(Queue, ModListArchive); + ExtractedModlistFolder = await TempFolder.Create(); + await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(ModListArchive), _ => true, + async (path, sfn) => + { + await using var s = await sfn.GetStream(); + var fp = ExtractedModlistFolder.Dir.Combine(path); + fp.Parent.CreateDirectory(); + await fp.WriteAllAsync(s); + return 0; + }); } @@ -73,8 +87,7 @@ namespace Wabbajack.Lib public async Task LoadBytesFromPath(RelativePath path) { - await using var e = await ExtractedModListFiles![path].OpenRead(); - return await e.ReadAllAsync(); + return await ExtractedModlistFolder!.Dir.Combine(path).ReadAllBytesAsync(); } public static ModList LoadFromFile(AbsolutePath path) @@ -113,23 +126,23 @@ namespace Wabbajack.Lib public async Task InstallArchives() { - await VFS.CopyTo(Queue, ModList.Directives - .OfType() - .Select(a => (VFS.Index.FileForArchiveHashPath(a.ArchiveHashPath), a.To.RelativeTo(OutputFolder)))); - /* - Info("Installing Archives"); - Info("Grouping Install Files"); var grouped = ModList.Directives .OfType() - .GroupBy(e => e.ArchiveHashPath.BaseHash) - .ToDictionary(k => k.Key); - var archives = ModList.Archives - .Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) }) - .Where(a => a.AbsolutePath != null) - .ToList(); + .Select(a => new {VF = VFS.Index.FileForArchiveHashPath(a.ArchiveHashPath), Directive = a}) + .GroupBy(a => a.VF) + .ToDictionary(a => a.Key); - Info("Installing Archives"); - await archives.PMap(Queue, UpdateTracker,a => InstallArchive(Queue, a.Archive, a.AbsolutePath, grouped[a.Archive.Hash]));*/ + if (grouped.Count == 0) return; + + await VFS.Extract(Queue, grouped.Keys.ToHashSet(), async (vf, sf) => + { + await using var s = await sf.GetStream(); + foreach (var directive in grouped[vf]) + { + s.Position = 0; + await directive.Directive.To.RelativeTo(OutputFolder).WriteAllAsync(s, false); + } + }); } private async Task InstallArchive(WorkQueue queue, Archive archive, AbsolutePath absolutePath, IGrouping grouping) diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 6a3651bc..9c980fc6 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -6,10 +6,12 @@ using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using AngleSharp.Common; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.Validation; +using Wabbajack.VirtualFileSystem; using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib @@ -495,17 +497,35 @@ namespace Wabbajack.Lib .ToArray(); if (toBuild.Length == 0) return; + + // Extract all the source files + var indexed = toBuild.GroupBy(f => (VFS.Index.FileForArchiveHashPath(f.ArchiveHashPath))) + .ToDictionary(f => f.Key); + await VFS.Extract(Queue, indexed.Keys.ToHashSet(), + async (vf, sf) => + { + // For each, extract the destination + var matches = indexed[vf]; + using var iqueue = new WorkQueue(1); + foreach (var match in matches) + { + var destFile = FindDestFile(match.To); + // Build the patch + await VFS.Extract(iqueue, new[] {destFile}.ToHashSet(), + async (destvf, destsfn) => + { + Info($"Patching {match.To}"); + Status($"Patching {match.To}"); + await using var srcStream = await sf.GetStream(); + await using var destStream = await destsfn.GetStream(); + var patchSize = await Utils.CreatePatchCached(srcStream, vf.Hash, destStream, destvf.Hash); + Info($"Patch size {patchSize} for {match.To}"); + }); + } + + }); - var groups = toBuild - .Where(p => p.PatchID == default) - .GroupBy(p => p.ArchiveHashPath.BaseHash) - .ToList(); - - Info($"Patching building patches from {groups.Count} archives"); - var absolutePaths = AllFiles.ToDictionary(e => e.Path, e => e.AbsolutePath); - await groups.PMap(Queue, group => BuildArchivePatches(group.Key, group, absolutePaths)); - - + // Load in the patches await InstallDirectives.OfType() .Where(p => p.PatchID == default) .PMap(Queue, async pfa => @@ -514,6 +534,7 @@ namespace Wabbajack.Lib .Select(c => (Utils.TryGetPatch(c.Hash, pfa.Hash, out var data), data, c)) .ToArray(); + // Pick the best patch if (patches.All(p => p.Item1)) { var (_, bytes, file) = IncludePatches.PickPatch(this, patches); @@ -529,42 +550,19 @@ namespace Wabbajack.Lib Error($"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}"); } - private async Task BuildArchivePatches(Hash archiveSha, IEnumerable group, - Dictionary absolutePaths) + private VirtualFile FindDestFile(RelativePath to) { - await using var files = await VFS.StageWith(@group.Select(g => VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath))); - var byPath = files.GroupBy(f => string.Join("|", f.FilesInFullPath.Skip(1).Select(i => i.Name))) - .ToDictionary(f => f.Key, f => f.First()); - // Now Create the patches - await @group.PMap(Queue, async entry => - { - Info($"Patching {entry.To}"); - Status($"Patching {entry.To}"); - var srcFile = byPath[string.Join("|", entry.ArchiveHashPath.Paths)]; - await using var srcStream = await srcFile.OpenRead(); - await using var destStream = await LoadDataForTo(entry.To, absolutePaths); - var patchSize = await Utils.CreatePatchCached(srcStream, srcFile.Hash, destStream, entry.Hash); - Info($"Patch size {patchSize} for {entry.To}"); - }); - } - - private async Task LoadDataForTo(RelativePath to, Dictionary absolutePaths) - { - if (absolutePaths.TryGetValue(to, out var absolute)) - return await absolute.OpenRead(); + var abs = to.RelativeTo(MO2Folder); + if (abs.Exists) + return VFS.Index.ByRootPath[abs]; if (to.StartsWith(Consts.BSACreationDir)) { var bsaId = (RelativePath)((string)to).Split('\\')[1]; var bsa = InstallDirectives.OfType().First(b => b.TempID == bsaId); - - var a = await BSADispatch.OpenRead(MO2Folder.Combine(bsa.To)); var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray()); - var file = a.Files.First(e => e.Path == find); - var returnStream = new TempStream(); - await file.CopyDataTo(returnStream); - returnStream.Position = 0; - return returnStream; + + return VFS.Index.ByRootPath[MO2Folder.Combine(bsa.To)].Children.First(c => c.RelativeName == find); } throw new ArgumentException($"Couldn't load data for {to}"); diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index 92877a92..8d1f3e6a 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -180,6 +180,7 @@ namespace Wabbajack.Lib SetScreenSizeInPrefs(); UpdateTracker.NextStep("Installation complete! You may exit the program."); + await ExtractedModlistFolder!.DisposeAsync(); await Metrics.Send(Metrics.FinishInstall, ModList.Name); return true; @@ -243,7 +244,8 @@ namespace Wabbajack.Lib Status($"Writing included .meta file {directive.To}"); var outPath = DownloadFolder.Combine(directive.To); if (outPath.IsFile) await outPath.DeleteAsync(); - await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID)); + var bytes = await LoadBytesFromPath(directive.SourceDataID); + await outPath.WriteAllBytesAsync(bytes); }); } diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index f980d9bf..9b6ba5cb 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -221,7 +221,7 @@ namespace Wabbajack.VirtualFileSystem r => fileNames.ContainsKey(r), async (rel, csf) => { - await HandleFile(fileNames[rel], sfn); + await HandleFile(fileNames[rel], csf); return 0; }); } @@ -261,12 +261,6 @@ namespace Wabbajack.VirtualFileSystem }; } - public async Task CopyTo(WorkQueue queue, IEnumerable<(VirtualFile src, AbsolutePath dest)> directives) - { - var plans = StagingPlan.StagingPlan.CreatePlan(directives).ToArray(); - await StagingPlan.StagingPlan.ExecutePlans(queue, plans); - } - public async Task> StageWith(IEnumerable files) { return new AsyncDisposableList(await Stage(files), files); diff --git a/Wabbajack.VirtualFileSystem/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor.cs index 850f6053..f736dbea 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor.cs @@ -30,6 +30,7 @@ namespace Wabbajack.VirtualFileSystem public static async Task ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable OnlyFiles = null, bool throwOnError = true) { + throw new NotImplementedException(); OnlyFiles ??= new RelativePath[0]; try diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs index 37d05eda..eaf965bf 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; +using Compression.BSA; using Wabbajack.Common; using Wabbajack.Common.FileSignatures; using Wabbajack.VirtualFileSystem.SevenZipExtractor; @@ -14,7 +15,7 @@ namespace Wabbajack.VirtualFileSystem Definitions.FileType.BSA, Definitions.FileType.BA2, Definitions.FileType.ZIP, - Definitions.FileType.EXE, + //Definitions.FileType.EXE, Definitions.FileType.RAR, Definitions.FileType._7Z); @@ -31,11 +32,33 @@ namespace Wabbajack.VirtualFileSystem case Definitions.FileType.ZIP: return await GatheringExtractWith7Zip(archive, (Definitions.FileType)sig, shouldExtract, mapfn); + case Definitions.FileType.TES3: + case Definitions.FileType.BSA: + case Definitions.FileType.BA2: + return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn); + + default: - throw new Exception("Invalid file format"); + throw new Exception($"Invalid file format {sFn.Name}"); } } + private static async Task> GatheringExtractWithBSA(IStreamFactory sFn, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) + { + var archive = await BSADispatch.OpenRead(sFn, sig); + var results = new Dictionary(); + foreach (var entry in archive.Files) + { + if (!shouldExtract(entry.Path)) + continue; + + var result = await mapfn(entry.Path, await entry.GetStreamFactory()); + results.Add(entry.Path, result); + } + + return results; + } + private static async Task> GatheringExtractWith7Zip(Stream stream, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) { return await new GatheringExtractor(stream, sig, shouldExtract, mapfn).Extract(); diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/IStreamFactory.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/StreamFactories.cs similarity index 51% rename from Wabbajack.VirtualFileSystem/FileExtractor2/IStreamFactory.cs rename to Wabbajack.VirtualFileSystem/FileExtractor2/StreamFactories.cs index e942ba5a..83294259 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/IStreamFactory.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/StreamFactories.cs @@ -5,13 +5,6 @@ using Wabbajack.Common; namespace Wabbajack.VirtualFileSystem { - public interface IStreamFactory - { - Task GetStream(); - - DateTime LastModifiedUtc { get; } - - } public class UnmanagedStreamFactory : IStreamFactory { @@ -23,7 +16,7 @@ namespace Wabbajack.VirtualFileSystem _data = data; _size = size; } - public async Task GetStream() + public async ValueTask GetStream() { unsafe { @@ -32,21 +25,8 @@ namespace Wabbajack.VirtualFileSystem } public DateTime LastModifiedUtc => DateTime.UtcNow; + public IPath Name => (RelativePath)"Unmanaged Memory Stream"; } - public class NativeFileStreamFactory : IStreamFactory - { - private AbsolutePath _file; - public NativeFileStreamFactory(AbsolutePath file) - { - _file = file; - } - public async Task GetStream() - { - return await _file.OpenRead(); - } - - public DateTime LastModifiedUtc => _file.LastModifiedUtc; - } } diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs b/Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs deleted file mode 100644 index bef272f3..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public abstract class ASubStage : ISubStage - { - private IEnumerable _plans; - - public ASubStage(IEnumerable plans) - { - _plans = plans; - } - - public abstract ValueTask DisposeAsync(); - public abstract AbsolutePath Destination { get; } - public abstract IPath Source { get; } - - public async Task Execute(WorkQueue queue) - { - await StagingPlan.ExecutePlan(queue, async () => await Destination.OpenWrite(), _plans); - } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs b/Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs deleted file mode 100644 index 056be6bf..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public class CopyTo : IStagingSrc - { - public CopyTo(IPath src, AbsolutePath destination) - { - Destination = destination; - Source = src; - } - - public AbsolutePath Destination { get; } - - public async ValueTask DisposeAsync() - { - - } - - public IPath Source { get; } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs b/Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs deleted file mode 100644 index 8255e229..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public class DuplicateTo : IStagingPlan - { - private readonly IStagingPlan _src; - - public DuplicateTo(IStagingPlan src, AbsolutePath destination) - { - _src = src; - Destination = destination; - } - public async ValueTask DisposeAsync() - { - } - - public async Task Execute() - { - await _src.Destination.CopyToAsync(Destination); - } - - public AbsolutePath Destination { get; } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs b/Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs deleted file mode 100644 index a6e5cba9..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs +++ /dev/null @@ -1,156 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Wabbajack.Common; -using Wabbajack.Common.FileSignatures; -using Wabbajack.VirtualFileSystem.SevenZipExtractor; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public interface IStagingPlan : IAsyncDisposable - { - AbsolutePath Destination { get; } - } - - public static class StagingPlan - { - - public static IEnumerable CreatePlan(IEnumerable<(VirtualFile src, AbsolutePath dest)> directives) - { - var top = new VirtualFile(); - var original = directives.ToHashSet(); - var files = directives.GroupBy(f => f.src).ToDictionary(f => f.Key); - - var childrenForParent = files.Keys.GroupBy(f => f.Parent ?? top).ToDictionary(f => f.Key); - - var allFilesForParent = files.Select(f => f.Key) - .SelectMany(f => f.FilesInFullPath) - .Distinct() - .GroupBy(f => f.Parent ?? top) - .ToDictionary(f => f.Key, f => f.ToArray()); - - var baseDirectives = new List(); - - // Recursive internal function to get the plans for a given parent, we'll then start at the - // null parent (the top) and recurse our way down into all the children files. - IEnumerable GetPlans(VirtualFile parent) - { - foreach (var forParent in allFilesForParent[parent]) - { - // Do we need files inside this file? - if (childrenForParent.TryGetValue(forParent, out var children)) - { - if (files.TryGetValue(forParent, out var copies)) - { - ASubStage subStage; - if (parent == top) - { - subStage = new NativeArchive(forParent.AbsoluteName, GetPlans(forParent)); - } - else - { - subStage = new SubStage(forParent.RelativeName, copies.First().dest, GetPlans(forParent)); - } - yield return subStage; - foreach (var copy in copies) - { - yield return new DuplicateTo(subStage, copy.dest); - } - } - else - { - if (parent == top) - { - yield return new NativeArchive(forParent.AbsoluteName, GetPlans(forParent)); - } - else - { - yield return new TempSubStage(forParent.RelativeName, GetPlans(forParent)); - } - } - } - else - { - // If not then we need to copy this file around - var copies = files[forParent]; - var firstCopy = new CopyTo(copies.Key.Name, copies.First().dest); - yield return firstCopy; - - foreach (var duplicate in copies.Skip(1)) - { - yield return new DuplicateTo(firstCopy, duplicate.dest); - } - } - } - } - - return GetPlans(top); - } - public static async ValueTask ExecutePlan(WorkQueue queue, Func> src, IEnumerable plans) - { - // Extract these files - - await using var stream = await src(); - var sig = await FileExtractor.ArchiveSigs.MatchesAsync(stream); - stream.Position = 0; - - switch (sig) - { - case Definitions.FileType.ZIP: - await ExtractWith7Zip((Definitions.FileType)sig, stream, plans); - - - break; - - default: - throw new Exception($"Invalid archive for extraction"); - } - - - // Copy around the duplicates - foreach (var file in plans.OfType()) - { - await file.Execute(); - } - - // Execute the sub-stages - foreach (var subStage in plans.OfType()) - { - await subStage.Execute(queue); - } - - // Dispose of all plans - foreach (var file in plans) - { - await file.DisposeAsync(); - } - - } - - private static async ValueTask ExtractWith7Zip(Definitions.FileType sig, Stream stream, IEnumerable plans) - { - using var archive = await ArchiveFile.Open(stream, sig); - - void HandleFile(RelativePath path, IStagingPlan src, Func sf) - { - - } - - var extractor = new TypedExtractor(plans.OfType().ToDictionary(s => s.Source.FileName, s => (IStagingPlan)s), - HandleFile); - extractor.Extract(archive); - - } - - - public static async Task ExecutePlans(WorkQueue queue, IEnumerable plans) - { - foreach (var file in plans.OfType()) - { - await file.Execute(queue); - } - } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs b/Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs deleted file mode 100644 index 6e22b250..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs +++ /dev/null @@ -1,9 +0,0 @@ -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public interface IStagingSrc : IStagingPlan - { - public IPath Source { get; } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs b/Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs deleted file mode 100644 index 6120194b..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public interface ISubStage : IStagingSrc - { - Task Execute(WorkQueue queue); - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs b/Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs deleted file mode 100644 index 2fcaf81b..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public class NativeArchive : ASubStage - { - public NativeArchive(AbsolutePath src, IEnumerable plans) : base(plans) - { - Source = src; - Destination = src; - } - - public override async ValueTask DisposeAsync() - { - } - - public override AbsolutePath Destination { get; } - public override IPath Source { get; } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs b/Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs deleted file mode 100644 index 33d2dfd7..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public class SubStage : ASubStage - { - public SubStage(RelativePath src, AbsolutePath destination, IEnumerable plans) : base(plans) - { - Source = src; - Destination = destination; - } - - public override async ValueTask DisposeAsync() - { - } - public override AbsolutePath Destination { get; } - public override IPath Source { get; } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs b/Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs deleted file mode 100644 index 32055793..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public class TempSubStage : ASubStage - { - private readonly TempFile _temp; - - public TempSubStage(RelativePath src, IEnumerable plans) : base(plans) - { - Source = src; - _temp = new TempFile(); - } - - public override async ValueTask DisposeAsync() - { - await _temp.DisposeAsync(); - } - - public override AbsolutePath Destination => _temp.Path; - public override IPath Source { get; } - } -} diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 9a24eb3e..e6d30bac 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -141,7 +141,7 @@ namespace Wabbajack.VirtualFileSystem itm.ThisAndAllChildrenReduced(fn); } - private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IExtractedFile extractedFile) + private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IStreamFactory extractedFile) { var vself = new VirtualFile { @@ -161,7 +161,7 @@ namespace Wabbajack.VirtualFileSystem return vself; } - private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IExtractedFile extractedFile, Hash hash, out VirtualFile found) + private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile, Hash hash, out VirtualFile found) { var result = _vfsCache.Get(hash.ToArray()); if (result == null) @@ -199,11 +199,9 @@ namespace Wabbajack.VirtualFileSystem var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream); - /* TODO if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself)) return vself; - */ - + var self = new VirtualFile { Context = context,