Patching and extraction works, fewer failing tests

Compression.BSA/BA2Reader.cs

@@ -27,7 +27,6 @@ namespace Compression.BSA
 
     public class BA2Reader : IBSAReader
     {
-        internal AbsolutePath _filename;
         private Stream _stream;
         internal BinaryReader _rdr;
         internal uint _version;
@@ -35,15 +34,16 @@ namespace Compression.BSA
         internal EntryType _type;
         internal uint _numFiles;
         internal ulong _nameTableOffset;
+        public IStreamFactory _streamFactory;
         public bool UseATIFourCC { get; set; } = false;
 
         public bool HasNameTable => _nameTableOffset > 0;
 
         
         
-        public static async Task<BA2Reader> Load(AbsolutePath filename)
+        public static async Task<BA2Reader> Load(IStreamFactory streamFactory)
         {
-            var rdr = new BA2Reader(await filename.OpenShared()) {_filename = filename};
+            var rdr = new BA2Reader(await streamFactory.GetStream()) {_streamFactory = streamFactory};
             await rdr.LoadHeaders();
             return rdr;
         }
@@ -206,7 +206,7 @@ namespace Compression.BSA
 
             WriteHeader(bw);
 
-            await using var fs = await _bsa._filename.OpenRead();
+            await using var fs = await _bsa._streamFactory.GetStream();
             using var br = new BinaryReader(fs);
             foreach (var chunk in _chunks)
             {
@@ -344,6 +344,14 @@ namespace Compression.BSA
                     break;
             }
         }
+        
+        public async ValueTask<IStreamFactory> GetStreamFactory()
+        {
+            var ms = new MemoryStream();
+            await CopyDataTo(ms);
+            ms.Position = 0;
+            return new MemoryStreamFactory(ms);
+        }
     }
 
     [JsonName("BA2DX10Entry")]
@@ -483,7 +491,7 @@ namespace Compression.BSA
 
         public async ValueTask CopyDataTo(Stream output)
         {
-            await using var fs = await _bsa._filename.OpenRead();
+            await using var fs = await _bsa._streamFactory.GetStream();
             fs.Seek((long) _offset, SeekOrigin.Begin);
             uint len = Compressed ? _size : _realSize;
 
@@ -503,6 +511,14 @@ namespace Compression.BSA
                 await output.WriteAsync(uncompressed, 0, uncompressed.Length);
             }
         }
+        
+        public async ValueTask<IStreamFactory> GetStreamFactory()
+        {
+            var ms = new MemoryStream();
+            await CopyDataTo(ms);
+            ms.Position = 0;
+            return new MemoryStreamFactory(ms);
+        }
     }
 
     [JsonName("BA2FileEntryState")]

Compression.BSA/BSA/Reader/BSAReader.cs

@@ -16,7 +16,6 @@ namespace Compression.BSA
         public const int HeaderLength = 0x24;
 
         internal uint _fileCount;
-        internal AbsolutePath _fileName;
         internal uint _folderCount;
         internal uint _folderRecordOffset;
         private Lazy<FolderRecord[]> _folders = null!;
@@ -24,6 +23,7 @@ namespace Compression.BSA
         internal string _magic = string.Empty;
         internal uint _totalFileNameLength;
         internal uint _totalFolderNameLength;
+        public IStreamFactory _streamFactory = new NativeFileStreamFactory(default);
 
         public VersionType HeaderType { get; private set; }
 
@@ -56,7 +56,7 @@ namespace Compression.BSA
 
         public void Dump(Action<string> print)
         {
-            print($"File Name: {_fileName}");
+            print($"File Name: {_streamFactory.Name}");
             print($"File Count: {_fileCount}");
             print($"Magic: {_magic}");
 
@@ -67,11 +67,11 @@ namespace Compression.BSA
             }
         }
 
-        public static async ValueTask<BSAReader> LoadAsync(AbsolutePath filename)
+        public static async ValueTask<BSAReader> LoadAsync(IStreamFactory factory)
         {
-            using var stream = await filename.OpenRead().ConfigureAwait(false);
+            await using var stream = await factory.GetStream().ConfigureAwait(false);
             using var br = new BinaryReader(stream);
-            var bsa = new BSAReader { _fileName = filename };
+            var bsa = new BSAReader { _streamFactory = factory };
             bsa.LoadHeaders(br);
             return bsa;
         }
@@ -79,7 +79,7 @@ namespace Compression.BSA
 
         public static BSAReader Load(AbsolutePath filename)
         {
-            var bsa = new BSAReader { _fileName = filename };
+            var bsa = new BSAReader { _streamFactory = new NativeFileStreamFactory(filename)};
             using var rdr = bsa.GetStream();
             bsa.LoadHeaders(rdr);
             return bsa;
@@ -87,7 +87,7 @@ namespace Compression.BSA
 
         internal BinaryReader GetStream()
         {
-            return new BinaryReader(File.Open(_fileName.ToString(), FileMode.Open, FileAccess.Read, FileShare.Read));
+            return new BinaryReader(_streamFactory.GetStream().Result);
         }
 
         private void LoadHeaders(BinaryReader rdr)

Compression.BSA/BSA/Reader/FileRecord.cs

@@ -74,7 +74,7 @@ namespace Compression.BSA
 
         public async ValueTask CopyDataTo(Stream output)
         {
-            await using var in_file = await BSA._fileName.OpenRead().ConfigureAwait(false);
+            await using var in_file = await BSA._streamFactory.GetStream().ConfigureAwait(false);
             using var rdr = new BinaryReader(in_file);
             rdr.BaseStream.Position = Offset;
 
@@ -165,5 +165,13 @@ namespace Compression.BSA
             print($"Raw Size: {RawSize}");
             print($"Index: {_index}");
         }
+
+        public async ValueTask<IStreamFactory> GetStreamFactory()
+        {
+            var ms = new MemoryStream();
+            await CopyDataTo(ms);
+            ms.Position = 0;
+            return new MemoryStreamFactory(ms);
+        }
     }
 }

Compression.BSA/BSADispatch.cs

@@ -1,4 +1,6 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Text;
 using System.Threading.Tasks;
@@ -13,9 +15,9 @@ namespace Compression.BSA
         {
             return await BSASignatures.MatchesAsync(filename) switch
             {
-                Definitions.FileType.TES3 => await TES3Reader.Load(filename),
+                Definitions.FileType.TES3 => await TES3Reader.Load(new NativeFileStreamFactory(filename)),
-                Definitions.FileType.BSA => await BSAReader.LoadAsync(filename),
+                Definitions.FileType.BSA => await BSAReader.LoadAsync(new NativeFileStreamFactory(filename)),
-                Definitions.FileType.BA2 => await BA2Reader.Load(filename),
+                Definitions.FileType.BA2 => await BA2Reader.Load(new NativeFileStreamFactory(filename)),
                 _ => throw new InvalidDataException("Filename is not a .bsa or .ba2")
             };
         }
@@ -25,5 +27,21 @@ namespace Compression.BSA
         {
             return await BSASignatures.MatchesAsync(filename) != null;
         }
+
+        public static async ValueTask<IBSAReader> OpenRead(IStreamFactory sFn, Definitions.FileType sig)
+        {
+            switch(sig)
+            {
+                case Definitions.FileType.TES3:
+                    return await TES3Reader.Load(sFn);
+                case Definitions.FileType.BSA:
+                    return await BSAReader.LoadAsync(sFn);
+                case Definitions.FileType.BA2:
+                    return await BA2Reader.Load(sFn);
+                default:
+                    throw new Exception($"Bad archive format for {sFn.Name}");
+                
+            }
+        }
     }
 }

Compression.BSA/Interfaces/IFile.cs

@@ -32,5 +32,6 @@ namespace Compression.BSA
         ValueTask CopyDataTo(Stream output);
 
         void Dump(Action<string> print);
+        ValueTask<IStreamFactory> GetStreamFactory();
     }
 }

Compression.BSA/MemoryStreamFactory.cs

@@ -0,0 +1,24 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Compression.BSA
+{
+    public class MemoryStreamFactory : IStreamFactory
+    {
+        private readonly MemoryStream _data;
+
+        public MemoryStreamFactory(MemoryStream data)
+        {
+            _data = data;
+        }
+        public async ValueTask<Stream> GetStream()
+        {
+            return new MemoryStream(_data.GetBuffer(), 0, (int)_data.Length);
+        }
+
+        public DateTime LastModifiedUtc => DateTime.UtcNow;
+        public IPath Name => (RelativePath)"BSA Memory Stream";
+    }
+}

Compression.BSA/StreamView.cs

@@ -0,0 +1,82 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Compression.BSA
+{
+    public class StreamView : Stream
+    {
+        private Stream _base;
+        private long _startPos;
+        private long _length;
+
+        public StreamView(Stream baseStream, long startPos, long length)
+        {
+            _base = baseStream;
+            _startPos = startPos;
+            _length = length;
+        }
+        
+        public override void Flush()
+        {
+            throw new System.NotImplementedException();
+        }
+
+        public override int Read(byte[] buffer, int offset, int count)
+        {
+            var realCount = Math.Min(count, Length - Position);
+            return _base.Read(buffer, offset, (int)realCount);
+        }
+
+        public override long Seek(long offset, SeekOrigin origin)
+        {
+            switch (origin)
+            {
+                case SeekOrigin.Begin:
+                    Position = offset;
+                    return Position;
+                case SeekOrigin.End:
+                    Position = _length - offset;
+                    return Position;
+                case SeekOrigin.Current:
+                    Position += offset;
+                    return Position;
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(origin), origin, null);
+            }
+        }
+
+        public override void SetLength(long value)
+        {
+            throw new System.NotImplementedException();
+        }
+
+        public override void Write(byte[] buffer, int offset, int count)
+        {
+            throw new System.NotImplementedException();
+        }
+
+        public override bool CanRead => true;
+        public override bool CanSeek => true;
+        public override bool CanWrite => false;
+        public override long Length => _length;
+
+        public override long Position
+        {
+            get
+            {
+                return _base.Position - _startPos;
+            }
+            set
+            {
+                _base.Position = _startPos + value;
+            }
+        }
+
+        public override async ValueTask DisposeAsync()
+        {
+            await _base.DisposeAsync();
+        }
+    }
+}

Compression.BSA/TES3Reader.cs

@@ -17,15 +17,15 @@ namespace Compression.BSA
         private uint _fileCount;
         private TES3FileEntry[] _files;
         internal long _dataOffset;
-        internal AbsolutePath _filename;
+        public IStreamFactory _streamFactory;
 
-        public static async ValueTask<TES3Reader> Load(AbsolutePath filename)
+        public static async ValueTask<TES3Reader> Load(IStreamFactory factory)
         {
-            await using var fs = await filename.OpenRead();
+            await using var fs = await factory.GetStream();
             using var br = new BinaryReader(fs);
             var rdr = new TES3Reader
             {
-                _filename = filename,
+                _streamFactory = factory,
                 _versionNumber = br.ReadUInt32(),
                 _hashTableOffset = br.ReadUInt32(),
                 _fileCount = br.ReadUInt32()
@@ -125,16 +125,27 @@ namespace Compression.BSA
 
         public async ValueTask CopyDataTo(Stream output)
         {
-            await using var fs = await Archive._filename.OpenRead();
+            await using var fs = await Archive._streamFactory.GetStream();
             fs.Position = Archive._dataOffset + Offset;
             await fs.CopyToLimitAsync(output, (int)Size);
         }
 
+        public async ValueTask<IStreamFactory> GetStreamFactory()
+        {
+            var ms = new MemoryStream();
+            await CopyDataTo(ms);
+            ms.Position = 0;
+            return new MemoryStreamFactory(ms);
+        }
+
+
+        
         public void Dump(Action<string> print)
         {
             throw new NotImplementedException();
         }
 
+
         public uint Offset { get; set; }
         public uint NameOffset { get; set; }
         public uint Hash1 { get; set; }

Wabbajack.Common/IStreamFactory.cs

@@ -0,0 +1,34 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.Common
+{
+    public interface IStreamFactory
+    {
+        ValueTask<Stream> GetStream();
+        
+        DateTime LastModifiedUtc { get; }
+        
+        IPath Name { get; }
+        
+    }
+    public class NativeFileStreamFactory : IStreamFactory
+    {
+        private AbsolutePath _file;
+
+        public NativeFileStreamFactory(AbsolutePath file)
+        {
+            _file = file;
+        }
+        public async ValueTask<Stream> GetStream()
+        {
+            return await _file.OpenRead();
+        }
+
+        public DateTime LastModifiedUtc => _file.LastModifiedUtc;
+        public IPath Name => _file;
+    }
+    
+}

Wabbajack.Common/Patches.cs

@@ -50,7 +50,7 @@ namespace Wabbajack.Common
             await patch.CopyToAsync(output);
         }
 
-        public static async Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash,
+        public static async Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash,
             Stream? patchOutStream = null)
         {
             var key = PatchKey(srcHash, destHash);
@@ -128,7 +128,7 @@ namespace Wabbajack.Common
         public static Task CreatePatchCached(byte[] a, byte[] b, Stream output) =>
             PatchCache.CreatePatchCached(a, b, output);
 
-        public static Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash, Stream? patchOutStream = null) =>
+        public static Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, Stream? patchOutStream = null) =>
             PatchCache.CreatePatchCached(srcStream, srcHash, destStream, destHash, patchOutStream);
 
         public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch) =>

Wabbajack.Lib/AInstaller.cs

@@ -9,6 +9,7 @@ using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.VirtualFileSystem;
+using Wabbajack.VirtualFileSystem.SevenZipExtractor;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
 using File = Alphaleonis.Win32.Filesystem.File;
 using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
@@ -35,6 +36,8 @@ namespace Wabbajack.Lib
         
         public bool UseCompression { get; set; }
 
+        public TempFolder? ExtractedModlistFolder { get; set; } = null;
+
 
         public AInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters? parameters, int steps, Game game)
             : base(steps)
@@ -45,12 +48,23 @@ namespace Wabbajack.Lib
             DownloadFolder = downloadFolder;
             SystemParameters = parameters;
             Game = game.MetaData();
+ 
         }
 
+        
         private ExtractedFiles? ExtractedModListFiles { get; set; } = null;
         public async Task ExtractModlist()
         {
-            ExtractedModListFiles = await FileExtractor.ExtractAll(Queue, ModListArchive);
+            ExtractedModlistFolder = await TempFolder.Create();
+            await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(ModListArchive), _ => true, 
+                async (path, sfn) =>
+                {
+                    await using var s = await sfn.GetStream();
+                    var fp = ExtractedModlistFolder.Dir.Combine(path);
+                    fp.Parent.CreateDirectory();
+                    await fp.WriteAllAsync(s);
+                    return 0; 
+                });
         }
 
 
@@ -73,8 +87,7 @@ namespace Wabbajack.Lib
 
         public async Task<byte[]> LoadBytesFromPath(RelativePath path)
         {
-            await using var e = await ExtractedModListFiles![path].OpenRead();
+            return await ExtractedModlistFolder!.Dir.Combine(path).ReadAllBytesAsync();
-            return await e.ReadAllAsync();
         }
 
         public static ModList LoadFromFile(AbsolutePath path)
@@ -113,23 +126,23 @@ namespace Wabbajack.Lib
 
         public async Task InstallArchives()
         {
-            await VFS.CopyTo(Queue, ModList.Directives
-                .OfType<FromArchive>()
-                .Select(a => (VFS.Index.FileForArchiveHashPath(a.ArchiveHashPath), a.To.RelativeTo(OutputFolder))));
-            /*
-            Info("Installing Archives");
-            Info("Grouping Install Files");
             var grouped = ModList.Directives
                 .OfType<FromArchive>()
-                .GroupBy(e => e.ArchiveHashPath.BaseHash)
+                .Select(a => new {VF = VFS.Index.FileForArchiveHashPath(a.ArchiveHashPath), Directive = a})
-                .ToDictionary(k => k.Key);
+                .GroupBy(a => a.VF)
-            var archives = ModList.Archives
+                .ToDictionary(a => a.Key);
-                .Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) })
-                .Where(a => a.AbsolutePath != null)
-                .ToList();
 
-            Info("Installing Archives");
+            if (grouped.Count == 0) return;
-            await archives.PMap(Queue, UpdateTracker,a => InstallArchive(Queue, a.Archive, a.AbsolutePath, grouped[a.Archive.Hash]));*/
+            
+            await VFS.Extract(Queue, grouped.Keys.ToHashSet(), async (vf, sf) =>
+            {
+                await using var s = await sf.GetStream();
+                foreach (var directive in grouped[vf])
+                {
+                    s.Position = 0;
+                    await directive.Directive.To.RelativeTo(OutputFolder).WriteAllAsync(s, false);
+                }
+            });
         }
 
         private async Task InstallArchive(WorkQueue queue, Archive archive, AbsolutePath absolutePath, IGrouping<Hash, FromArchive> grouping)

Wabbajack.Lib/MO2Compiler.cs

@@ -6,10 +6,12 @@ using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using AngleSharp.Common;
 using Wabbajack.Common;
 using Wabbajack.Lib.CompilationSteps;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.Validation;
+using Wabbajack.VirtualFileSystem;
 using Path = Alphaleonis.Win32.Filesystem.Path;
 
 namespace Wabbajack.Lib
@@ -495,17 +497,35 @@ namespace Wabbajack.Lib
                 .ToArray();
 
             if (toBuild.Length == 0) return;
+
+            // Extract all the source files
+            var indexed = toBuild.GroupBy(f => (VFS.Index.FileForArchiveHashPath(f.ArchiveHashPath)))
+                .ToDictionary(f => f.Key);
+            await VFS.Extract(Queue, indexed.Keys.ToHashSet(), 
+                async (vf, sf) =>
+            {
+                // For each, extract the destination
+                var matches = indexed[vf];
+                using var iqueue = new WorkQueue(1);
+                foreach (var match in matches)
+                {
+                    var destFile = FindDestFile(match.To);
+                    // Build the patch
+                    await VFS.Extract(iqueue, new[] {destFile}.ToHashSet(), 
+                        async (destvf, destsfn) =>
+                    {
+                        Info($"Patching {match.To}");
+                        Status($"Patching {match.To}");
+                        await using var srcStream = await sf.GetStream();
+                        await using var destStream = await destsfn.GetStream();
+                        var patchSize = await Utils.CreatePatchCached(srcStream, vf.Hash, destStream, destvf.Hash);
+                        Info($"Patch size {patchSize} for {match.To}");
+                    });
+                }
+
+            });
  
-            var groups = toBuild
+            // Load in the patches
-                .Where(p => p.PatchID == default)
-                .GroupBy(p => p.ArchiveHashPath.BaseHash)
-                .ToList();
-
-            Info($"Patching building patches from {groups.Count} archives");
-            var absolutePaths = AllFiles.ToDictionary(e => e.Path, e => e.AbsolutePath);
-            await groups.PMap(Queue, group => BuildArchivePatches(group.Key, group, absolutePaths));
-
-
             await InstallDirectives.OfType<PatchedFromArchive>()
                 .Where(p => p.PatchID == default)
                 .PMap(Queue, async pfa =>
@@ -514,6 +534,7 @@ namespace Wabbajack.Lib
                         .Select(c => (Utils.TryGetPatch(c.Hash, pfa.Hash, out var data), data, c))
                         .ToArray();
 
+                    // Pick the best patch
                     if (patches.All(p => p.Item1))
                     {
                         var (_, bytes, file) = IncludePatches.PickPatch(this, patches);
@@ -529,42 +550,19 @@ namespace Wabbajack.Lib
                 Error($"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}");
         }
 
-        private async Task BuildArchivePatches(Hash archiveSha, IEnumerable<PatchedFromArchive> group,
+        private VirtualFile FindDestFile(RelativePath to)
-            Dictionary<RelativePath, AbsolutePath> absolutePaths)
         {
-            await using var files = await VFS.StageWith(@group.Select(g => VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath)));
+            var abs = to.RelativeTo(MO2Folder);
-            var byPath = files.GroupBy(f => string.Join("|", f.FilesInFullPath.Skip(1).Select(i => i.Name)))
+            if (abs.Exists)
-                .ToDictionary(f => f.Key, f => f.First());
+                return VFS.Index.ByRootPath[abs];
-            // Now Create the patches
-            await @group.PMap(Queue, async entry =>
-            {
-                Info($"Patching {entry.To}");
-                Status($"Patching {entry.To}");
-                var srcFile = byPath[string.Join("|", entry.ArchiveHashPath.Paths)];
-                await using var srcStream = await srcFile.OpenRead();
-                await using var destStream = await LoadDataForTo(entry.To, absolutePaths);
-                var patchSize = await Utils.CreatePatchCached(srcStream, srcFile.Hash, destStream, entry.Hash);
-                Info($"Patch size {patchSize} for {entry.To}");
-            });
-        }
-
-        private async Task<FileStream> LoadDataForTo(RelativePath to, Dictionary<RelativePath, AbsolutePath> absolutePaths)
-        {
-            if (absolutePaths.TryGetValue(to, out var absolute))
-                return await absolute.OpenRead();
 
             if (to.StartsWith(Consts.BSACreationDir))
             {
                 var bsaId = (RelativePath)((string)to).Split('\\')[1];
                 var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsaId);
-
-                var a = await BSADispatch.OpenRead(MO2Folder.Combine(bsa.To));
                 var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray());
-                var file = a.Files.First(e => e.Path == find);
+
-                var returnStream = new TempStream();
+                return VFS.Index.ByRootPath[MO2Folder.Combine(bsa.To)].Children.First(c => c.RelativeName == find);
-                await file.CopyDataTo(returnStream);
-                returnStream.Position = 0;
-                return returnStream;
             }
 
             throw new ArgumentException($"Couldn't load data for {to}");

Wabbajack.Lib/MO2Installer.cs

@@ -180,6 +180,7 @@ namespace Wabbajack.Lib
             SetScreenSizeInPrefs();
 
             UpdateTracker.NextStep("Installation complete! You may exit the program.");
+            await ExtractedModlistFolder!.DisposeAsync();
             await Metrics.Send(Metrics.FinishInstall, ModList.Name);
 
             return true;
@@ -243,7 +244,8 @@ namespace Wabbajack.Lib
                        Status($"Writing included .meta file {directive.To}");
                        var outPath = DownloadFolder.Combine(directive.To);
                        if (outPath.IsFile) await outPath.DeleteAsync();
-                       await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID));
+                       var bytes = await LoadBytesFromPath(directive.SourceDataID);
+                       await outPath.WriteAllBytesAsync(bytes);
                    });
         }
 

Wabbajack.VirtualFileSystem/Context.cs

@@ -221,7 +221,7 @@ namespace Wabbajack.VirtualFileSystem
                         r => fileNames.ContainsKey(r),
                         async (rel, csf) =>
                         {
-                            await HandleFile(fileNames[rel], sfn);
+                            await HandleFile(fileNames[rel], csf);
                             return 0;
                         });
                 }
@@ -261,12 +261,6 @@ namespace Wabbajack.VirtualFileSystem
             };
         }
 
-        public async Task CopyTo(WorkQueue queue, IEnumerable<(VirtualFile src, AbsolutePath dest)> directives)
-        {
-            var plans = StagingPlan.StagingPlan.CreatePlan(directives).ToArray();
-            await StagingPlan.StagingPlan.ExecutePlans(queue, plans);
-        }
-
         public async Task<AsyncDisposableList<VirtualFile>> StageWith(IEnumerable<VirtualFile> files)
         {
             return new AsyncDisposableList<VirtualFile>(await Stage(files), files);

Wabbajack.VirtualFileSystem/FileExtractor.cs

@@ -30,6 +30,7 @@ namespace Wabbajack.VirtualFileSystem
 
         public static async Task<ExtractedFiles> ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable<RelativePath> OnlyFiles = null, bool throwOnError = true)
         {
+            throw new NotImplementedException();
             OnlyFiles ??= new RelativePath[0];
 
             try

Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Threading.Tasks;
+using Compression.BSA;
 using Wabbajack.Common;
 using Wabbajack.Common.FileSignatures;
 using Wabbajack.VirtualFileSystem.SevenZipExtractor;
@@ -14,7 +15,7 @@ namespace Wabbajack.VirtualFileSystem
             Definitions.FileType.BSA,
             Definitions.FileType.BA2,
             Definitions.FileType.ZIP,
-            Definitions.FileType.EXE,
+            //Definitions.FileType.EXE,
             Definitions.FileType.RAR,
             Definitions.FileType._7Z);
 
@@ -31,11 +32,33 @@ namespace Wabbajack.VirtualFileSystem
                 case Definitions.FileType.ZIP:
                     return await GatheringExtractWith7Zip<T>(archive, (Definitions.FileType)sig, shouldExtract, mapfn);
                 
+                case Definitions.FileType.TES3:
+                case Definitions.FileType.BSA:
+                case Definitions.FileType.BA2:
+                    return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
+                
+
                 default:
-                    throw new Exception("Invalid file format");
+                    throw new Exception($"Invalid file format {sFn.Name}");
             }
         }
 
+        private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithBSA<T>(IStreamFactory sFn, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
+        {
+            var archive = await BSADispatch.OpenRead(sFn, sig);
+            var results = new Dictionary<RelativePath, T>();
+            foreach (var entry in archive.Files)
+            {
+                if (!shouldExtract(entry.Path))
+                    continue;
+
+                var result = await mapfn(entry.Path, await entry.GetStreamFactory());
+                results.Add(entry.Path, result);
+            }
+
+            return results;
+        }
+
         private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
         {
             return await new GatheringExtractor<T>(stream, sig, shouldExtract, mapfn).Extract();

Wabbajack.VirtualFileSystem/FileExtractor2/StreamFactories.cs

@@ -5,13 +5,6 @@ using Wabbajack.Common;
 
 namespace Wabbajack.VirtualFileSystem
 {
-    public interface IStreamFactory
-    {
-        Task<Stream> GetStream();
-        
-        DateTime LastModifiedUtc { get; }
-        
-    }
 
     public class UnmanagedStreamFactory : IStreamFactory
     {
@@ -23,7 +16,7 @@ namespace Wabbajack.VirtualFileSystem
             _data = data;
             _size = size;
         }
-        public async Task<Stream> GetStream()
+        public async ValueTask<Stream> GetStream()
         {
             unsafe
             {
@@ -32,21 +25,8 @@ namespace Wabbajack.VirtualFileSystem
         }
 
         public DateTime LastModifiedUtc => DateTime.UtcNow;
+        public IPath Name => (RelativePath)"Unmanaged Memory Stream";
     }
 
-    public class NativeFileStreamFactory : IStreamFactory
-    {
-        private AbsolutePath _file;
 
-        public NativeFileStreamFactory(AbsolutePath file)
-        {
-            _file = file;
-        }
-        public async Task<Stream> GetStream()
-        {
-            return await _file.OpenRead();
-        }
-
-        public DateTime LastModifiedUtc => _file.LastModifiedUtc;
-    }
 }

Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs

@@ -1,25 +0,0 @@
-using System.Collections.Generic;
-using System.Threading.Tasks;
-using Wabbajack.Common;
-
-namespace Wabbajack.VirtualFileSystem.StagingPlan
-{
-    public abstract class ASubStage : ISubStage
-    {
-        private IEnumerable<IStagingPlan> _plans;
-
-        public ASubStage(IEnumerable<IStagingPlan> plans)
-        {
-            _plans = plans;
-        }
-        
-        public abstract ValueTask DisposeAsync();
-        public abstract AbsolutePath Destination { get; }
-        public abstract IPath Source { get; }
-        
-        public async Task Execute(WorkQueue queue)
-        {
-            await StagingPlan.ExecutePlan(queue, async () => await Destination.OpenWrite(), _plans);
-        }
-    }
-}

Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs

@@ -1,23 +0,0 @@
-using System.Threading.Tasks;
-using Wabbajack.Common;
-
-namespace Wabbajack.VirtualFileSystem.StagingPlan
-{
-    public class CopyTo : IStagingSrc
-    {
-        public CopyTo(IPath src, AbsolutePath destination)
-        {
-            Destination = destination;
-            Source = src;
-        }
-
-        public AbsolutePath Destination { get; }
-
-        public async ValueTask DisposeAsync()
-        {
-            
-        }
-
-        public IPath Source { get; }
-    }
-}

Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs

@@ -1,26 +0,0 @@
-using System.Threading.Tasks;
-using Wabbajack.Common;
-
-namespace Wabbajack.VirtualFileSystem.StagingPlan
-{
-    public class DuplicateTo : IStagingPlan
-    {
-        private readonly IStagingPlan _src;
-
-        public DuplicateTo(IStagingPlan src, AbsolutePath destination)
-        {
-            _src = src;
-            Destination = destination;
-        }
-        public async ValueTask DisposeAsync()
-        {
-        }
-
-        public async Task Execute()
-        {
-            await _src.Destination.CopyToAsync(Destination);
-        }
-
-        public AbsolutePath Destination { get; }
-    }
-}

Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs

@@ -1,156 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Threading.Tasks;
-using Wabbajack.Common;
-using Wabbajack.Common.FileSignatures;
-using Wabbajack.VirtualFileSystem.SevenZipExtractor;
-
-namespace Wabbajack.VirtualFileSystem.StagingPlan
-{
-    public interface IStagingPlan : IAsyncDisposable
-    {
-         AbsolutePath Destination { get; }
-    }
-
-    public static class StagingPlan
-    {
-
-        public static IEnumerable<IStagingPlan> CreatePlan(IEnumerable<(VirtualFile src, AbsolutePath dest)> directives)
-        {
-            var top = new VirtualFile();
-            var original = directives.ToHashSet();
-            var files = directives.GroupBy(f => f.src).ToDictionary(f => f.Key);
-
-            var childrenForParent = files.Keys.GroupBy(f => f.Parent ?? top).ToDictionary(f => f.Key);
-
-            var allFilesForParent = files.Select(f => f.Key)
-                .SelectMany(f => f.FilesInFullPath)
-                .Distinct()
-                .GroupBy(f => f.Parent ?? top)
-                .ToDictionary(f => f.Key, f => f.ToArray());
-
-            var baseDirectives = new List<IStagingPlan>();
-
-            // Recursive internal function to get the plans for a given parent, we'll then start at the 
-            // null parent (the top) and recurse our way down into all the children files.
-            IEnumerable<IStagingPlan> GetPlans(VirtualFile parent)
-            {
-                foreach (var forParent in allFilesForParent[parent])
-                {
-                    // Do we need files inside this file?
-                    if (childrenForParent.TryGetValue(forParent, out var children))
-                    {
-                        if (files.TryGetValue(forParent, out var copies))
-                        {
-                            ASubStage subStage;
-                            if (parent == top)
-                            {
-                                subStage = new NativeArchive(forParent.AbsoluteName, GetPlans(forParent));
-                            }
-                            else
-                            {
-                                subStage = new SubStage(forParent.RelativeName, copies.First().dest, GetPlans(forParent));
-                            }
-                            yield return subStage;
-                            foreach (var copy in copies)
-                            {
-                                yield return new DuplicateTo(subStage, copy.dest);
-                            }
-                        }
-                        else
-                        {
-                            if (parent == top)
-                            {
-                                yield return new NativeArchive(forParent.AbsoluteName, GetPlans(forParent));
-                            }
-                            else
-                            {
-                                yield return new TempSubStage(forParent.RelativeName, GetPlans(forParent));
-                            }
-                        }
-                    }
-                    else
-                    {
-                        // If not then we need to copy this file around
-                        var copies = files[forParent];
-                        var firstCopy = new CopyTo(copies.Key.Name, copies.First().dest);
-                        yield return firstCopy;
-                        
-                        foreach (var duplicate in copies.Skip(1))
-                        {
-                            yield return new DuplicateTo(firstCopy, duplicate.dest);
-                        }
-                    }
-                }
-            }
-
-            return GetPlans(top);
-        }
-        public static async ValueTask ExecutePlan(WorkQueue queue, Func<ValueTask<Stream>> src, IEnumerable<IStagingPlan> plans)
-        {
-            // Extract these files
-
-            await using var stream = await src();
-            var sig = await FileExtractor.ArchiveSigs.MatchesAsync(stream);
-            stream.Position = 0;
-
-            switch (sig)
-            {
-                case Definitions.FileType.ZIP:
-                    await ExtractWith7Zip((Definitions.FileType)sig, stream, plans);
-
-                    
-                    break;
-                
-                default:
-                    throw new Exception($"Invalid archive for extraction");
-            }
-            
-            
-            // Copy around the duplicates
-            foreach (var file in plans.OfType<DuplicateTo>())
-            {
-                await file.Execute();
-            }
-            
-            // Execute the sub-stages
-            foreach (var subStage in plans.OfType<ISubStage>())
-            {
-                await subStage.Execute(queue);
-            }
-
-            // Dispose of all plans
-            foreach (var file in plans)
-            {
-                await file.DisposeAsync();
-            }
-
-        }
-
-        private static async ValueTask ExtractWith7Zip(Definitions.FileType sig, Stream stream, IEnumerable<IStagingPlan> plans)
-        {
-            using var archive = await ArchiveFile.Open(stream, sig);
-
-            void HandleFile(RelativePath path, IStagingPlan src, Func<Stream> sf)
-            {
-                
-            }
-            
-            var extractor = new TypedExtractor<IStagingPlan>(plans.OfType<IStagingSrc>().ToDictionary(s => s.Source.FileName, s => (IStagingPlan)s),
-                HandleFile);
-            extractor.Extract(archive);
-
-        }
-
-
-        public static async Task ExecutePlans(WorkQueue queue, IEnumerable<IStagingPlan> plans)
-        {
-            foreach (var file in plans.OfType<NativeArchive>())
-            {
-                await file.Execute(queue);
-            }
-        }
-    }
-}

Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs

@@ -1,9 +0,0 @@
-using Wabbajack.Common;
-
-namespace Wabbajack.VirtualFileSystem.StagingPlan
-{
-    public interface IStagingSrc : IStagingPlan
-    {
-        public IPath Source { get; }
-    }
-}

Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs

@@ -1,10 +0,0 @@
-using System.Threading.Tasks;
-using Wabbajack.Common;
-
-namespace Wabbajack.VirtualFileSystem.StagingPlan
-{
-    public interface ISubStage : IStagingSrc
-    {
-        Task Execute(WorkQueue queue);
-    }
-}

Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs

@@ -1,22 +0,0 @@
-using System.Collections.Generic;
-using System.Threading.Tasks;
-using Wabbajack.Common;
-
-namespace Wabbajack.VirtualFileSystem.StagingPlan
-{
-    public class NativeArchive : ASubStage
-    {
-        public NativeArchive(AbsolutePath src, IEnumerable<IStagingPlan> plans) : base(plans)
-        {
-            Source = src;
-            Destination = src;
-        }
-
-        public override async ValueTask DisposeAsync()
-        {
-        }
-
-        public override AbsolutePath Destination { get; }
-        public override IPath Source { get; }
-    }
-}

Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs

@@ -1,21 +0,0 @@
-using System.Collections.Generic;
-using System.Threading.Tasks;
-using Wabbajack.Common;
-
-namespace Wabbajack.VirtualFileSystem.StagingPlan
-{
-    public class SubStage : ASubStage
-    {
-        public SubStage(RelativePath src, AbsolutePath destination, IEnumerable<IStagingPlan> plans) : base(plans)
-        {
-            Source = src;
-            Destination = destination;
-        }
-        
-        public override async ValueTask DisposeAsync()
-        {
-        }
-        public override AbsolutePath Destination { get; }
-        public override IPath Source { get; }
-    }
-}

Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs

@@ -1,25 +0,0 @@
-using System.Collections.Generic;
-using System.Threading.Tasks;
-using Wabbajack.Common;
-
-namespace Wabbajack.VirtualFileSystem.StagingPlan
-{
-    public class TempSubStage : ASubStage
-    {
-        private readonly TempFile _temp;
-
-        public TempSubStage(RelativePath src, IEnumerable<IStagingPlan> plans) : base(plans)
-        {
-            Source = src;
-            _temp = new TempFile();
-        }
-        
-        public override async ValueTask DisposeAsync()
-        {
-           await _temp.DisposeAsync();
-        }
-
-        public override AbsolutePath Destination => _temp.Path;
-        public override IPath Source { get; }
-    }
-}

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -141,7 +141,7 @@ namespace Wabbajack.VirtualFileSystem
                 itm.ThisAndAllChildrenReduced(fn);
         }
         
-        private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IExtractedFile extractedFile)
+        private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IStreamFactory extractedFile)
         {
             var vself = new VirtualFile
             {
@@ -161,7 +161,7 @@ namespace Wabbajack.VirtualFileSystem
             return vself;
         }
 
-        private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IExtractedFile extractedFile, Hash hash, out VirtualFile found)
+        private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile, Hash hash, out VirtualFile found)
         {
             var result = _vfsCache.Get(hash.ToArray());
             if (result == null)
@@ -199,11 +199,9 @@ namespace Wabbajack.VirtualFileSystem
 
             var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);
             
-                /* TODO
             if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
                 return vself;
-                */
+            
-
             var self = new VirtualFile
             {
                 Context = context,