Rework file extraction to combine the old and new methods

Compression.BSA.Test/BSATests.cs

@@ -83,7 +83,7 @@ namespace Compression.BSA.Test
             var folder = _bsaFolder.Combine(game.ToString(), modid.ToString());
             await folder.DeleteDirectory();
             folder.CreateDirectory();
-            await FileExtractor2.ExtractAll(filename, folder);
+            await FileExtractor2.ExtractAll(Queue, filename, folder);
 
             foreach (var bsa in folder.EnumerateFiles().Where(f => Consts.SupportedBSAs.Contains(f.Extension)))
             {

Wabbajack.Common/IStreamFactory.cs

@@ -16,7 +16,7 @@ namespace Wabbajack.Common
     }
     public class NativeFileStreamFactory : IStreamFactory
     {
-        private AbsolutePath _file;
+        protected AbsolutePath _file;
 
         public NativeFileStreamFactory(AbsolutePath file, IPath path)
         {

Wabbajack.Common/Paths/AbsolutePath.cs

@@ -171,7 +171,7 @@ namespace Wabbajack.Common
         /// </summary>
         /// <param name="otherPath"></param>
         /// <param name="overwrite">Replace the destination file if it exists</param>
-        public async Task MoveToAsync(AbsolutePath otherPath, bool overwrite = false)
+        public async ValueTask MoveToAsync(AbsolutePath otherPath, bool overwrite = false)
         {
             if (Root != otherPath.Root)
             {

Wabbajack.Common/Util/TempFile.cs

@@ -18,6 +18,11 @@ namespace Wabbajack.Common
             : this(new FileInfo((string)GetTempFilePath()))
         {
         }
+        
+        public TempFile(AbsolutePath path, bool deleteAfter = true, bool createFolder = true)
+            : this(new FileInfo((string)path))
+        {
+        }
 
         private static AbsolutePath GetTempFilePath()
         {

Wabbajack.Common/Util/TempFolder.cs

@@ -13,9 +13,11 @@ namespace Wabbajack.Common
         public bool DeleteAfter = true;
         private static Task _cleanTask;
 
+        public static AbsolutePath BaseFolder => AbsolutePath.EntryPoint.Combine("tmp_files");
+
         static TempFolder()
         {
-            _cleanTask = Task.Run(() => "tmp_files".RelativeTo(AbsolutePath.EntryPoint).DeleteDirectory());
+            _cleanTask = Task.Run(() => BaseFolder.DeleteDirectory());
         }
 
         /// <summary>
@@ -28,7 +30,7 @@ namespace Wabbajack.Common
 
         private TempFolder(bool deleteAfter = true)
         {
-            Dir = Path.Combine("tmp_files", Guid.NewGuid().ToString()).RelativeTo(AbsolutePath.EntryPoint);
+            Dir = BaseFolder.Combine(Guid.NewGuid().ToString());
             if (!Dir.Exists) 
                 Dir.CreateDirectory();
             DeleteAfter = deleteAfter;

Wabbajack.Common/Utils.cs

@@ -477,6 +477,7 @@ namespace Wabbajack.Common
 
             await Task.WhenAll(tasks);
         }
+        
 
         public static async Task PMap<TI>(this IEnumerable<TI> coll, WorkQueue queue, Action<TI> f)
         {

Wabbajack.Lib/AInstaller.cs

@@ -3,16 +3,10 @@ using System.Collections.Generic;
 using System.IO;
 using System.IO.Compression;
 using System.Linq;
-using System.Threading;
 using System.Threading.Tasks;
-using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.VirtualFileSystem;
-using Wabbajack.VirtualFileSystem.SevenZipExtractor;
-using Directory = Alphaleonis.Win32.Filesystem.Directory;
-using File = Alphaleonis.Win32.Filesystem.File;
-using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
 using Path = Alphaleonis.Win32.Filesystem.Path;
 
 namespace Wabbajack.Lib
@@ -54,19 +48,9 @@ namespace Wabbajack.Lib
         public async Task ExtractModlist()
         {
             ExtractedModlistFolder = await TempFolder.Create();
-            await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(ModListArchive), _ => true, 
+            await FileExtractor2.ExtractAll(Queue, ModListArchive, ExtractedModlistFolder.Dir);
-                async (path, sfn) =>
-                {
-                    await using var s = await sfn.GetStream();
-                    var fp = ExtractedModlistFolder.Dir.Combine(path);
-                    fp.Parent.CreateDirectory();
-                    await fp.WriteAllAsync(s);
-                    return 0; 
-                });
         }
 
-
-
         public void Info(string msg)
         {
             Utils.Log(msg);
@@ -138,16 +122,16 @@ namespace Wabbajack.Lib
             
             await VFS.Extract(Queue, grouped.Keys.ToHashSet(), async (vf, sf) =>
             {
-                await using var s = await sf.GetStream();
                 foreach (var directive in grouped[vf])
                 {
                     var file = directive.Directive;
-                    s.Position = 0;
 
                     switch (file)
                     {
                         case PatchedFromArchive pfa:
                         {
+                            await using var s = await sf.GetStream();
+                            s.Position = 0;
                             var patchData = await LoadBytesFromPath(pfa.PatchID);
                             var toFile = file.To.RelativeTo(OutputFolder);
                             {
@@ -165,7 +149,16 @@ namespace Wabbajack.Lib
                             break;
 
                         case FromArchive _:
-                            await directive.Directive.To.RelativeTo(OutputFolder).WriteAllAsync(s, false);
+                            if (grouped[vf].Count() == 1)
+                            {
+                                await sf.Move(directive.Directive.To.RelativeTo(OutputFolder));
+                            }
+                            else
+                            {
+                                await using var s = await sf.GetStream();
+                                await directive.Directive.To.RelativeTo(OutputFolder).WriteAllAsync(s, false);
+                            }
+
                             break;
                         default:
                             throw new Exception($"No handler for {directive}");
@@ -188,7 +181,7 @@ namespace Wabbajack.Lib
                         await file.To.RelativeTo(OutputFolder).Compact(FileCompaction.Algorithm.XPRESS16K);
                     }
                 }
-            });
+            }, tempFolder: OutputFolder);
         }
 
         public async Task DownloadArchives()

Wabbajack.Test/EndToEndTests.cs

@@ -111,7 +111,7 @@ namespace Wabbajack.Test
             await src.CopyToAsync(destFile);
 
             var modFolder = modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName);
-            await FileExtractor2.ExtractAll(src, modFolder);
+            await FileExtractor2.ExtractAll(Queue, src, modFolder);
             return (destFile, modFolder);
         }
 
@@ -146,7 +146,7 @@ namespace Wabbajack.Test
             await src.CopyToAsync(dest);
 
             var modFolder = utils.ModsFolder.Combine(modName);
-            await FileExtractor2.ExtractAll(src, modFolder);
+            await FileExtractor2.ExtractAll(Queue, src, modFolder);
             
             await dest.WithExtension(Consts.MetaFileExtension).WriteAllTextAsync(ini);
             return (dest, modFolder);

Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs

@@ -15,6 +15,7 @@ namespace Wabbajack.VirtualFileSystem.Test
     {
         private ITestOutputHelper _helper;
         private IDisposable _unsub;
+        private WorkQueue _queue;
 
         public FileExtractorTests(ITestOutputHelper helper)
         {
@@ -30,6 +31,7 @@ namespace Wabbajack.VirtualFileSystem.Test
                     // ignored
                 }
             });
+            _queue = new WorkQueue();
         }
 
         public async Task InitializeAsync()
@@ -53,7 +55,7 @@ namespace Wabbajack.VirtualFileSystem.Test
 
             await ZipUpFolder(temp.Dir, archive.Path, false);
             
-            var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path), 
+            var results = await FileExtractor2.GatheringExtract(_queue, new NativeFileStreamFactory(archive.Path), 
                 _ => true,
                 async (path, sfn) =>
                 {
@@ -80,7 +82,7 @@ namespace Wabbajack.VirtualFileSystem.Test
 
             await ZipUpFolder(temp.Dir, archive.Path, false);
 
-            var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path), 
+            var results = await FileExtractor2.GatheringExtract(_queue, new NativeFileStreamFactory(archive.Path), 
                 _ => true,
                 async (path, sfn) =>
                 {
@@ -131,7 +133,7 @@ namespace Wabbajack.VirtualFileSystem.Test
 
             await ZipUpFolder(temp.Dir, archive.Path, false);
             
-            var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path), 
+            var results = await FileExtractor2.GatheringExtract(_queue, new NativeFileStreamFactory(archive.Path), 
                 _ => true,
                 async (path, sfn) =>
                 {
@@ -154,10 +156,10 @@ namespace Wabbajack.VirtualFileSystem.Test
         {
             var src = await DownloadMod(Game.Oblivion, 18498);
 
-            await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(src),
+            await FileExtractor2.GatheringExtract(_queue, new NativeFileStreamFactory(src),
                 p => p.Extension == OMODExtension, async (path, sfn) =>
                 {
-                    await FileExtractor2.GatheringExtract(sfn, _ => true, async (ipath, isfn) => {
+                    await FileExtractor2.GatheringExtract(_queue, sfn, _ => true, async (ipath, isfn) => {
                         // We shouldn't have any .crc files because this file should be recognized as a OMOD and extracted correctly
                         Assert.NotEqual(CRCExtension, ipath.Extension);
                         return 0;
@@ -171,7 +173,7 @@ namespace Wabbajack.VirtualFileSystem.Test
         {
             var src = await DownloadMod(Game.Fallout4, 29596, 120918);
             await using var tmpFolder = await TempFolder.Create();
-            await FileExtractor2.ExtractAll(src, tmpFolder.Dir);
+            await FileExtractor2.ExtractAll(_queue, src, tmpFolder.Dir);
         }
 
 

Wabbajack.VirtualFileSystem/Context.cs

@@ -9,6 +9,7 @@ using System.Text;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
+using Wabbajack.VirtualFileSystem.ExtractedFiles;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
 using File = System.IO.File;
 using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
@@ -206,7 +207,7 @@ namespace Wabbajack.VirtualFileSystem
         /// <param name="files"></param>
         /// <param name="callback"></param>
         /// <returns></returns>
-        public async Task Extract(WorkQueue queue, HashSet<VirtualFile> files, Func<VirtualFile, IStreamFactory, ValueTask> callback)
+        public async Task Extract(WorkQueue queue, HashSet<VirtualFile> files, Func<VirtualFile, IExtractedFile, ValueTask> callback, AbsolutePath? tempFolder = null)
         {
             var top = new VirtualFile();
             var filesByParent = files.SelectMany(f => f.FilesInFullPath)
@@ -214,23 +215,28 @@ namespace Wabbajack.VirtualFileSystem
                 .GroupBy(f => f.Parent ?? top)
                 .ToDictionary(f => f.Key);
 
-            async Task HandleFile(VirtualFile file, IStreamFactory sfn)
+            async Task HandleFile(VirtualFile file, IExtractedFile sfn)
             {
+                if (filesByParent.ContainsKey(file))
+                    sfn.CanMove = false;
+                
                 if (files.Contains(file)) await callback(file, sfn);
                 if (filesByParent.TryGetValue(file, out var children))
                 {
                     var fileNames = children.ToDictionary(c => c.RelativeName);
-                    await FileExtractor2.GatheringExtract(sfn,
+                    await FileExtractor2.GatheringExtract(queue, sfn,
                         r => fileNames.ContainsKey(r),
                         async (rel, csf) =>
                         {
                             await HandleFile(fileNames[rel], csf);
                             return 0;
-                        });
+                        },
+                        tempFolder: tempFolder,
+                        onlyFiles: fileNames.Keys.ToHashSet());
                 }
 
             }
-            await filesByParent[top].PMap(queue, async file => await HandleFile(file, new NativeFileStreamFactory(file.AbsoluteName)));
+            await filesByParent[top].PMap(queue, async file => await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName)));
         }
 
         #region KnownFiles

Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/ExtractedMemoryfile.cs

@@ -0,0 +1,33 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem.ExtractedFiles
+{
+    public class ExtractedMemoryFile : IExtractedFile
+    {
+        private IStreamFactory _factory;
+
+        public ExtractedMemoryFile(IStreamFactory factory)
+        {
+            _factory = factory;
+        }
+
+
+        public ValueTask<Stream> GetStream()
+        {
+            return _factory.GetStream();
+        }
+
+        public DateTime LastModifiedUtc => _factory.LastModifiedUtc;
+        public IPath Name => _factory.Name;
+        public async ValueTask Move(AbsolutePath newPath)
+        {
+            await using var stream = await _factory.GetStream();
+            await newPath.WriteAllAsync(stream);
+        }
+
+        public bool CanMove { get; set; } = true;
+    }
+}

Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/ExtractedNativeFile.cs

@@ -0,0 +1,26 @@
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem.ExtractedFiles
+{
+    public class ExtractedNativeFile : NativeFileStreamFactory, IExtractedFile
+    {
+        public bool CanMove { get; set; } = true;
+
+        public ExtractedNativeFile(AbsolutePath file, IPath path) : base(file, path)
+        {
+        }
+
+        public ExtractedNativeFile(AbsolutePath file) : base(file)
+        {
+        }
+
+        public async ValueTask Move(AbsolutePath newPath)
+        {
+            if (CanMove) 
+                await _file.MoveToAsync(newPath, overwrite: true);
+            else 
+                await _file.CopyToAsync(newPath);
+        }
+    }
+}

Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/IExtractedFile.cs

@@ -0,0 +1,19 @@
+using System.IO;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem.ExtractedFiles
+{
+    public interface IExtractedFile : IStreamFactory
+    {
+        /// <summary>
+        /// Possibly destructive move operation. Should greatly optimize file copies when the file
+        /// exists on the same disk as the newPath. Performs a copy if a move is not possible.
+        /// </summary>
+        /// <param name="newPath">destination to move the entry to</param>
+        /// <returns></returns>
+        public ValueTask Move(AbsolutePath newPath);
+        
+        public bool CanMove { get; set; }
+    }
+}

Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs

@@ -1,15 +1,16 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
+using System.Reactive.Linq;
 using System.Threading.Tasks;
 using Compression.BSA;
-using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
 using OMODFramework;
-using SharpCompress.Archives.SevenZip;
-using SharpCompress.Readers;
 using Wabbajack.Common;
 using Wabbajack.Common.FileSignatures;
-using Wabbajack.VirtualFileSystem.SevenZipExtractor;
+using Wabbajack.Common.StatusFeed;
+using Wabbajack.Common.StatusFeed.Errors;
+using Wabbajack.VirtualFileSystem.ExtractedFiles;
 using Utils = Wabbajack.Common.Utils;
 
 namespace Wabbajack.VirtualFileSystem
@@ -47,9 +48,14 @@ namespace Wabbajack.VirtualFileSystem
         public static bool FavorPerfOverRAM { get; set; }
 
 
-        public static async Task<Dictionary<RelativePath, T>> GatheringExtract<T>(IStreamFactory sFn,
+        public static async Task<Dictionary<RelativePath, T>> GatheringExtract<T>(WorkQueue queue, IStreamFactory sFn,
-            Predicate<RelativePath> shouldExtract, Func<RelativePath, IStreamFactory, ValueTask<T>> mapfn)
+            Predicate<RelativePath> shouldExtract, Func<RelativePath, IExtractedFile, ValueTask<T>> mapfn,
+            AbsolutePath? tempFolder = null,
+            HashSet<RelativePath> onlyFiles = null)
         {
+            if (tempFolder == null)
+                tempFolder = TempFolder.BaseFolder;
+            
             if (sFn is NativeFileStreamFactory)
             {
                 Utils.Log($"Extracting {sFn.Name}");
@@ -58,6 +64,8 @@ namespace Wabbajack.VirtualFileSystem
             var sig = await ArchiveSigs.MatchesAsync(archive);
             archive.Position = 0;
 
+            Dictionary<RelativePath, T> results = new Dictionary<RelativePath, T>();
+
             switch (sig)
             {
                 case Definitions.FileType.RAR_OLD:
@@ -67,33 +75,42 @@ namespace Wabbajack.VirtualFileSystem
                 {
                     if (sFn.Name.FileName.Extension == OMODExtension)
                     {
-                        return await GatheringExtractWithOMOD(archive, shouldExtract, mapfn);
+                        results = await GatheringExtractWithOMOD(archive, shouldExtract, mapfn);
 
                     }
                     else
                     {
-                        return await GatheringExtractWith7Zip<T>(sFn, (Definitions.FileType)sig, shouldExtract,
+                        results = await GatheringExtractWith7Zip<T>(queue, sFn, (Definitions.FileType)sig, shouldExtract,
-                            mapfn);
+                            mapfn, tempFolder.Value, onlyFiles);
                     }
+
+                    break;
                 }
 
                 case Definitions.FileType.BSA:
                 case Definitions.FileType.BA2:
-                    return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
+                    results = await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
+                    break;
 
                 case Definitions.FileType.TES3:
                     if (sFn.Name.FileName.Extension == BSAExtension) 
-                        return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
+                        results = await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
                     else
                         throw new Exception($"Invalid file format {sFn.Name}");
-
+                    break;
-
                 default:
                     throw new Exception($"Invalid file format {sFn.Name}");
             }
+
+            if (onlyFiles != null && onlyFiles.Count != results.Count)
+            {
+                throw new Exception(
+                    $"Sanity check error extracting {sFn.Name} - {results.Count} results, expected {onlyFiles.Count}");
+            }
+            return results;
         }
 
-        private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithOMOD<T>(Stream archive, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
+        private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithOMOD<T>(Stream archive, Predicate<RelativePath> shouldExtract, Func<RelativePath,IExtractedFile,ValueTask<T>> mapfn)
         {
             var tmpFile = new TempFile();
             await tmpFile.Path.WriteAllAsync(archive);
@@ -113,7 +130,7 @@ namespace Wabbajack.VirtualFileSystem
                 var path = file.RelativeTo(dest.Dir);
                 if (!shouldExtract(path)) continue;
 
-                var result = await mapfn(path, new NativeFileStreamFactory(file, path));
+                var result = await mapfn(path, new ExtractedNativeFile(file, path));
                 results.Add(path, result);
             }
 
@@ -141,7 +158,7 @@ namespace Wabbajack.VirtualFileSystem
         }
 
 
-        private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithBSA<T>(IStreamFactory sFn, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
+        private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithBSA<T>(IStreamFactory sFn, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IExtractedFile,ValueTask<T>> mapfn)
         {
             var archive = await BSADispatch.OpenRead(sFn, sig);
             var results = new Dictionary<RelativePath, T>();
@@ -150,21 +167,123 @@ namespace Wabbajack.VirtualFileSystem
                 if (!shouldExtract(entry.Path))
                     continue;
 
-                var result = await mapfn(entry.Path, await entry.GetStreamFactory());
+                var result = await mapfn(entry.Path, new ExtractedMemoryFile(await entry.GetStreamFactory()));
                 results.Add(entry.Path, result);
             }
 
             return results;
         }
 
-        private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(IStreamFactory sf, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
+        private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(WorkQueue queue, IStreamFactory sf, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IExtractedFile,ValueTask<T>> mapfn,
+            AbsolutePath tempPath, HashSet<RelativePath> onlyFiles)
         {
-            return await new GatheringExtractor<T>(sf, sig, shouldExtract, mapfn).Extract();
+            TempFile tmpFile = null;
+            var dest = tempPath.Combine(Guid.NewGuid().ToString());
+            dest.CreateDirectory();
+            
+            TempFile spoolFile = null;
+            AbsolutePath source;
+
+            try
+            {
+                if (sf.Name is AbsolutePath abs)
+                {
+                    source = abs;
+                }
+                else
+                {
+                    spoolFile = new TempFile(tempPath.Combine(Guid.NewGuid().ToString())
+                        .WithExtension(source.Extension));
+                    await using var s = await sf.GetStream();
+                    await spoolFile.Path.WriteAllAsync(s);
+                }
+
+                Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}",
+                    $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe"));
+
+                var process = new ProcessHelper {Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),};
+
+                if (onlyFiles != null)
+                {
+                    //It's stupid that we have to do this, but 7zip's file pattern matching isn't very fuzzy
+                    IEnumerable<string> AllVariants(string input)
+                    {
+                        yield return $"\"{input}\"";
+                        yield return $"\"\\{input}\"";
+                    }
+
+                    tmpFile = new TempFile();
+                    await tmpFile.Path.WriteAllLinesAsync(onlyFiles.SelectMany(f => AllVariants((string)f)).ToArray());
+                    process.Arguments = new object[]
+                    {
+                        "x", "-bsp1", "-y", $"-o\"{dest}\"", source, $"@\"{tmpFile.Path}\"", "-mmt=off"
+                    };
+                }
+                else
+                {
+                    process.Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest}\"", source, "-mmt=off"};
+                }
+
+
+                var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
+                    .ForEachAsync(p =>
+                    {
+                        var (_, line) = p;
+                        if (line == null)
+                            return;
+
+                        if (line.Length <= 4 || line[3] != '%') return;
+
+                        int.TryParse(line.Substring(0, 3), out var percentInt);
+                        Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}",
+                            Percent.FactoryPutInRange(percentInt / 100d));
+                    });
+
+                var exitCode = await process.Start();
+
+
+                if (exitCode != 0)
+                {
+                    Utils.ErrorThrow(new _7zipReturnError(exitCode, source, dest, ""));
+                }
+                else
+                {
+                    Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true);
+                }
+
+                var results = await dest.EnumerateFiles()
+                    .PMap(queue, async f =>
+                    {
+                        var path = f.RelativeTo(dest);
+                        if (!shouldExtract(path)) return ((RelativePath, T))default;
+                        var file = new ExtractedNativeFile(f);
+                        var result = await mapfn(path, file);
+                        await f.DeleteAsync();
+                        return (path, result);
+                    });
+                
+                return results.Where(d => d.Item1 != default)
+                    .ToDictionary(d => d.Item1, d => d.Item2);
+            }
+            finally
+            {
+                await dest.DeleteDirectory();
+                
+                if (tmpFile != null)
+                {
+                    await tmpFile.DisposeAsync();
+                }
+
+                if (spoolFile != null)
+                {
+                    await spoolFile.DisposeAsync();
+                }
+            }
         }
 
-        public static async Task ExtractAll(AbsolutePath src, AbsolutePath dest)
+        public static async Task ExtractAll(WorkQueue queue, AbsolutePath src, AbsolutePath dest)
         {
-            await GatheringExtract(new NativeFileStreamFactory(src), _ => true, async (path, factory) =>
+            await GatheringExtract(queue, new NativeFileStreamFactory(src), _ => true, async (path, factory) =>
             {
                 var abs = path.RelativeTo(dest);
                 abs.Parent.CreateDirectory();

Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs

@@ -1,329 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Reactive.Linq;
-using System.Runtime.InteropServices;
-using System.Threading;
-using System.Threading.Tasks;
-using Compression.BSA;
-using Wabbajack.Common;
-using Wabbajack.Common.FileSignatures;
-using Wabbajack.Common.StatusFeed.Errors;
-using Wabbajack.VirtualFileSystem.SevenZipExtractor;
-
-namespace Wabbajack.VirtualFileSystem
-{
-    public class GatheringExtractor<T> : IArchiveExtractCallback
-    {
-        private ArchiveFile _archive;
-        private Predicate<RelativePath> _shouldExtract;
-        private Func<RelativePath, IStreamFactory, ValueTask<T>> _mapFn;
-        private Dictionary<RelativePath, T> _results;
-        private Definitions.FileType _sig;
-        private Exception _killException;
-        private uint _itemsCount;
-        private IStreamFactory _streamFactory;
-
-        public GatheringExtractor(IStreamFactory sF, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory, ValueTask<T>> mapfn)
-        {
-            _shouldExtract = shouldExtract;
-            _mapFn = mapfn;
-            _results = new Dictionary<RelativePath, T>();
-            _streamFactory = sF;
-            _sig = sig;
-        }
-
-        public async Task<Dictionary<RelativePath, T>> Extract()
-        {
-            var source = new TaskCompletionSource<bool>();
-
-            var th = new Thread(() =>
-            {
-                try
-                {
-                    using var stream = _streamFactory.GetStream().Result;
-                    _archive = ArchiveFile.Open(stream, _sig).Result;
-                    ulong checkPos = (ulong)stream.Length;
-                    var oresult = _archive._archive.Open(_archive._archiveStream, ref checkPos, new ArchiveCallback());
-                    // Can't read this with the COM interface for some reason
-                    if (oresult != 0)
-                    {
-                        var _ = ExtractSlow(source, _streamFactory);
-                        return;
-                    }
-                    _itemsCount = _archive._archive.GetNumberOfItems();
-                    var result = _archive._archive.Extract(null, 0xFFFFFFFF, 0, this);
-                    _archive.Dispose();
-                    if (_killException != null)
-                    {
-                        source.SetException(_killException);
-                    }
-                    else
-                    {
-                        source.SetResult(true);
-                    }
-                }
-                catch (Exception ex)
-                {
-                    source.SetException(ex);
-                }
-                
-            }) {Priority = ThreadPriority.BelowNormal, Name = "7Zip Extraction Worker Thread"};
-            th.Start();
-            
-            
-
-            await source.Task;
-            return _results;
-        }
-
-        private async Task ExtractSlow(TaskCompletionSource<bool> tcs, IStreamFactory streamFactory)
-        {
-            try
-            {
-                TempFile tempFile = null;
-                AbsolutePath source;
-                if (streamFactory is NativeFileStreamFactory nsf)
-                {
-                    source = (AbsolutePath)nsf.Name;
-                }
-                else
-                {
-                    await using var stream = await streamFactory.GetStream();
-                    tempFile = new TempFile();
-                    await tempFile.Path.WriteAllAsync(stream);
-                }
-
-                var dest = await TempFolder.Create();
-                Utils.Log(
-                    $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe");
-
-                var process = new ProcessHelper {Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),};
-
-
-                process.Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, "-mmt=off"};
-
-
-                var _ = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
-                    .ForEachAsync(p =>
-                    {
-                        var (_, line) = p;
-                        if (line == null)
-                            return;
-
-                        if (line.Length <= 4 || line[3] != '%') return;
-
-                        int.TryParse(line.Substring(0, 3), out var percentInt);
-                        Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}",
-                            Percent.FactoryPutInRange(percentInt / 100d));
-                    });
-
-                var exitCode = await process.Start();
-
-
-                if (exitCode != 0)
-                {
-                    Utils.ErrorThrow(new _7zipReturnError(exitCode, source, dest.Dir, ""));
-                }
-                else
-                {
-                    Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true);
-                }
-
-                if (tempFile != null)
-                {
-                    await tempFile.DisposeAsync();
-                }
-
-                foreach (var file in dest.Dir.EnumerateFiles())
-                {
-                    var relPath = file.RelativeTo(dest.Dir);
-                    if (!_shouldExtract(relPath)) continue;
-
-                    var result = await _mapFn(relPath, new NativeFileStreamFactory(file));
-                    _results[relPath] = result;
-                    await file.DeleteAsync();
-                }
-
-                tcs.SetResult(true);
-            }
-            catch (Exception ex)
-            {
-                tcs.SetException(ex);
-            }
-        }       
-
-        public void SetTotal(ulong total)
-        {
-            
-        }
-
-        public void SetCompleted(ref ulong completeValue)
-        {
-            
-        }
-
-        public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode)
-        {
-            var entry = _archive.GetEntry(index);
-            var path = (RelativePath)entry.FileName;
-            if (entry.IsFolder || !_shouldExtract(path))
-            {
-                outStream = null;
-                return 0;
-            }
-
-            Utils.Status($"Extracting {path}", Percent.FactoryPutInRange(_results.Count, _itemsCount));
-            // Empty files are never extracted via a write call, so we have to fake that now
-            if (entry.Size == 0)
-            {
-                var result = _mapFn(path, new MemoryStreamFactory(new MemoryStream(), path)).Result;
-                _results.Add(path, result);
-            }
-            outStream =  new GatheringExtractorStream<T>(this, entry, path);
-            return 0;
-        }
-
-        public void PrepareOperation(AskMode askExtractMode)
-        {
-            
-        }
-
-        public void SetOperationResult(OperationResult resultEOperationResult)
-        {
-            
-        }
-
-        private class GatheringExtractorStream<T> : ISequentialOutStream, IOutStream
-        {
-            private GatheringExtractor<T> _extractor;
-            private ulong _totalSize;
-            private Stream _tmpStream;
-            private TempFile _tmpFile;
-            private bool _diskCached;
-            private RelativePath _path;
-
-            public GatheringExtractorStream(GatheringExtractor<T> extractor, Entry entry, RelativePath path)
-            {
-                _path = path;
-                _extractor = extractor;
-                _totalSize = entry.Size;
-                _diskCached = _totalSize >= int.MaxValue - 1024;
-            }
-
-            private IPath GetPath()
-            {
-                return _path;
-            }
-
-            public int Write(byte[] data, uint size, IntPtr processedSize)
-            {
-                try
-                {
-                    if (size == _totalSize)
-                        WriteSingleCall(data, size);
-                    else if (_diskCached)
-                        WriteDiskCached(data, size);
-                    else
-                        WriteMemoryCached(data, size);
-
-                    if (processedSize != IntPtr.Zero)
-                    {
-                        Marshal.WriteInt32(processedSize, (int)size);
-                    }
-                    
-                    return 0;
-                }
-                catch (Exception ex)
-                {
-                    Utils.Log($"Error during extraction {ex}");
-                    _extractor.Kill(ex);
-
-                    return 1;
-                }
-            }
-
-            private void WriteSingleCall(byte[] data, in uint size)
-            {
-                var result = _extractor._mapFn(_path, new MemoryBufferFactory(data, (int)size, GetPath())).Result;
-                AddResult(result);
-                Cleanup();
-            }
-
-            private void Cleanup()
-            {
-                _tmpStream?.Dispose();
-                _tmpFile?.DisposeAsync().AsTask().Wait();
-            }
-
-            private void AddResult(T result)
-            {
-                _extractor._results.Add(_path, result);
-            }
-
-            private void WriteMemoryCached(byte[] data, in uint size)
-            {
-                if (_tmpStream == null)
-                    _tmpStream = new MemoryStream();
-                _tmpStream.Write(data, 0, (int)size);
-
-                if (_tmpStream.Length != (long)_totalSize) return;
-
-                _tmpStream.Flush();
-                _tmpStream.Position = 0;
-                var result = _extractor._mapFn(_path, new MemoryStreamFactory((MemoryStream)_tmpStream, GetPath())).Result;
-                AddResult(result);
-                Cleanup();
-            }
-
-            private void WriteDiskCached(byte[] data, in uint size)
-            {
-                if (_tmpFile == null)
-                {
-                    _tmpFile = new TempFile();
-                    _tmpStream = _tmpFile.Path.Create().Result;
-                }
-                
-                _tmpStream.Write(data, 0, (int)size);
-
-                if (_tmpStream.Length != (long)_totalSize) return;
-
-                _tmpStream.Flush();
-                _tmpStream.Close();
-                
-                var result = _extractor._mapFn(_path, new NativeFileStreamFactory(_tmpFile.Path, GetPath())).Result;
-                AddResult(result);
-                Cleanup();
-            }
-
-            public void Seek(long offset, uint seekOrigin, IntPtr newPosition)
-            {
-                
-            }
-
-            public int SetSize(long newSize)
-            {
-                return 0;
-            }
-        }
-
-        private void Kill(Exception ex)
-        {
-            _killException = ex;
-        }
-
-        class ArchiveCallback : IArchiveOpenCallback
-        {
-            public void SetTotal(IntPtr files, IntPtr bytes)
-            {
-                
-            }
-
-            public void SetCompleted(IntPtr files, IntPtr bytes)
-            {
-                
-            }
-        }
-    }
-}

Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs

@@ -1,255 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Globalization;
-using System.IO;
-using System.Linq;
-using System.Runtime.InteropServices;
-using System.Threading.Tasks;
-using Wabbajack.Common;
-using Wabbajack.Common.FileSignatures;
-
-namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
-{
-    public class ArchiveFile : IDisposable
-    {
-        private SevenZipHandle _sevenZipHandle;
-        internal IInArchive _archive;
-        public InStreamWrapper _archiveStream;
-        private IList<Entry> _entries;
-
-        private static readonly AbsolutePath LibraryFilePath = @"Extractors\7z.dll".RelativeTo(AbsolutePath.EntryPoint);
-        private static SignatureChecker _checker = new SignatureChecker(Formats.FileTypeGuidMapping.Keys.ToArray());
-
-        public static async Task<ArchiveFile> Open(Stream archiveStream, Definitions.FileType format)
-        {
-            var self = new ArchiveFile();
-            self.InitializeAndValidateLibrary();
-            self._archive = self._sevenZipHandle.CreateInArchive(Formats.FileTypeGuidMapping[format]);
-
-            if (!FileExtractor2.FavorPerfOverRAM)
-            {
-                self.SetCompressionProperties(new Dictionary<string, string>() {{"mt", "off"}});
-
-            }
-
-
-            self._archiveStream = new InStreamWrapper(archiveStream);
-            return self;
-        }
-
-        /// <summary>
-        /// Sets the compression properties
-        /// </summary>
-        private void SetCompressionProperties(Dictionary<string, string> CustomParameters)
-        {
-
-
-            {
-                ISetProperties setter;
-                try
-                {
-                    setter = (ISetProperties)_archive;
-                }
-                catch (InvalidCastException)
-                {
-                    return;
-                }
-
-                var names = new List<IntPtr>(1 + CustomParameters.Count);
-                var values = new List<PropVariant>(1 + CustomParameters.Count);
-                //var sp = new SecurityPermission(SecurityPermissionFlag.UnmanagedCode);
-                //sp.Demand();
-
-                #region Initialize compression properties
-
-                names.Add(Marshal.StringToBSTR("x"));
-                values.Add(new PropVariant());
-
-                foreach (var pair in CustomParameters)
-                {
-
-                    names.Add(Marshal.StringToBSTR(pair.Key));
-                    var pv = new PropVariant();
-
-                    #region List of parameters to cast as integers
-
-                    var integerParameters = new HashSet<string>
-                    {
-                        "fb",
-                        "pass",
-                        "o",
-                        "yx",
-                        "a",
-                        "mc",
-                        "lc",
-                        "lp",
-                        "pb",
-                        "cp"
-                    };
-
-                    #endregion
-
-                    if (integerParameters.Contains(pair.Key))
-                    {
-                        pv.VarType = VarEnum.VT_UI4;
-                        pv.UInt32Value = Convert.ToUInt32(pair.Value, CultureInfo.InvariantCulture);
-                    }
-                    else
-                    {
-                        pv.VarType = VarEnum.VT_BSTR;
-                        pv.pointerValue = Marshal.StringToBSTR(pair.Value);
-                    }
-
-                    values.Add(pv);
-                }
-
-                #endregion
-
-                #region Set compression level
-
-                var clpv = values[0];
-                clpv.VarType = VarEnum.VT_UI4;
-                clpv.UInt32Value = 0;
-
-                values[0] = clpv;
-
-                #endregion
-
-                var namesHandle = GCHandle.Alloc(names.ToArray(), GCHandleType.Pinned);
-                var valuesHandle = GCHandle.Alloc(values.ToArray(), GCHandleType.Pinned);
-
-                try
-                {
-                    setter?.SetProperties(namesHandle.AddrOfPinnedObject(), valuesHandle.AddrOfPinnedObject(),
-                        names.Count);
-                }
-                finally
-                {
-                    namesHandle.Free();
-                    valuesHandle.Free();
-                }
-
-            }
-        }
-        internal Entry GetEntry(uint fileIndex)
-        {
-            string fileName = this.GetProperty<string>(fileIndex, ItemPropId.kpidPath);
-            bool isFolder = this.GetProperty<bool>(fileIndex, ItemPropId.kpidIsFolder);
-            bool isEncrypted = this.GetProperty<bool>(fileIndex, ItemPropId.kpidEncrypted);
-            ulong size = this.GetProperty<ulong>(fileIndex, ItemPropId.kpidSize);
-            ulong packedSize = this.GetProperty<ulong>(fileIndex, ItemPropId.kpidPackedSize);
-            DateTime creationTime = this.GetPropertySafe<DateTime>(fileIndex, ItemPropId.kpidCreationTime);
-            DateTime lastWriteTime = this.GetPropertySafe<DateTime>(fileIndex, ItemPropId.kpidLastWriteTime);
-            DateTime lastAccessTime = this.GetPropertySafe<DateTime>(fileIndex, ItemPropId.kpidLastAccessTime);
-            uint crc = this.GetPropertySafe<uint>(fileIndex, ItemPropId.kpidCRC);
-            uint attributes = this.GetPropertySafe<uint>(fileIndex, ItemPropId.kpidAttributes);
-            string comment = this.GetPropertySafe<string>(fileIndex, ItemPropId.kpidComment);
-            string hostOS = this.GetPropertySafe<string>(fileIndex, ItemPropId.kpidHostOS);
-            string method = this.GetPropertySafe<string>(fileIndex, ItemPropId.kpidMethod);
-
-            bool isSplitBefore = this.GetPropertySafe<bool>(fileIndex, ItemPropId.kpidSplitBefore);
-            bool isSplitAfter = this.GetPropertySafe<bool>(fileIndex, ItemPropId.kpidSplitAfter);
-
-            var entry = new Entry(this._archive, fileIndex)
-            {
-                FileName = fileName,
-                IsFolder = isFolder,
-                IsEncrypted = isEncrypted,
-                Size = size,
-                PackedSize = packedSize,
-                CreationTime = creationTime,
-                LastWriteTime = lastWriteTime,
-                LastAccessTime = lastAccessTime,
-                CRC = crc,
-                Attributes = attributes,
-                Comment = comment,
-                HostOS = hostOS,
-                Method = method,
-                IsSplitBefore = isSplitBefore,
-                IsSplitAfter = isSplitAfter
-            };
-            return entry;
-        }
-
-        private T GetPropertySafe<T>(uint fileIndex, ItemPropId name)
-        {
-            try
-            {
-                return this.GetProperty<T>(fileIndex, name);
-            }
-            catch (InvalidCastException)
-            {
-                return default(T);
-            }
-        }
-
-        private T GetProperty<T>(uint fileIndex, ItemPropId name)
-        {
-            PropVariant propVariant = new PropVariant();
-            this._archive.GetProperty(fileIndex, name, ref propVariant);
-            object value = propVariant.GetObject();
-
-            if (propVariant.VarType == VarEnum.VT_EMPTY)
-            {
-                propVariant.Clear();
-                return default(T);
-            }
-
-            propVariant.Clear();
-
-            if (value == null)
-            {
-                return default(T);
-            }
-
-            Type type = typeof(T);
-            bool isNullable = type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>);
-            Type underlyingType = isNullable ? Nullable.GetUnderlyingType(type) : type;
-
-            T result = (T)Convert.ChangeType(value.ToString(), underlyingType);
-
-            return result;
-        }
-
-        private void InitializeAndValidateLibrary()
-        {
-            try
-            {
-                this._sevenZipHandle = new SevenZipHandle((string)LibraryFilePath);
-            }
-            catch (Exception e)
-            {
-                throw new Exception("Unable to initialize SevenZipHandle", e);
-            }
-        }
-
-        ~ArchiveFile()
-        {
-            this.Dispose(false);
-        }
-
-        protected void Dispose(bool disposing)
-        {
-            if (this._archiveStream != null)
-            {
-                this._archiveStream.Dispose();
-            }
-
-            if (this._archive != null)
-            {
-                Marshal.ReleaseComObject(this._archive);
-            }
-
-            if (this._sevenZipHandle != null)
-            {
-                this._sevenZipHandle.Dispose();
-            }
-        }
-
-        public void Dispose()
-        {
-            this.Dispose(true);
-            GC.SuppressFinalize(this);
-        }
-    }
-}

Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs

@@ -1,89 +0,0 @@
-using System;
-using System.IO;
-
-namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
-{
-    public class Entry
-    {
-        private readonly IInArchive archive;
-        private readonly uint index;
-
-        internal Entry(IInArchive archive, uint index)
-        {
-            this.archive = archive;
-            this.index = index;
-        }
-
-        /// <summary>
-        /// Name of the file with its relative path within the archive
-        /// </summary>
-        public string FileName { get; internal set; }
-        /// <summary>
-        /// True if entry is a folder, false if it is a file
-        /// </summary>
-        public bool IsFolder { get; internal set; }
-        /// <summary>
-        /// Original entry size
-        /// </summary>
-        public ulong Size { get; internal set; }
-        /// <summary>
-        /// Entry size in a archived state
-        /// </summary>
-        public ulong PackedSize { get; internal set; }
-
-        /// <summary>
-        /// Date and time of the file (entry) creation
-        /// </summary>
-        public DateTime CreationTime { get; internal set; }
-
-        /// <summary>
-        /// Date and time of the last change of the file (entry)
-        /// </summary>
-        public DateTime LastWriteTime { get; internal set; }
-
-        /// <summary>
-        /// Date and time of the last access of the file (entry)
-        /// </summary>
-        public DateTime LastAccessTime { get; internal set; }
-        
-        /// <summary>
-        /// CRC hash of the entry
-        /// </summary>
-        public UInt32 CRC { get; internal set; }
-
-        /// <summary>
-        /// Attributes of the entry
-        /// </summary>
-        public UInt32 Attributes { get; internal set; }
-
-        /// <summary>
-        /// True if entry is encrypted, otherwise false
-        /// </summary>
-        public bool IsEncrypted { get; internal set; }
-
-        /// <summary>
-        /// Comment of the entry
-        /// </summary>
-        public string Comment { get; internal set; }
-
-        /// <summary>
-        /// Compression method of the entry
-        /// </summary>
-        public string Method { get; internal set; }
-
-        /// <summary>
-        /// Host operating system of the entry
-        /// </summary>
-        public string HostOS { get; internal set; }
-
-        /// <summary>
-        /// True if there are parts of this file in previous split archive parts
-        /// </summary>
-        public bool IsSplitBefore { get; set; }
-
-        /// <summary>
-        /// True if there are parts of this file in next split archive parts
-        /// </summary>
-        public bool IsSplitAfter { get; set; }
-    }
-}

Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs

@@ -1,20 +0,0 @@
-using System;
-using System.Collections.Generic;
-using Wabbajack.Common.FileSignatures;
-
-namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
-{
-    public class Formats
-    {
-      
-        internal static Dictionary<Definitions.FileType, Guid> FileTypeGuidMapping = new Dictionary<Definitions.FileType, Guid>
-        {
-            {Definitions.FileType._7Z, new Guid("23170f69-40c1-278a-1000-000110070000")},
-            {Definitions.FileType.BZ2, new Guid("23170f69-40c1-278a-1000-000110020000")},
-            {Definitions.FileType.RAR_OLD, new Guid("23170f69-40c1-278a-1000-000110030000")},
-            {Definitions.FileType.RAR_NEW, new Guid("23170f69-40c1-278a-1000-000110CC0000")},
-            {Definitions.FileType.ZIP, new Guid("23170f69-40c1-278a-1000-000110010000")},
-        };
-
-    }
-}

Wabbajack.VirtualFileSystem/SevenZipExtractor/IArchiveExtractCallback.cs

@@ -1,23 +0,0 @@
-using System.Runtime.InteropServices;
-
-namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
-{
-    [ComImport]
-    [Guid("23170F69-40C1-278A-0000-000600200000")]
-    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
-    internal interface IArchiveExtractCallback //: IProgress
-    {
-        void SetTotal(ulong total);
-        void SetCompleted([In] ref ulong completeValue);
-
-        [PreserveSig]
-        int GetStream(
-            uint index,
-            [MarshalAs(UnmanagedType.Interface)] out ISequentialOutStream outStream,
-            AskMode askExtractMode);
-        // GetStream OUT: S_OK - OK, S_FALSE - skeep this file
-
-        void PrepareOperation(AskMode askExtractMode);
-        void SetOperationResult(OperationResult resultEOperationResult);
-    }
-}

Wabbajack.VirtualFileSystem/SevenZipExtractor/Kernel32Dll.cs

@@ -1,20 +0,0 @@
-using System;
-using System.Runtime.InteropServices;
-using System.Security;
-
-namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
-{
-    internal static class Kernel32Dll
-    {
-        [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
-        internal static extern SafeLibraryHandle LoadLibrary([MarshalAs(UnmanagedType.LPTStr)] string lpFileName);
-
-        [DllImport("kernel32.dll", CharSet = CharSet.Ansi, SetLastError = true)]
-        internal static extern IntPtr GetProcAddress(SafeLibraryHandle hModule, [MarshalAs(UnmanagedType.LPStr)] string procName);
-
-        [SuppressUnmanagedCodeSecurity]
-        [DllImport("kernel32.dll")]
-        [return: MarshalAs(UnmanagedType.Bool)]
-        internal static extern bool FreeLibrary(IntPtr hModule);
-    }
-}

Wabbajack.VirtualFileSystem/SevenZipExtractor/SafeLibraryHandle.cs

@@ -1,21 +0,0 @@
-using System;
-using System.Runtime.ConstrainedExecution;
-using Microsoft.Win32.SafeHandles;
-
-namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
-{
-    internal sealed class SafeLibraryHandle : SafeHandleZeroOrMinusOneIsInvalid
-    {
-        public SafeLibraryHandle() : base(true)
-        {
-        }
-
-        /// <summary>Release library handle</summary>
-        /// <returns>true if the handle was released</returns>
-        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success)]
-        protected override bool ReleaseHandle()
-        {
-            return Kernel32Dll.FreeLibrary(this.handle);
-        }
-    }
-}

Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipHandle.cs

@@ -1,68 +0,0 @@
-using System;
-using System.ComponentModel;
-using System.Runtime.InteropServices;
-
-namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
-{
-    internal class SevenZipHandle : IDisposable
-    {
-        private SafeLibraryHandle sevenZipSafeHandle;
-
-        public SevenZipHandle(string sevenZipLibPath)
-        {
-            this.sevenZipSafeHandle = Kernel32Dll.LoadLibrary(sevenZipLibPath);
-
-            if (this.sevenZipSafeHandle.IsInvalid)
-            {
-                throw new Win32Exception();
-            }
-
-            IntPtr functionPtr = Kernel32Dll.GetProcAddress(this.sevenZipSafeHandle, "GetHandlerProperty");
-            
-            // Not valid dll
-            if (functionPtr == IntPtr.Zero)
-            {
-                this.sevenZipSafeHandle.Close();
-                throw new ArgumentException();
-            }
-        }
-
-        ~SevenZipHandle()
-        {
-            this.Dispose(false);
-        }
-
-        protected void Dispose(bool disposing)
-        {
-            if ((this.sevenZipSafeHandle != null) && !this.sevenZipSafeHandle.IsClosed)
-            {
-                this.sevenZipSafeHandle.Close();
-            }
-
-            this.sevenZipSafeHandle = null;
-        }
-
-        public void Dispose()
-        {
-            this.Dispose(true);
-            GC.SuppressFinalize(this);
-        }
-
-        public IInArchive CreateInArchive(Guid classId)
-        {
-            if (this.sevenZipSafeHandle == null)
-            {
-                throw new ObjectDisposedException("SevenZipHandle");
-            }
-
-            IntPtr procAddress = Kernel32Dll.GetProcAddress(this.sevenZipSafeHandle, "CreateObject");
-            CreateObjectDelegate createObject = (CreateObjectDelegate) Marshal.GetDelegateForFunctionPointer(procAddress, typeof (CreateObjectDelegate));
-
-            object result;
-            Guid interfaceId = typeof (IInArchive).GUID;
-            createObject(ref classId, ref interfaceId, out result);
-
-            return result as IInArchive;
-        }
-    }
-}

Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs

@@ -1,457 +0,0 @@
-// Version 1.5
-
-using System;
-using System.Globalization;
-using System.IO;
-using System.Runtime.InteropServices;
-using System.Security.Permissions;
-using System.Threading;
-
-namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
-{
-    [StructLayout(LayoutKind.Sequential)]
-    internal struct PropArray
-    {
-        uint length;
-        IntPtr pointerValues;
-    }
-
-    [StructLayout(LayoutKind.Explicit)]
-    internal struct PropVariant
-    {
-        [DllImport("ole32.dll")]
-        private static extern int PropVariantClear(ref PropVariant pvar);
-
-        [FieldOffset(0)] public ushort vt;
-        [FieldOffset(8)] public IntPtr pointerValue;
-        [FieldOffset(8)] public byte byteValue;
-        [FieldOffset(8)] public long longValue;
-        [FieldOffset(8)] public UInt32 UInt32Value;
-        [FieldOffset(8)] public System.Runtime.InteropServices.ComTypes.FILETIME filetime;
-        [FieldOffset(8)] public PropArray propArray;
-
-        public VarEnum VarType
-        {
-            get
-            {
-                return (VarEnum) this.vt;
-            }
-            set
-            {
-                vt = (ushort)value;
-            }
-        }
-
-        public void Clear()
-        {
-            switch (this.VarType)
-            {
-                case VarEnum.VT_EMPTY:
-                    break;
-
-                case VarEnum.VT_NULL:
-                case VarEnum.VT_I2:
-                case VarEnum.VT_I4:
-                case VarEnum.VT_R4:
-                case VarEnum.VT_R8:
-                case VarEnum.VT_CY:
-                case VarEnum.VT_DATE:
-                case VarEnum.VT_ERROR:
-                case VarEnum.VT_BOOL:
-                //case VarEnum.VT_DECIMAL:
-                case VarEnum.VT_I1:
-                case VarEnum.VT_UI1:
-                case VarEnum.VT_UI2:
-                case VarEnum.VT_UI4:
-                case VarEnum.VT_I8:
-                case VarEnum.VT_UI8:
-                case VarEnum.VT_INT:
-                case VarEnum.VT_UINT:
-                case VarEnum.VT_HRESULT:
-                case VarEnum.VT_FILETIME:
-                    this.vt = 0;
-                    break;
-
-                default:
-                    PropVariantClear(ref this);
-                    break;
-            }
-        }
-
-        public object GetObject()
-        {
-            switch (this.VarType)
-            {
-                case VarEnum.VT_EMPTY:
-                    return null;
-
-                case VarEnum.VT_FILETIME:
-                    return DateTime.FromFileTime(this.longValue);
-
-                default:
-                    GCHandle PropHandle = GCHandle.Alloc(this, GCHandleType.Pinned);
-
-                    try
-                    {
-                        return Marshal.GetObjectForNativeVariant(PropHandle.AddrOfPinnedObject());
-                    }
-                    finally
-                    {
-                        PropHandle.Free();
-                    }
-            }
-        }
-    }
-
-    [ComImport]
-    [Guid("23170F69-40C1-278A-0000-000000050000")]
-    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
-    internal interface IProgress
-    {
-        void SetTotal(ulong total);
-        void SetCompleted([In] ref ulong completeValue);
-    }
-
-    [ComImport]
-    [Guid("23170F69-40C1-278A-0000-000600100000")]
-    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
-    internal interface IArchiveOpenCallback
-    {
-        // ref ulong replaced with IntPtr because handlers ofter pass null value
-        // read actual value with Marshal.ReadInt64
-        void SetTotal(
-            IntPtr files, // [In] ref ulong files, can use 'ulong* files' but it is unsafe
-            IntPtr bytes); // [In] ref ulong bytes
-
-        void SetCompleted(
-            IntPtr files, // [In] ref ulong files
-            IntPtr bytes); // [In] ref ulong bytes
-    }
-
-    [ComImport]
-    [Guid("23170F69-40C1-278A-0000-000500100000")]
-    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
-    internal interface ICryptoGetTextPassword
-    {
-        [PreserveSig]
-        int CryptoGetTextPassword(
-            [MarshalAs(UnmanagedType.BStr)] out string password);
-
-        //[return : MarshalAs(UnmanagedType.BStr)]
-        //string CryptoGetTextPassword();
-    }
-
-    public enum AskMode : int
-    {
-        kExtract = 0,
-        kTest,
-        kSkip
-    }
-
-    public enum OperationResult : int
-    {
-        kOK = 0,
-        kUnSupportedMethod,
-        kDataError,
-        kCRCError
-    }
-
-    [ComImport]
-    [Guid("23170F69-40C1-278A-0000-000600300000")]
-    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
-    internal interface IArchiveOpenVolumeCallback
-    {
-        void GetProperty(
-            ItemPropId propID, // PROPID
-            IntPtr value); // PROPVARIANT
-
-        [PreserveSig]
-        int GetStream(
-            [MarshalAs(UnmanagedType.LPWStr)] string name,
-            [MarshalAs(UnmanagedType.Interface)] out IInStream inStream);
-    }
-
-    [ComImport]
-    [Guid("23170F69-40C1-278A-0000-000600400000")]
-    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
-    internal interface IInArchiveGetStream
-    {
-        [return: MarshalAs(UnmanagedType.Interface)]
-        ISequentialInStream GetStream(uint index);
-    }
-
-    [ComImport]
-    [Guid("23170F69-40C1-278A-0000-000300010000")]
-    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
-    internal interface ISequentialInStream
-    {
-        //[PreserveSig]
-        //int Read(
-        //  [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
-        //  uint size,
-        //  IntPtr processedSize); // ref uint processedSize
-
-        uint Read(
-            [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
-            uint size);
-
-        /*
-    Out: if size != 0, return_value = S_OK and (*processedSize == 0),
-      then there are no more bytes in stream.
-    if (size > 0) && there are bytes in stream, 
-    this function must read at least 1 byte.
-    This function is allowed to read less than number of remaining bytes in stream.
-    You must call Read function in loop, if you need exact amount of data
-    */
-    }
-
-    [ComImport]
-    [Guid("23170F69-40C1-278A-0000-000300020000")]
-    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
-    public interface ISequentialOutStream
-    {
-        [PreserveSig]
-        int Write(
-            [In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)]byte[] data,
-            uint size,
-            IntPtr processedSize); // ref uint processedSize
-        /*
-    if (size > 0) this function must write at least 1 byte.
-    This function is allowed to write less than "size".
-    You must call Write function in loop, if you need to write exact amount of data
-    */
-    }
-
-    [ComImport]
-    [Guid("23170F69-40C1-278A-0000-000300030000")]
-    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
-    internal interface IInStream //: ISequentialInStream
-    {
-        //[PreserveSig]
-        //int Read(
-        //  [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
-        //  uint size,
-        //  IntPtr processedSize); // ref uint processedSize
-
-        uint Read(
-            [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
-            uint size);
-
-        //[PreserveSig]
-        void Seek(
-            long offset,
-            uint seekOrigin,
-            IntPtr newPosition); // ref long newPosition
-    }
-
-    [ComImport]
-    [Guid("23170F69-40C1-278A-0000-000300040000")]
-    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
-    internal interface IOutStream //: ISequentialOutStream
-    {
-        [PreserveSig]
-        int Write(
-            [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
-            uint size,
-            IntPtr processedSize); // ref uint processedSize
-
-        //[PreserveSig]
-        void Seek(
-            long offset,
-            uint seekOrigin,
-            IntPtr newPosition); // ref long newPosition
-
-        [PreserveSig]
-        int SetSize(long newSize);
-    }
-
-    internal enum ItemPropId : uint
-    {
-        kpidNoProperty = 0,
-
-        kpidHandlerItemIndex = 2,
-        kpidPath,
-        kpidName,
-        kpidExtension,
-        kpidIsFolder,
-        kpidSize,
-        kpidPackedSize,
-        kpidAttributes,
-        kpidCreationTime,
-        kpidLastAccessTime,
-        kpidLastWriteTime,
-        kpidSolid,
-        kpidCommented,
-        kpidEncrypted,
-        kpidSplitBefore,
-        kpidSplitAfter,
-        kpidDictionarySize,
-        kpidCRC,
-        kpidType,
-        kpidIsAnti,
-        kpidMethod,
-        kpidHostOS,
-        kpidFileSystem,
-        kpidUser,
-        kpidGroup,
-        kpidBlock,
-        kpidComment,
-        kpidPosition,
-        kpidPrefix,
-
-        kpidTotalSize = 0x1100,
-        kpidFreeSpace,
-        kpidClusterSize,
-        kpidVolumeName,
-
-        kpidLocalName = 0x1200,
-        kpidProvider,
-
-        kpidUserDefined = 0x10000
-    }
-    
-    /// <summary>
-    /// 7-zip ISetProperties interface for setting various archive properties
-    /// </summary>
-    [ComImport]
-    [Guid("23170F69-40C1-278A-0000-000600030000")]
-    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
-    internal interface ISetProperties
-    {
-        /// <summary>
-        /// Sets the archive properties
-        /// </summary>
-        /// <param name="names">The names of the properties</param>
-        /// <param name="values">The values of the properties</param>
-        /// <param name="numProperties">The properties count</param>
-        /// <returns></returns>        
-        int SetProperties(IntPtr names, IntPtr values, int numProperties);
-    }
-
-
-    [ComImport]
-    [Guid("23170F69-40C1-278A-0000-000600600000")]
-    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
-    //[AutomationProxy(true)]
-    internal interface IInArchive
-    {
-        [PreserveSig]
-        int Open(
-            IInStream stream,
-            /*[MarshalAs(UnmanagedType.U8)]*/ [In] ref ulong maxCheckStartPosition,
-            [MarshalAs(UnmanagedType.Interface)] IArchiveOpenCallback openArchiveCallback);
-
-        void Close();
-        //void GetNumberOfItems([In] ref uint numItem);
-        uint GetNumberOfItems();
-
-        void GetProperty(
-            uint index,
-            ItemPropId propID, // PROPID
-            ref PropVariant value); // PROPVARIANT
-
-        [PreserveSig]
-        int Extract(
-            [MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] uint[] indices, //[In] ref uint indices,
-            uint numItems,
-            int testMode,
-            [MarshalAs(UnmanagedType.Interface)] IArchiveExtractCallback extractCallback);
-
-        // indices must be sorted 
-        // numItems = 0xFFFFFFFF means all files
-        // testMode != 0 means "test files operation"
-
-        void GetArchiveProperty(
-            uint propID, // PROPID
-            ref PropVariant value); // PROPVARIANT
-
-        //void GetNumberOfProperties([In] ref uint numProperties);
-        uint GetNumberOfProperties();
-
-        void GetPropertyInfo(
-            uint index,
-            [MarshalAs(UnmanagedType.BStr)] out string name,
-            out ItemPropId propID, // PROPID
-            out ushort varType); //VARTYPE
-
-        //void GetNumberOfArchiveProperties([In] ref uint numProperties);
-        uint GetNumberOfArchiveProperties();
-
-        void GetArchivePropertyInfo(
-            uint index,
-            [MarshalAs(UnmanagedType.BStr)] string name,
-            ref uint propID, // PROPID
-            ref ushort varType); //VARTYPE
-    }
-
-    internal enum ArchivePropId : uint
-    {
-        kName = 0,
-        kClassID,
-        kExtension,
-        kAddExtension,
-        kUpdate,
-        kKeepName,
-        kStartSignature,
-        kFinishSignature,
-        kAssociate
-    }
-
-    [UnmanagedFunctionPointer(CallingConvention.StdCall)]
-    internal delegate int CreateObjectDelegate(
-        [In] ref Guid classID,
-        [In] ref Guid interfaceID,
-        //out IntPtr outObject);
-        [MarshalAs(UnmanagedType.Interface)] out object outObject);
-
-    [UnmanagedFunctionPointer(CallingConvention.StdCall)]
-    internal delegate int GetHandlerPropertyDelegate(
-        ArchivePropId propID,
-        ref PropVariant value); // PROPVARIANT
-
-    [UnmanagedFunctionPointer(CallingConvention.StdCall)]
-    internal delegate int GetNumberOfFormatsDelegate(out uint numFormats);
-
-    [UnmanagedFunctionPointer(CallingConvention.StdCall)]
-    internal delegate int GetHandlerProperty2Delegate(
-        uint formatIndex,
-        ArchivePropId propID,
-        ref PropVariant value); // PROPVARIANT
-
-    public class StreamWrapper : IDisposable
-    {
-        protected Stream BaseStream;
-
-        protected StreamWrapper(Stream baseStream)
-        {
-            this.BaseStream = baseStream;
-        }
-
-        public void Dispose()
-        {
-            this.BaseStream.Close();
-        }
-
-        public virtual void Seek(long offset, uint seekOrigin, IntPtr newPosition)
-        {
-            long Position = (uint) this.BaseStream.Seek(offset, (SeekOrigin) seekOrigin);
-
-            if (newPosition != IntPtr.Zero)
-            {
-                Marshal.WriteInt64(newPosition, Position);
-            }
-        }
-    }
-
-    public class InStreamWrapper : StreamWrapper, ISequentialInStream, IInStream
-    {
-        public InStreamWrapper(Stream baseStream) : base(baseStream)
-        {
-        }
-
-        public uint Read(byte[] data, uint size)
-        {
-            return (uint) this.BaseStream.Read(data, 0, (int) size);
-        }
-    }
-}

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -218,7 +218,7 @@ namespace Wabbajack.VirtualFileSystem
             try
             {
 
-                var list = await FileExtractor2.GatheringExtract(extractedFile,
+                var list = await FileExtractor2.GatheringExtract(context.Queue, extractedFile,
                     _ => true,
                     async (path, sfactory) => await Analyze(context, self, sfactory, path, depth + 1));