WIP, massive rework of file extraction

Compression.BSA/BSA/Reader/BSAReader.cs

@@ -76,6 +76,7 @@ namespace Compression.BSA
             return bsa;
         }
 
+
         public static BSAReader Load(AbsolutePath filename)
         {
             var bsa = new BSAReader { _fileName = filename };

Wabbajack.Common/FileSignatures/SignatureChecker.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
 
@@ -21,9 +22,14 @@ namespace Wabbajack.Common.FileSignatures
         public async Task<Definitions.FileType?> MatchesAsync(AbsolutePath path)
         {
             await using var fs = await path.OpenShared();
+            return await MatchesAsync(fs);
+        }
+        
+        public async Task<Definitions.FileType?> MatchesAsync(Stream stream)
+        {
             var buffer = new byte[_maxLength];
-            fs.Position = 0;
+            stream.Position = 0;
-            await fs.ReadAsync(buffer);
+            await stream.ReadAsync(buffer);
 
             foreach (var (fileType, signature) in _signatures)
             {

Wabbajack.Lib/AInstaller.cs

@@ -113,6 +113,10 @@ namespace Wabbajack.Lib
 
         public async Task InstallArchives()
         {
+            await VFS.CopyTo(Queue, ModList.Directives
+                .OfType<FromArchive>()
+                .Select(a => (VFS.Index.FileForArchiveHashPath(a.ArchiveHashPath), a.To.RelativeTo(OutputFolder))));
+            /*
             Info("Installing Archives");
             Info("Grouping Install Files");
             var grouped = ModList.Directives
@@ -125,7 +129,7 @@ namespace Wabbajack.Lib
                 .ToList();
 
             Info("Installing Archives");
-            await archives.PMap(Queue, UpdateTracker,a => InstallArchive(Queue, a.Archive, a.AbsolutePath, grouped[a.Archive.Hash]));
+            await archives.PMap(Queue, UpdateTracker,a => InstallArchive(Queue, a.Archive, a.AbsolutePath, grouped[a.Archive.Hash]));*/
         }
 
         private async Task InstallArchive(WorkQueue queue, Archive archive, AbsolutePath absolutePath, IGrouping<Hash, FromArchive> grouping)

Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs

@@ -0,0 +1,63 @@
+using System;
+using System.IO.Compression;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Xunit;
+
+namespace Wabbajack.VirtualFileSystem.Test
+{
+    public class FileExtractorTests
+    {
+        [Fact]
+        public async Task CanGatherDataFromZipFiles()
+        {
+            await using var temp = await TempFolder.Create();
+            await using var archive = new TempFile();
+            for (int i = 0; i < 10; i ++)
+            {
+                await WriteRandomData(temp.Dir.Combine($"{i}.bin"), _rng.Next(10, 1024));
+            }
+
+            await ZipUpFolder(temp.Dir, archive.Path, false);
+            
+            var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path), 
+                _ => true,
+                async (path, sfn) =>
+            {
+                await using var s = await sfn.GetStream();
+                return await s.xxHashAsync();
+            });
+            
+            Assert.Equal(10, results.Count);
+            foreach (var (path, hash) in results)
+            {
+                Assert.Equal(await temp.Dir.Combine(path).FileHashAsync(), hash);
+            }
+
+
+        }
+
+
+        private static readonly Random _rng = new Random();
+        private static async Task WriteRandomData(AbsolutePath path, int size)
+        {
+            var buff = new byte[size];
+            _rng.NextBytes(buff);
+            await path.WriteAllBytesAsync(buff);
+        }
+        
+        private static async Task AddFile(AbsolutePath filename, string text)
+        {
+            filename.Parent.CreateDirectory();
+            await filename.WriteAllTextAsync(text);
+        }
+
+        private static async Task ZipUpFolder(AbsolutePath folder, AbsolutePath output, bool deleteSource = true)
+        {
+            ZipFile.CreateFromDirectory((string)folder, (string)output);
+            if (deleteSource) 
+                await folder.DeleteDirectory();
+        }
+        
+    }
+}

Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs

@@ -139,15 +139,15 @@ namespace Wabbajack.VirtualFileSystem.Test
             await AddTestRoot();
 
             var res = new FullPath(TEST_ZIP, new[] {(RelativePath)"test.txt"});
-            var file = context.Index.ByFullPath[res];
+            var files = new [] {context.Index.ByFullPath[res]};
 
-            var cleanup = await context.Stage(new List<VirtualFile> {file});
+            var queue = new WorkQueue();
+            await context.Extract(queue, files.ToHashSet(), async (file, factory) =>
+            {
+                await using var s = await factory.GetStream();
+                Assert.Equal("This is a test", await s.ReadAllTextAsync());
+            });
 
-            await using var stream = await file.StagedFile.OpenRead();
-           
-            Assert.Equal("This is a test", await stream.ReadAllTextAsync());
-
-            await cleanup();
         }
 
         [Fact]
@@ -165,16 +165,13 @@ namespace Wabbajack.VirtualFileSystem.Test
 
             var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")];
 
-            var cleanup = await context.Stage(files);
+            var queue = new WorkQueue();
-
+            await context.Extract(queue, files.ToHashSet(), async (file, factory) =>
-            foreach (var file in files)
             {
-                await using var stream = await file.StagedFile.OpenRead();
+                await using var s = await factory.GetStream();
+                Assert.Equal("This is a test", await s.ReadAllTextAsync());
+            });
 
-                Assert.Equal("This is a test", await stream.ReadAllTextAsync());
-            }
-
-            await cleanup();
         }
 
         private static async Task AddFile(AbsolutePath filename, string text)

Wabbajack.VirtualFileSystem/Context.cs

@@ -71,7 +71,7 @@ namespace Wabbajack.VirtualFileSystem
                                         return found;
                                 }
 
-                                return await VirtualFile.Analyze(this, null, new RootDiskFile(f), f, 0);
+                                return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, 0);
                             });
 
             var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@@ -103,7 +103,7 @@ namespace Wabbajack.VirtualFileSystem
                             return found;
                     }
 
-                    return await VirtualFile.Analyze(this, null, new RootDiskFile(f), f, 0);
+                    return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, 0);
                 });
 
             var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@@ -196,6 +196,40 @@ namespace Wabbajack.VirtualFileSystem
             }
         }
 
+        /// <summary>
+        /// Extract the set of files and call the callback for each, handing it a stream factory and the virtual file,
+        /// top level archives (native archives) will be processed in parallel. Duplicate files will not be 
+        /// </summary>
+        /// <param name="files"></param>
+        /// <param name="callback"></param>
+        /// <returns></returns>
+        public async Task Extract(WorkQueue queue, HashSet<VirtualFile> files, Func<VirtualFile, IStreamFactory, ValueTask> callback)
+        {
+            var top = new VirtualFile();
+            var filesByParent = files.SelectMany(f => f.FilesInFullPath)
+                .Distinct()
+                .GroupBy(f => f.Parent ?? top)
+                .ToDictionary(f => f.Key);
+
+            async Task HandleFile(VirtualFile file, IStreamFactory sfn)
+            {
+                if (files.Contains(file)) await callback(file, sfn);
+                if (filesByParent.TryGetValue(file, out var children))
+                {
+                    var fileNames = children.ToDictionary(c => c.RelativeName);
+                    await FileExtractor2.GatheringExtract(sfn,
+                        r => fileNames.ContainsKey(r),
+                        async (rel, csf) =>
+                        {
+                            await HandleFile(fileNames[rel], sfn);
+                            return 0;
+                        });
+                }
+
+            }
+            await filesByParent[top].PMap(queue, async file => await HandleFile(file, new NativeFileStreamFactory(file.AbsoluteName)));
+        }
+
         public async Task<Func<Task>> Stage(IEnumerable<VirtualFile> files)
         {
             await _cleanupTask;
@@ -227,6 +261,12 @@ namespace Wabbajack.VirtualFileSystem
             };
         }
 
+        public async Task CopyTo(WorkQueue queue, IEnumerable<(VirtualFile src, AbsolutePath dest)> directives)
+        {
+            var plans = StagingPlan.StagingPlan.CreatePlan(directives).ToArray();
+            await StagingPlan.StagingPlan.ExecutePlans(queue, plans);
+        }
+
         public async Task<AsyncDisposableList<VirtualFile>> StageWith(IEnumerable<VirtualFile> files)
         {
             return new AsyncDisposableList<VirtualFile>(await Stage(files), files);

Wabbajack.VirtualFileSystem/FileExtractor.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
 using System.Linq;
 using System.Reactive.Linq;
 using System.Threading.Tasks;
@@ -19,8 +20,7 @@ namespace Wabbajack.VirtualFileSystem
 {
     public class FileExtractor
     {
-
+        public static readonly SignatureChecker ArchiveSigs = new SignatureChecker(Definitions.FileType.TES3, 
-        private static SignatureChecker archiveSigs = new SignatureChecker(Definitions.FileType.TES3, 
             Definitions.FileType.BSA,
             Definitions.FileType.BA2,
             Definitions.FileType.ZIP,
@@ -34,7 +34,7 @@ namespace Wabbajack.VirtualFileSystem
 
             try
             {
-                var sig = await archiveSigs.MatchesAsync(source);
+                var sig = await ArchiveSigs.MatchesAsync(source);
                 
                 if (source.Extension == Consts.OMOD)
                     return await ExtractAllWithOMOD(source);
@@ -71,7 +71,7 @@ namespace Wabbajack.VirtualFileSystem
 
             if (isArchive)
             {
-                return await ExtractAllWith7Zip(source, null);
+                return await ExtractAllWith7Zip(source, (IEnumerable<RelativePath>) null);
             }
 
             var dest = await TempFolder.Create();
@@ -183,7 +183,7 @@ namespace Wabbajack.VirtualFileSystem
         /// <returns></returns>
         public static async Task<bool> CanExtract(AbsolutePath v)
         {
-            var found = await archiveSigs.MatchesAsync(v);
+            var found = await ArchiveSigs.MatchesAsync(v);
             switch (found)
             {
                 case null:
@@ -220,5 +220,64 @@ namespace Wabbajack.VirtualFileSystem
         {
             return ext == _exeExtension || Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext);
         }
+
+        /// <summary>
+        /// Extract the specific files to the specific locations
+        /// </summary>
+        /// <param name="queue"></param>
+        /// <param name="archive"></param>
+        /// <param name="indexed"></param>
+        /// <exception cref="NotImplementedException"></exception>
+        public static async Task ExtractTo(WorkQueue queue, AbsolutePath source, Dictionary<RelativePath,AbsolutePath> indexed)
+        {
+            try
+            {
+                var sig = await ArchiveSigs.MatchesAsync(source);
+                
+                /*if (source.Extension == Consts.OMOD)
+                    return await ExtractAllWithOMOD(source);*/
+
+                switch (sig)
+                {
+                    case Definitions.FileType.BSA:
+                    case Definitions.FileType.TES3:
+                    case Definitions.FileType.BA2:
+                        await ExtractAllWithBSA(queue, source, indexed);
+                        return;
+                    case Definitions.FileType.EXE:
+                        await ExtractAllExe(source, indexed);
+                        return;
+                    case Definitions.FileType._7Z:
+                    case Definitions.FileType.ZIP:
+                    case Definitions.FileType.RAR:
+                        await ExtractAllWith7Zip(source, indexed);
+                        return;
+                }
+                throw new Exception("Invalid archive format");
+            }
+            catch (Exception ex)
+            {
+                Utils.Log(ex.ToString());
+                Utils.ErrorThrow(ex, $"Error while extracting {source}");
+                throw new Exception();
+            }
+        }
+
+        private static async Task ExtractAllWith7Zip(AbsolutePath source, Dictionary<RelativePath,AbsolutePath> onlyFiles)
+        {
+            using var archive = await ArchiveFile.Open(source);
+            await archive.Extract(f => onlyFiles.TryGetValue(f, out var dest) ? dest : default);
+        }
+
+        private static async Task ExtractAllExe(AbsolutePath source, Dictionary<RelativePath,AbsolutePath> indexed)
+        {
+            throw new NotImplementedException();
+        }
+
+        private static async Task ExtractAllWithBSA(WorkQueue queue, AbsolutePath source, Dictionary<RelativePath,AbsolutePath> indexed)
+        {
+            throw new NotImplementedException();
+
+        }
     }
 }

Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs

@@ -0,0 +1,139 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Threading;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Common.FileSignatures;
+using Wabbajack.VirtualFileSystem.SevenZipExtractor;
+
+namespace Wabbajack.VirtualFileSystem
+{
+    public class GatheringExtractor<T> : IArchiveExtractCallback
+    {
+        private ArchiveFile _archive;
+        private Predicate<RelativePath> _shouldExtract;
+        private Func<RelativePath, IStreamFactory, ValueTask<T>> _mapFn;
+        private Dictionary<RelativePath, T> _results;
+        private Dictionary<uint, RelativePath> _indexes;
+        private Stream _stream;
+        private Definitions.FileType _sig;
+
+        public GatheringExtractor(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory, ValueTask<T>> mapfn)
+        {
+            
+            _shouldExtract = shouldExtract;
+            _mapFn = mapfn;
+            _results = new Dictionary<RelativePath, T>();
+            _stream = stream;
+            _sig = sig;
+        }
+
+        public async Task<Dictionary<RelativePath, T>> Extract()
+        {
+            var source = new TaskCompletionSource<bool>();
+
+            var th = new Thread(() =>
+            {
+                try
+                {
+                    _archive = ArchiveFile.Open(_stream, _sig).Result;
+                    _indexes = _archive.Entries
+                        .Where(f => !f.IsFolder)
+                        .Select((entry, idx) => ((RelativePath)entry.FileName, (uint)idx))
+                        .Where(t => _shouldExtract(t.Item1))
+                        .ToDictionary(t => t.Item2, t => t.Item1);
+
+
+                    _archive._archive.Extract(null, 0xFFFFFFFF, 0, this);
+                    _archive.Dispose();
+                    source.SetResult(true);
+                }
+                catch (Exception ex)
+                {
+                    source.SetException(ex);
+                }
+                
+            }) {Priority = ThreadPriority.BelowNormal, Name = "7Zip Extraction Worker Thread"};
+            th.Start();
+            
+            
+
+            await source.Task;
+            return _results;
+        }
+
+        public void SetTotal(ulong total)
+        {
+            
+        }
+
+        public void SetCompleted(ref ulong completeValue)
+        {
+            
+        }
+
+        public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode)
+        {
+            if (_indexes.ContainsKey(index))
+            {
+                outStream =  new GatheringExtractorStream<T>(this, index);
+                return 0;
+            }
+
+            outStream = null;
+            return 0;
+        }
+
+        public void PrepareOperation(AskMode askExtractMode)
+        {
+            
+        }
+
+        public void SetOperationResult(OperationResult resultEOperationResult)
+        {
+            
+        }
+
+        private class GatheringExtractorStream<T> : ISequentialOutStream, IOutStream
+        {
+            private GatheringExtractor<T> _extractor;
+            private uint _index;
+
+            public GatheringExtractorStream(GatheringExtractor<T> extractor, uint index)
+            {
+                _extractor = extractor;
+                _index = index;
+            }
+
+            public int Write(IntPtr data, uint size, IntPtr processedSize)
+            {
+                unsafe
+                {
+                    var result = _extractor._mapFn(_extractor._indexes[_index], new UnmanagedStreamFactory((byte*)data, size)).AsTask().Result;
+
+                    _extractor._results[_extractor._indexes[_index]] = result;
+                
+                    if (processedSize != IntPtr.Zero)
+                    {
+                        Marshal.WriteInt32(processedSize, (int) size);
+                    }
+
+                    return 0;
+                }
+            }
+
+            public void Seek(long offset, uint seekOrigin, IntPtr newPosition)
+            {
+                
+            }
+
+            public int SetSize(long newSize)
+            {
+                return 0;
+            }
+        }
+    }
+}

Wabbajack.VirtualFileSystem/FileExtractor2/IStreamFactory.cs

@@ -0,0 +1,52 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem
+{
+    public interface IStreamFactory
+    {
+        Task<Stream> GetStream();
+        
+        DateTime LastModifiedUtc { get; }
+        
+    }
+
+    public class UnmanagedStreamFactory : IStreamFactory
+    {
+        private readonly unsafe byte* _data;
+        private readonly long _size;
+
+        public unsafe UnmanagedStreamFactory(byte* data, long size)
+        {
+            _data = data;
+            _size = size;
+        }
+        public async Task<Stream> GetStream()
+        {
+            unsafe
+            {
+                return new UnmanagedMemoryStream(_data, _size);
+            }
+        }
+
+        public DateTime LastModifiedUtc => DateTime.UtcNow;
+    }
+
+    public class NativeFileStreamFactory : IStreamFactory
+    {
+        private AbsolutePath _file;
+
+        public NativeFileStreamFactory(AbsolutePath file)
+        {
+            _file = file;
+        }
+        public async Task<Stream> GetStream()
+        {
+            return await _file.OpenRead();
+        }
+
+        public DateTime LastModifiedUtc => _file.LastModifiedUtc;
+    }
+}

Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs

@@ -12,7 +12,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
     public class ArchiveFile : IDisposable
     {
         private SevenZipHandle _sevenZipHandle;
-        private IInArchive _archive;
+        internal IInArchive _archive;
         private InStreamWrapper _archiveStream;
         private IList<Entry> _entries;
 
@@ -42,11 +42,11 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
             return self;
         }
         
-        public static async Task<ArchiveFile> Open(Stream archiveStream, SevenZipFormat? format = null, string libraryFilePath = null)
+        public static async Task<ArchiveFile> Open(Stream archiveStream, Definitions.FileType format)
         {
             var self = new ArchiveFile();
             self.InitializeAndValidateLibrary();
-            self._archive = self._sevenZipHandle.CreateInArchive(Formats.FormatGuidMapping[SevenZipFormat.SevenZip]);
+            self._archive = self._sevenZipHandle.CreateInArchive(Formats.FileTypeGuidMapping[format]);
             self._archiveStream = new InStreamWrapper(archiveStream);
             return self;
         }

Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs

@@ -136,14 +136,14 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
         //string CryptoGetTextPassword();
     }
 
-    internal enum AskMode : int
+    public enum AskMode : int
     {
         kExtract = 0,
         kTest,
         kSkip
     }
 
-    internal enum OperationResult : int
+    public enum OperationResult : int
     {
         kOK = 0,
         kUnSupportedMethod,
@@ -203,11 +203,11 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
     [ComImport]
     [Guid("23170F69-40C1-278A-0000-000300020000")]
     [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
-    internal interface ISequentialOutStream
+    public interface ISequentialOutStream
     {
         [PreserveSig]
         int Write(
-            [In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
+            IntPtr data,
             uint size,
             IntPtr processedSize); // ref uint processedSize
         /*
@@ -246,7 +246,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
     {
         [PreserveSig]
         int Write(
-            [In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
+            IntPtr data,
             uint size,
             IntPtr processedSize); // ref uint processedSize
 
@@ -444,15 +444,17 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
             return 0;
         }
 
-        public int Write(byte[] data, uint size, IntPtr processedSize)
+        public int Write(IntPtr data, uint size, IntPtr processedSize)
         {
+            throw new NotImplementedException();
+            /*
             this.BaseStream.Write(data, 0, (int) size);
 
             if (processedSize != IntPtr.Zero)
             {
                 Marshal.WriteInt32(processedSize, (int) size);
             }
-
+*/
             return 0;
         }
     }

Wabbajack.VirtualFileSystem/SevenZipExtractor/TypedExtractor.cs

@@ -0,0 +1,64 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
+{
+    internal class TypedExtractor<T> : IArchiveExtractCallback
+    {
+        private Dictionary<RelativePath, T> _mappings;
+        private Action<RelativePath, T, Func<Stream>> _callback;
+        private Dictionary<uint, RelativePath> _indexToFile;
+
+        public TypedExtractor(Dictionary<RelativePath, T> mappings, Action<RelativePath, T, Func<Stream>> callback)
+        {
+            _mappings = mappings;
+            _callback = callback;
+        }
+
+        public void Extract(ArchiveFile file)
+        {
+            _indexToFile = new Dictionary<uint, RelativePath>();
+
+            uint idx = 0;
+            foreach (var entry in file.Entries)
+            {
+                var rel = (RelativePath)entry.FileName;
+                if (_mappings.ContainsKey(rel)) ;
+                {
+                    _indexToFile.Add(idx, rel);
+                }
+                idx += 1;
+            }
+
+            file._archive.Extract(null, 0xFFFFFFFF, 0, this);
+        }
+        
+        public void SetTotal(ulong total)
+        {
+            throw new System.NotImplementedException();
+        }
+
+        public void SetCompleted(ref ulong completeValue)
+        {
+            throw new System.NotImplementedException();
+        }
+
+        public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode)
+        {
+            outStream = null;
+            throw new System.NotImplementedException();
+        }
+
+        public void PrepareOperation(AskMode askExtractMode)
+        {
+            throw new System.NotImplementedException();
+        }
+
+        public void SetOperationResult(OperationResult resultEOperationResult)
+        {
+            throw new System.NotImplementedException();
+        }
+    }
+}

Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs

@@ -0,0 +1,25 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem.StagingPlan
+{
+    public abstract class ASubStage : ISubStage
+    {
+        private IEnumerable<IStagingPlan> _plans;
+
+        public ASubStage(IEnumerable<IStagingPlan> plans)
+        {
+            _plans = plans;
+        }
+        
+        public abstract ValueTask DisposeAsync();
+        public abstract AbsolutePath Destination { get; }
+        public abstract IPath Source { get; }
+        
+        public async Task Execute(WorkQueue queue)
+        {
+            await StagingPlan.ExecutePlan(queue, async () => await Destination.OpenWrite(), _plans);
+        }
+    }
+}

Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs

@@ -0,0 +1,23 @@
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem.StagingPlan
+{
+    public class CopyTo : IStagingSrc
+    {
+        public CopyTo(IPath src, AbsolutePath destination)
+        {
+            Destination = destination;
+            Source = src;
+        }
+
+        public AbsolutePath Destination { get; }
+
+        public async ValueTask DisposeAsync()
+        {
+            
+        }
+
+        public IPath Source { get; }
+    }
+}

Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs

@@ -0,0 +1,26 @@
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem.StagingPlan
+{
+    public class DuplicateTo : IStagingPlan
+    {
+        private readonly IStagingPlan _src;
+
+        public DuplicateTo(IStagingPlan src, AbsolutePath destination)
+        {
+            _src = src;
+            Destination = destination;
+        }
+        public async ValueTask DisposeAsync()
+        {
+        }
+
+        public async Task Execute()
+        {
+            await _src.Destination.CopyToAsync(Destination);
+        }
+
+        public AbsolutePath Destination { get; }
+    }
+}

Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs

@@ -0,0 +1,156 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Common.FileSignatures;
+using Wabbajack.VirtualFileSystem.SevenZipExtractor;
+
+namespace Wabbajack.VirtualFileSystem.StagingPlan
+{
+    public interface IStagingPlan : IAsyncDisposable
+    {
+         AbsolutePath Destination { get; }
+    }
+
+    public static class StagingPlan
+    {
+
+        public static IEnumerable<IStagingPlan> CreatePlan(IEnumerable<(VirtualFile src, AbsolutePath dest)> directives)
+        {
+            var top = new VirtualFile();
+            var original = directives.ToHashSet();
+            var files = directives.GroupBy(f => f.src).ToDictionary(f => f.Key);
+
+            var childrenForParent = files.Keys.GroupBy(f => f.Parent ?? top).ToDictionary(f => f.Key);
+
+            var allFilesForParent = files.Select(f => f.Key)
+                .SelectMany(f => f.FilesInFullPath)
+                .Distinct()
+                .GroupBy(f => f.Parent ?? top)
+                .ToDictionary(f => f.Key, f => f.ToArray());
+
+            var baseDirectives = new List<IStagingPlan>();
+
+            // Recursive internal function to get the plans for a given parent, we'll then start at the 
+            // null parent (the top) and recurse our way down into all the children files.
+            IEnumerable<IStagingPlan> GetPlans(VirtualFile parent)
+            {
+                foreach (var forParent in allFilesForParent[parent])
+                {
+                    // Do we need files inside this file?
+                    if (childrenForParent.TryGetValue(forParent, out var children))
+                    {
+                        if (files.TryGetValue(forParent, out var copies))
+                        {
+                            ASubStage subStage;
+                            if (parent == top)
+                            {
+                                subStage = new NativeArchive(forParent.AbsoluteName, GetPlans(forParent));
+                            }
+                            else
+                            {
+                                subStage = new SubStage(forParent.RelativeName, copies.First().dest, GetPlans(forParent));
+                            }
+                            yield return subStage;
+                            foreach (var copy in copies)
+                            {
+                                yield return new DuplicateTo(subStage, copy.dest);
+                            }
+                        }
+                        else
+                        {
+                            if (parent == top)
+                            {
+                                yield return new NativeArchive(forParent.AbsoluteName, GetPlans(forParent));
+                            }
+                            else
+                            {
+                                yield return new TempSubStage(forParent.RelativeName, GetPlans(forParent));
+                            }
+                        }
+                    }
+                    else
+                    {
+                        // If not then we need to copy this file around
+                        var copies = files[forParent];
+                        var firstCopy = new CopyTo(copies.Key.Name, copies.First().dest);
+                        yield return firstCopy;
+                        
+                        foreach (var duplicate in copies.Skip(1))
+                        {
+                            yield return new DuplicateTo(firstCopy, duplicate.dest);
+                        }
+                    }
+                }
+            }
+
+            return GetPlans(top);
+        }
+        public static async ValueTask ExecutePlan(WorkQueue queue, Func<ValueTask<Stream>> src, IEnumerable<IStagingPlan> plans)
+        {
+            // Extract these files
+
+            await using var stream = await src();
+            var sig = await FileExtractor.ArchiveSigs.MatchesAsync(stream);
+            stream.Position = 0;
+
+            switch (sig)
+            {
+                case Definitions.FileType.ZIP:
+                    await ExtractWith7Zip((Definitions.FileType)sig, stream, plans);
+
+                    
+                    break;
+                
+                default:
+                    throw new Exception($"Invalid archive for extraction");
+            }
+            
+            
+            // Copy around the duplicates
+            foreach (var file in plans.OfType<DuplicateTo>())
+            {
+                await file.Execute();
+            }
+            
+            // Execute the sub-stages
+            foreach (var subStage in plans.OfType<ISubStage>())
+            {
+                await subStage.Execute(queue);
+            }
+
+            // Dispose of all plans
+            foreach (var file in plans)
+            {
+                await file.DisposeAsync();
+            }
+
+        }
+
+        private static async ValueTask ExtractWith7Zip(Definitions.FileType sig, Stream stream, IEnumerable<IStagingPlan> plans)
+        {
+            using var archive = await ArchiveFile.Open(stream, sig);
+
+            void HandleFile(RelativePath path, IStagingPlan src, Func<Stream> sf)
+            {
+                
+            }
+            
+            var extractor = new TypedExtractor<IStagingPlan>(plans.OfType<IStagingSrc>().ToDictionary(s => s.Source.FileName, s => (IStagingPlan)s),
+                HandleFile);
+            extractor.Extract(archive);
+
+        }
+
+
+        public static async Task ExecutePlans(WorkQueue queue, IEnumerable<IStagingPlan> plans)
+        {
+            foreach (var file in plans.OfType<NativeArchive>())
+            {
+                await file.Execute(queue);
+            }
+        }
+    }
+}

Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs

@@ -0,0 +1,9 @@
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem.StagingPlan
+{
+    public interface IStagingSrc : IStagingPlan
+    {
+        public IPath Source { get; }
+    }
+}

Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs

@@ -0,0 +1,10 @@
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem.StagingPlan
+{
+    public interface ISubStage : IStagingSrc
+    {
+        Task Execute(WorkQueue queue);
+    }
+}

Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs

@@ -0,0 +1,22 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem.StagingPlan
+{
+    public class NativeArchive : ASubStage
+    {
+        public NativeArchive(AbsolutePath src, IEnumerable<IStagingPlan> plans) : base(plans)
+        {
+            Source = src;
+            Destination = src;
+        }
+
+        public override async ValueTask DisposeAsync()
+        {
+        }
+
+        public override AbsolutePath Destination { get; }
+        public override IPath Source { get; }
+    }
+}

Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs

@@ -0,0 +1,21 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem.StagingPlan
+{
+    public class SubStage : ASubStage
+    {
+        public SubStage(RelativePath src, AbsolutePath destination, IEnumerable<IStagingPlan> plans) : base(plans)
+        {
+            Source = src;
+            Destination = destination;
+        }
+        
+        public override async ValueTask DisposeAsync()
+        {
+        }
+        public override AbsolutePath Destination { get; }
+        public override IPath Source { get; }
+    }
+}

Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs

@@ -0,0 +1,25 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem.StagingPlan
+{
+    public class TempSubStage : ASubStage
+    {
+        private readonly TempFile _temp;
+
+        public TempSubStage(RelativePath src, IEnumerable<IStagingPlan> plans) : base(plans)
+        {
+            Source = src;
+            _temp = new TempFile();
+        }
+        
+        public override async ValueTask DisposeAsync()
+        {
+           await _temp.DisposeAsync();
+        }
+
+        public override AbsolutePath Destination => _temp.Path;
+        public override IPath Source { get; }
+    }
+}

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -190,34 +190,26 @@ namespace Wabbajack.VirtualFileSystem
         }
 
 
-        public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile,
+        public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile,
             IPath relPath, int depth = 0)
         {
-            var hash = await extractedFile.HashAsync();
+            await using var stream = await extractedFile.GetStream();
+            var hash = await stream.xxHashAsync();
+            stream.Position = 0;
 
-            if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension))
+            var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);
-            {
-                // Disabled because it isn't enabled on the server
-                IndexedVirtualFile result = null; //await TryGetContentsFromServer(hash);
-
-                if (result != null)
-                {
-                    Utils.Log($"Downloaded VFS data for {relPath.FileName}");
-
-
-                    return ConvertFromIndexedFile(context, result, relPath, parent, extractedFile);
-                }
-            }
             
+                /* TODO
             if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
                 return vself;
+                */
 
             var self = new VirtualFile
             {
                 Context = context,
                 Name = relPath,
                 Parent = parent,
-                Size = extractedFile.Size,
+                Size = stream.Length,
                 LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
                 LastAnalyzed = DateTime.Now.AsUnixTime(),
                 Hash = hash
@@ -226,20 +218,19 @@ namespace Wabbajack.VirtualFileSystem
             self.FillFullPath(depth);
             
             if (context.UseExtendedHashes)
-                self.ExtendedHashes = await ExtendedHashes.FromFile(extractedFile);
+                self.ExtendedHashes = await ExtendedHashes.FromStream(stream);
 
-            if (!await extractedFile.CanExtract()) return self;
+            // Can't extract, so return
+            if (!sig.HasValue) return self;
 
             try
             {
 
-                await using var extracted = await extractedFile.ExtractAll(context.Queue, throwOnError:false);
+                var list = await FileExtractor2.GatheringExtract(extractedFile, 
+                    _ => true, 
+                    async (path, sfactory) => await Analyze(context, self, sfactory, path, depth + 1));
                
-                var list = await extracted
+                self.Children = list.Values.ToImmutableList();
-                    .PMap(context.Queue,
-                        file => Analyze(context, self, file.Value, file.Key, depth + 1));
-
-                self.Children = list.ToImmutableList();
             }
             catch (Exception ex)
             {
@@ -405,10 +396,10 @@ namespace Wabbajack.VirtualFileSystem
         public string MD5 { get; set; }
         public string CRC { get; set; }
 
-        public static async ValueTask<ExtendedHashes> FromFile(IExtractedFile file)
+        public static async ValueTask<ExtendedHashes> FromStream(Stream stream)
         {
             var hashes = new ExtendedHashes();
-            await using var stream = await file.OpenRead();
+            stream.Position = 0;
             hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex();
             stream.Position = 0;
             hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex();

Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj

@@ -4,6 +4,7 @@
       <TargetFramework>netstandard2.1</TargetFramework>
       <Platforms>x64</Platforms>
       <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
+      <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     </PropertyGroup>
     <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
       <DocumentationFile>Wabbajack.VirtualFileSystem.xml</DocumentationFile>