EndToEnd Tests pass

Wabbajack.Common/StatusFileStream.cs

@@ -17,7 +17,7 @@ namespace Wabbajack.Common
             _inner = fs;
             _message = message;
             _lastUpdate = DateTime.UnixEpoch;
-            _span = TimeSpan.FromMilliseconds(500);
+            _span = TimeSpan.FromMilliseconds(100);
         }
 
         public override void Flush()

Wabbajack.Common/Wabbajack.Common.csproj

@@ -18,6 +18,7 @@
         <Version>2.2.1.2</Version>
         <AssemblyVersion>2.2.1.2</AssemblyVersion>
         <FileVersion>2.2.1.2</FileVersion>
+        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     </PropertyGroup>
     <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
       <DocumentationFile>Wabbajack.Common.xml</DocumentationFile>

Wabbajack.Lib/ACompiler.cs

@@ -315,6 +315,7 @@ namespace Wabbajack.Lib
                 .GroupBy(f => f.SourceDataFile)
                 .ToDictionary(f => f.Key);
 
+            if (grouped.Count == 0) return;
             await VFS.Extract(Queue, grouped.Keys.ToHashSet(), async (vf, sfn) =>
             {
                 await using var stream = await sfn.GetStream();

Wabbajack.Test/EndToEndTests.cs

@@ -110,9 +110,8 @@ namespace Wabbajack.Test
             var destFile = utils.DownloadsFolder.Combine(filename);
             await src.CopyToAsync(destFile);
 
-            await using var dest = await FileExtractor.ExtractAll(Queue, src);
             var modFolder = modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName);
-            await dest.MoveAllTo(modFolder);
+            await FileExtractor2.ExtractAll(src, modFolder);
             return (destFile, modFolder);
         }
 
@@ -147,9 +146,8 @@ namespace Wabbajack.Test
             await src.CopyToAsync(dest);
 
             var modFolder = utils.ModsFolder.Combine(modName);
-            await using var files = await FileExtractor.ExtractAll(Queue, src);
+            await FileExtractor2.ExtractAll(src, modFolder);
-            await files.MoveAllTo(modFolder);
+            
-
             await dest.WithExtension(Consts.MetaFileExtension).WriteAllTextAsync(ini);
             return (dest, modFolder);
         }

Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs

@@ -3,6 +3,9 @@ using System.Collections.Generic;
 using System.IO;
 using System.Threading.Tasks;
 using Compression.BSA;
+using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
+using SharpCompress.Archives.SevenZip;
+using SharpCompress.Readers;
 using Wabbajack.Common;
 using Wabbajack.Common.FileSignatures;
 using Wabbajack.VirtualFileSystem.SevenZipExtractor;
@@ -16,7 +19,8 @@ namespace Wabbajack.VirtualFileSystem
             Definitions.FileType.BA2,
             Definitions.FileType.ZIP,
             //Definitions.FileType.EXE,
-            Definitions.FileType.RAR,
+            Definitions.FileType.RAR_OLD,
+            Definitions.FileType.RAR_NEW,
             Definitions.FileType._7Z);
 
 
@@ -29,6 +33,9 @@ namespace Wabbajack.VirtualFileSystem
 
             switch (sig)
             {
+                case Definitions.FileType.RAR_OLD:
+                case Definitions.FileType.RAR_NEW:
+                case Definitions.FileType._7Z:    
                 case Definitions.FileType.ZIP:
                     return await GatheringExtractWith7Zip<T>(archive, (Definitions.FileType)sig, shouldExtract, mapfn);
                 
@@ -62,6 +69,43 @@ namespace Wabbajack.VirtualFileSystem
         private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
         {
             return await new GatheringExtractor<T>(stream, sig, shouldExtract, mapfn).Extract();
+            /*
+            IReader reader;
+            if (sig == Definitions.FileType._7Z)
+                reader = SevenZipArchive.Open(stream).ExtractAllEntries();
+            else
+            {
+                reader = ReaderFactory.Open(stream);
+            }
+
+            var results = new Dictionary<RelativePath, T>();
+            while (reader.MoveToNextEntry())
+            {
+                var path = (RelativePath)reader.Entry.Key;
+                if (!reader.Entry.IsDirectory && shouldExtract(path))
+                {
+                    var ms = new MemoryStream();
+                    reader.WriteEntryTo(ms);
+                    ms.Position = 0;
+                    var result = await mapfn(path, new MemoryStreamFactory(ms));
+                    results.Add(path, result);
+                }
+            }
+
+            return results;
+            */
+        }
+
+        public static async Task ExtractAll(AbsolutePath src, AbsolutePath dest)
+        {
+            await GatheringExtract(new NativeFileStreamFactory(src), _ => true, async (path, factory) =>
+            {
+                var abs = path.RelativeTo(dest);
+                abs.Parent.CreateDirectory();
+                await using var stream = await factory.GetStream();
+                await abs.WriteAllAsync(stream);
+                return 0;
+            });
         }
     }
 }

Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs

@@ -5,6 +5,7 @@ using System.Linq;
 using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
+using Compression.BSA;
 using Wabbajack.Common;
 using Wabbajack.Common.FileSignatures;
 using Wabbajack.VirtualFileSystem.SevenZipExtractor;
@@ -17,7 +18,7 @@ namespace Wabbajack.VirtualFileSystem
         private Predicate<RelativePath> _shouldExtract;
         private Func<RelativePath, IStreamFactory, ValueTask<T>> _mapFn;
         private Dictionary<RelativePath, T> _results;
-        private Dictionary<uint, RelativePath> _indexes;
+        private Dictionary<uint, (RelativePath, ulong)> _indexes;
         private Stream _stream;
         private Definitions.FileType _sig;
 
@@ -41,10 +42,11 @@ namespace Wabbajack.VirtualFileSystem
                 {
                     _archive = ArchiveFile.Open(_stream, _sig).Result;
                     _indexes = _archive.Entries
-                        .Where(f => !f.IsFolder)
+                        .Select((entry, idx) => (entry, (uint)idx))
-                        .Select((entry, idx) => ((RelativePath)entry.FileName, (uint)idx))
+                        .Where(f => !f.entry.IsFolder)
+                        .Select(t => ((RelativePath)t.entry.FileName, t.Item2, t.entry.Size))
                         .Where(t => _shouldExtract(t.Item1))
-                        .ToDictionary(t => t.Item2, t => t.Item1);
+                        .ToDictionary(t => t.Item2, t => (t.Item1, t.Size));
 
 
                     _archive._archive.Extract(null, 0xFFFFFFFF, 0, this);
@@ -101,27 +103,52 @@ namespace Wabbajack.VirtualFileSystem
         {
             private GatheringExtractor<T> _extractor;
             private uint _index;
+            private bool _written;
+            private ulong _totalSize;
+            private MemoryStream _tmpStream;
 
             public GatheringExtractorStream(GatheringExtractor<T> extractor, uint index)
             {
                 _extractor = extractor;
                 _index = index;
+                _written = false;
+                _totalSize = extractor._indexes[index].Item2;
+                _tmpStream = new MemoryStream();
             }
 
             public int Write(IntPtr data, uint size, IntPtr processedSize)
             {
                 unsafe
                 {
-                    var result = _extractor._mapFn(_extractor._indexes[_index], new UnmanagedStreamFactory((byte*)data, size)).AsTask().Result;
+                    var ums = new UnmanagedMemoryStream((byte*)data, size);
+                    ums.CopyTo(_tmpStream);
+                    if ((ulong)_tmpStream.Length >= _totalSize)
+                    {
+                        _tmpStream.Position = 0;
+                        var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryStreamFactory(_tmpStream)).AsTask().Result;
 
-                    _extractor._results[_extractor._indexes[_index]] = result;
+                        _extractor._results[_extractor._indexes[_index].Item1] = result;
-                
+                    }
+                    
                     if (processedSize != IntPtr.Zero)
                     {
                         Marshal.WriteInt32(processedSize, (int) size);
                     }
+                }
+
+                return 0;
+                
+                if (_written) throw new Exception("TODO");
+                unsafe
+                {
+
+                
+
+
+                    _written = true;
 
                     return 0;
+                    
                 }
             }
 

Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs

@@ -91,8 +91,8 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
         {
             {Definitions.FileType._7Z, new Guid("23170f69-40c1-278a-1000-000110070000")},
             {Definitions.FileType.BZ2, new Guid("23170f69-40c1-278a-1000-000110020000")},
-            {Definitions.FileType.RAR_NEW, new Guid("23170f69-40c1-278a-1000-000110030000")},
+            {Definitions.FileType.RAR_OLD, new Guid("23170f69-40c1-278a-1000-000110030000")},
-            {Definitions.FileType.RAR_OLD, new Guid("23170f69-40c1-278a-1000-000110CC0000")},
+            {Definitions.FileType.RAR_NEW, new Guid("23170f69-40c1-278a-1000-000110CC0000")},
             {Definitions.FileType.ZIP, new Guid("23170f69-40c1-278a-1000-000110010000")},
         };
 

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -5,6 +5,7 @@ using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Threading.Tasks;
+using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
 using K4os.Hash.Crc;
 using RocksDbSharp;
 using Wabbajack.Common;
@@ -198,8 +199,9 @@ namespace Wabbajack.VirtualFileSystem
             stream.Position = 0;
 
             var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);
+            stream.Position = 0;
             
-            if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
+            if (sig.HasValue && TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
                 return vself;
             
             var self = new VirtualFile

Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj

@@ -17,6 +17,7 @@
       <PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
       <PackageReference Include="K4os.Hash.Crc" Version="1.1.4" />
       <PackageReference Include="OMODFramework" Version="2.0.1" />
+      <PackageReference Include="SharpCompress" Version="0.26.0" />
       <PackageReference Include="System.Collections.Immutable" Version="5.0.0-preview.6.20305.6" />
     </ItemGroup>
     <ItemGroup>