Merge pull request #1116 from wabbajack-tools/extraction-fixes

Extraction fixes

CHANGELOG.md

@@ -1,5 +1,8 @@
 ### Changelog
 
+#### Version - 2.3.0.2 - 10/5/2020
+* Fixed a situation where 7zip would refuse to extract very large archives
+
 #### Version - 2.3.0.1 - 10/4/2020
 * Rewrote the file extraction routines. New code uses less memory, less disk space, and performs less thrashing on HDDs
 * Reworked IPS4 integration to reduce download failures

Wabbajack.CLI/Wabbajack.CLI.csproj

@@ -6,8 +6,8 @@
         <AssemblyName>wabbajack-cli</AssemblyName>
         <Company>Wabbajack</Company>
         <Platforms>x64</Platforms>
-        <AssemblyVersion>2.3.0.1</AssemblyVersion>
+        <AssemblyVersion>2.3.0.2</AssemblyVersion>
-        <FileVersion>2.3.0.1</FileVersion>
+        <FileVersion>2.3.0.2</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>An automated ModList installer</Description>
         <PublishReadyToRun>true</PublishReadyToRun>        

Wabbajack.Launcher/Wabbajack.Launcher.csproj

@@ -4,8 +4,8 @@
         <OutputType>WinExe</OutputType>
         <TargetFramework>netcoreapp3.1</TargetFramework>
         <UseWPF>true</UseWPF>
-        <AssemblyVersion>2.3.0.1</AssemblyVersion>
+        <AssemblyVersion>2.3.0.2</AssemblyVersion>
-        <FileVersion>2.3.0.1</FileVersion>
+        <FileVersion>2.3.0.2</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Application Launcher</Description>
         <PublishReadyToRun>true</PublishReadyToRun>

Wabbajack.Lib/MO2Compiler.cs

@@ -557,7 +557,17 @@ namespace Wabbajack.Lib
 
             var firstFailedPatch = InstallDirectives.OfType<PatchedFromArchive>().FirstOrDefault(f => f.PatchID == default);
             if (firstFailedPatch != null)
-                Error($"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}");
+            {
+                Utils.Log($"Missing data from failed patch, starting data dump");
+                Utils.Log($"Dest File: {firstFailedPatch.To}");
+                Utils.Log($"Options ({firstFailedPatch.Choices.Length}:");
+                foreach (var choice in firstFailedPatch.Choices)
+                {
+                    Utils.Log($"  {choice.FullPath}");
+                }
+                Error(
+                    $"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}");
+            }
         }
 
         private VirtualFile FindDestFile(RelativePath to)

Wabbajack.Lib/MO2Installer.cs

@@ -325,6 +325,8 @@ namespace Wabbajack.Lib
                 await a.Build(OutputFolder.Combine(bsa.To));
                 streams.Do(s => s.Dispose());
 
+                await sourceDir.DeleteDirectory();
+
                 if (UseCompression)
                     await OutputFolder.Combine(bsa.To).Compact(FileCompaction.Algorithm.XPRESS16K);
             }

Wabbajack.Server/Wabbajack.Server.csproj

@@ -3,8 +3,8 @@
     <PropertyGroup>
         <OutputType>Exe</OutputType>
         <TargetFramework>netcoreapp3.1</TargetFramework>
-        <AssemblyVersion>2.3.0.1</AssemblyVersion>
+        <AssemblyVersion>2.3.0.2</AssemblyVersion>
-        <FileVersion>2.3.0.1</FileVersion>
+        <FileVersion>2.3.0.2</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Server</Description>
         <RuntimeIdentifier>win-x64</RuntimeIdentifier>

Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs

@@ -95,6 +95,27 @@ namespace Wabbajack.VirtualFileSystem.Test
             }
         }
 
+        /* Takes to long to run in CI, enable when needed for verification
+
+        [Fact]
+        public async Task MissingFileFromArchiveShouldBeFound()
+        {
+            FileExtractor2.FavorPerfOverRAM = true;
+
+            // From a bug in 2.3.0.1
+            var src = await DownloadMod(Game.SkyrimSpecialEdition, 21166, 136259);
+                        
+            var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(src), 
+                f => true,
+                async (p, s) =>
+                {
+                    await using var stream = await s.GetStream();
+                    return stream.Length;
+                });
+            
+            Assert.NotEmpty(results);
+        }
+*/
 
         [Fact]
         public async Task CanExtractEmptyFiles()

Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs

@@ -72,7 +72,7 @@ namespace Wabbajack.VirtualFileSystem
                     }
                     else
                     {
-                        return await GatheringExtractWith7Zip<T>(archive, (Definitions.FileType)sig, shouldExtract,
+                        return await GatheringExtractWith7Zip<T>(sFn, (Definitions.FileType)sig, shouldExtract,
                             mapfn);
                     }
                 }
@@ -157,9 +157,9 @@ namespace Wabbajack.VirtualFileSystem
             return results;
         }
 
-        private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
+        private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(IStreamFactory sf, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
         {
-            return await new GatheringExtractor<T>(stream, sig, shouldExtract, mapfn).Extract();
+            return await new GatheringExtractor<T>(sf, sig, shouldExtract, mapfn).Extract();
         }
 
         public static async Task ExtractAll(AbsolutePath src, AbsolutePath dest)

Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs

@@ -2,12 +2,14 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Reactive.Linq;
 using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
 using Compression.BSA;
 using Wabbajack.Common;
 using Wabbajack.Common.FileSignatures;
+using Wabbajack.Common.StatusFeed.Errors;
 using Wabbajack.VirtualFileSystem.SevenZipExtractor;
 
 namespace Wabbajack.VirtualFileSystem
@@ -18,17 +20,17 @@ namespace Wabbajack.VirtualFileSystem
         private Predicate<RelativePath> _shouldExtract;
         private Func<RelativePath, IStreamFactory, ValueTask<T>> _mapFn;
         private Dictionary<RelativePath, T> _results;
-        private Stream _stream;
         private Definitions.FileType _sig;
         private Exception _killException;
         private uint _itemsCount;
+        private IStreamFactory _streamFactory;
 
-        public GatheringExtractor(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory, ValueTask<T>> mapfn)
+        public GatheringExtractor(IStreamFactory sF, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory, ValueTask<T>> mapfn)
         {
             _shouldExtract = shouldExtract;
             _mapFn = mapfn;
             _results = new Dictionary<RelativePath, T>();
-            _stream = stream;
+            _streamFactory = sF;
             _sig = sig;
         }
 
@@ -40,11 +42,18 @@ namespace Wabbajack.VirtualFileSystem
             {
                 try
                 {
-                    _archive = ArchiveFile.Open(_stream, _sig).Result;
+                    using var stream = _streamFactory.GetStream().Result;
-                    ulong checkPos = 1024 * 32;
+                    _archive = ArchiveFile.Open(stream, _sig).Result;
-                    _archive._archive.Open(_archive._archiveStream, ref checkPos, new ArchiveCallback());
+                    ulong checkPos = (ulong)stream.Length;
+                    var oresult = _archive._archive.Open(_archive._archiveStream, ref checkPos, new ArchiveCallback());
+                    // Can't read this with the COM interface for some reason
+                    if (oresult != 0)
+                    {
+                        var _ = ExtractSlow(source, _streamFactory);
+                        return;
+                    }
                     _itemsCount = _archive._archive.GetNumberOfItems();
-                    _archive._archive.Extract(null, 0xFFFFFFFF, 0, this);
+                    var result = _archive._archive.Extract(null, 0xFFFFFFFF, 0, this);
                     _archive.Dispose();
                     if (_killException != null)
                     {
@@ -69,6 +78,82 @@ namespace Wabbajack.VirtualFileSystem
             return _results;
         }
 
+        private async Task ExtractSlow(TaskCompletionSource<bool> tcs, IStreamFactory streamFactory)
+        {
+            try
+            {
+                TempFile tempFile = null;
+                AbsolutePath source;
+                if (streamFactory is NativeFileStreamFactory nsf)
+                {
+                    source = (AbsolutePath)nsf.Name;
+                }
+                else
+                {
+                    await using var stream = await streamFactory.GetStream();
+                    tempFile = new TempFile();
+                    await tempFile.Path.WriteAllAsync(stream);
+                }
+
+                var dest = await TempFolder.Create();
+                Utils.Log(
+                    $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe");
+
+                var process = new ProcessHelper {Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),};
+
+
+                process.Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, "-mmt=off"};
+
+
+                var _ = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
+                    .ForEachAsync(p =>
+                    {
+                        var (_, line) = p;
+                        if (line == null)
+                            return;
+
+                        if (line.Length <= 4 || line[3] != '%') return;
+
+                        int.TryParse(line.Substring(0, 3), out var percentInt);
+                        Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}",
+                            Percent.FactoryPutInRange(percentInt / 100d));
+                    });
+
+                var exitCode = await process.Start();
+
+
+                if (exitCode != 0)
+                {
+                    Utils.ErrorThrow(new _7zipReturnError(exitCode, source, dest.Dir, ""));
+                }
+                else
+                {
+                    Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true);
+                }
+
+                if (tempFile != null)
+                {
+                    await tempFile.DisposeAsync();
+                }
+
+                foreach (var file in dest.Dir.EnumerateFiles())
+                {
+                    var relPath = file.RelativeTo(dest.Dir);
+                    if (!_shouldExtract(relPath)) continue;
+
+                    var result = await _mapFn(relPath, new NativeFileStreamFactory(file));
+                    _results[relPath] = result;
+                    await file.DeleteAsync();
+                }
+
+                tcs.SetResult(true);
+            }
+            catch (Exception ex)
+            {
+                tcs.SetException(ex);
+            }
+        }       
+
         public void SetTotal(ulong total)
         {
             

Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs

@@ -29,6 +29,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
             if (!FileExtractor2.FavorPerfOverRAM)
             {
                 self.SetCompressionProperties(new Dictionary<string, string>() {{"mt", "off"}});
+
             }
 
 
@@ -130,38 +131,6 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
 
             }
         }
-        
-        public IList<Entry> Entries
-        {
-            get
-            {
-                if (this._entries != null)
-                {
-                    return this._entries;
-                }
-
-                ulong checkPos = 32 * 1024;
-                int open = this._archive.Open(this._archiveStream, ref checkPos, null);
-
-                if (open != 0)
-                {
-                    throw new Exception("Unable to open archive");
-                }
-
-                uint itemsCount = this._archive.GetNumberOfItems();
-
-                this._entries = new List<Entry>();
-
-                for (uint fileIndex = 0; fileIndex < itemsCount; fileIndex++)
-                {
-                    var entry = GetEntry(fileIndex);
-                    this._entries.Add(entry);
-                }
-
-                return this._entries;
-            }
-        }
-
         internal Entry GetEntry(uint fileIndex)
         {
             string fileName = this.GetProperty<string>(fileIndex, ItemPropId.kpidPath);

Wabbajack/Wabbajack.csproj

@@ -6,8 +6,8 @@
     <UseWPF>true</UseWPF>
     <Platforms>x64</Platforms>
     <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
-      <AssemblyVersion>2.3.0.1</AssemblyVersion>
+      <AssemblyVersion>2.3.0.2</AssemblyVersion>
-      <FileVersion>2.3.0.1</FileVersion>
+      <FileVersion>2.3.0.2</FileVersion>
     <Copyright>Copyright © 2019-2020</Copyright>
     <Description>An automated ModList installer</Description>
     <PublishReadyToRun>true</PublishReadyToRun>