Add basic zip64 support so we can handle zips larger than 4GB

Wabbajack.CLI/Program.cs

@@ -71,6 +71,7 @@ internal class Program
                 services.AddSingleton<IVerb, ListCreationClubContent>();
                 services.AddSingleton<IVerb, ListModlists>();
                 services.AddSingleton<IVerb, Extract>();
+                services.AddSingleton<IVerb, DumpZipInfo>();
 
                 services.AddSingleton<IUserInterventionHandler, UserInterventionHandler>();
             }).Build();

Wabbajack.CLI/Verbs/DumpZipInfo.cs

@@ -0,0 +1,57 @@
+using System;
+using System.CommandLine;
+using System.CommandLine.Invocation;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.Common;
+using Wabbajack.Compression.Zip;
+using Wabbajack.Downloaders;
+using Wabbajack.DTOs;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+
+namespace Wabbajack.CLI.Verbs;
+
+public class DumpZipInfo : IVerb
+{
+    private readonly ILogger<DumpZipInfo> _logger;
+
+    public DumpZipInfo(ILogger<DumpZipInfo> logger)
+    {
+        _logger = logger;
+    }
+
+    public Command MakeCommand()
+    {
+        var command = new Command("dump-zip-info");
+        command.Add(new Option<AbsolutePath>(new[] {"-i", "-input"}, "Zip file ot parse"));
+        command.Add(new Option<bool>(new[] {"-t", "-test"}, "Test extracting each file"));
+        command.Description = "Dumps the contents of a zip file to the console, for use in debugging wabbajack files";
+        command.Handler = CommandHandler.Create(Run);
+        return command;
+    }
+
+    private async Task<int> Run(AbsolutePath input, bool test)
+    {
+        await using var ar = new ZipReader(input.Open(FileMode.Open), false);
+        foreach (var value in (await ar.GetFiles()))
+        {
+            if (test)
+            {
+                _logger.LogInformation("Extracting {File}", value.FileName);
+                await ar.Extract(value, new MemoryStream(), CancellationToken.None);
+            }
+            else
+            {
+                
+                _logger.LogInformation("Read {File}", value.FileName);
+            }
+        }
+
+        return 0;
+
+    }
+}

Wabbajack.Compression.Zip/ExtractedEntry.cs

@@ -5,6 +5,6 @@ public class ExtractedEntry
     public long FileOffset { get; set; }
     public string FileName { get; set; }
     public long CompressedSize { get; set; }
-    public uint UncompressedSize { get; set; }
+    public long UncompressedSize { get; set; }
     public short CompressionMethod { get; set; }
 }

Wabbajack.Compression.Zip/ZipReader.cs

@@ -1,6 +1,7 @@
 using System.IO.Compression;
 using System.Text;
 using Wabbajack.IO.Async;
+using static System.UInt32;
 
 namespace Wabbajack.Compression.Zip;
 
@@ -11,7 +12,9 @@ public class ZipReader : IAsyncDisposable
     private readonly bool _leaveOpen;
 
     private const uint EndOfCentralDirectoryRecordSignature = 0x06054b50;
+    private const uint EndOfCentralDirectoryOffsetSignature64 = 0x7064b50;
     private const uint CentralDirectoryFileHeaderSignature = 0x02014b50;
+    private const uint EndOfCentralDirectorySignature64 = 0x6064b50;
 
     public ZipReader(Stream s, bool leaveOpen = false)
     {
@@ -20,9 +23,8 @@ public class ZipReader : IAsyncDisposable
         _rdr = new AsyncBinaryReader(s);
     }
 
-    public async Task<ExtractedEntry[]> GetFiles()
+    public async Task<(long sigOffset, uint TotalRecords, long CDOffset)> ReadZip32EODR(long sigOffset)
     {
-        var sigOffset = 0;
         while (true)
         {
             _rdr.Position = _rdr.Length - 22 - sigOffset;
@@ -31,6 +33,8 @@ public class ZipReader : IAsyncDisposable
             sigOffset++;
         }
 
+        var hdrOffset = _rdr.Position - 4;
+
         if (await _rdr.ReadUInt16() != 0)
         {
             throw new NotImplementedException("Only single disk archives are supported");
@@ -47,6 +51,72 @@ public class ZipReader : IAsyncDisposable
         var sizeOfCentralDirectory = await _rdr.ReadUInt32();
         var centralDirectoryOffset = await _rdr.ReadUInt32();
 
+        return (hdrOffset, totalCentralDirectoryRecords, centralDirectoryOffset);
+    }
+    
+    public async Task<(long sigOffset, uint TotalRecords, long CDOffset)> ReadZip64EODR(long sigOffset)
+    {
+        while (true)
+        {
+            _rdr.Position = sigOffset;
+            if (await _rdr.ReadUInt32() == EndOfCentralDirectoryOffsetSignature64)
+                break;
+            sigOffset--;
+        }
+
+        var hdrOffset = sigOffset - 4;
+
+        if (await _rdr.ReadUInt32() != 0)
+        {
+            throw new NotImplementedException("Only single disk archives are supported");
+        }
+
+        var ecodOffset = await _rdr.ReadUInt64();
+
+        _rdr.Position = (long)ecodOffset;
+        if (await _rdr.ReadUInt32() != EndOfCentralDirectorySignature64)
+            throw new Exception("Corrupt Zip64 structure, can't find EOCD");
+
+        var sizeOfECDR = await _rdr.ReadUInt64();
+
+        _rdr.Position += 4; // Skip version info
+        
+        if (await _rdr.ReadUInt32() != 0)
+        {
+            throw new NotImplementedException("Only single disk archives are supported");
+        }
+
+        if (await _rdr.ReadInt32() != 0)
+        {
+            throw new NotImplementedException("Only single disk archives are supported");
+        }
+
+        var recordsOnDisk = await _rdr.ReadUInt64();
+        var totalRecords = await _rdr.ReadUInt64();
+        
+        if (recordsOnDisk != totalRecords)
+        {
+            throw new NotImplementedException("Only single disk archives are supported");
+        }
+        
+        var sizeOfCD = await _rdr.ReadUInt64();
+        var cdOffset = await _rdr.ReadUInt64();
+        
+        
+        
+        return (hdrOffset, (uint)totalRecords, (long)cdOffset);
+    }
+
+    public async Task<ExtractedEntry[]> GetFiles()
+    {
+        var (sigOffset, totalCentralDirectoryRecords, centralDirectoryOffset) = await ReadZip32EODR(0);
+        var isZip64 = false;
+        if (centralDirectoryOffset == uint.MaxValue)
+        {
+            isZip64 = true;
+            (sigOffset, totalCentralDirectoryRecords, centralDirectoryOffset) = await ReadZip64EODR(sigOffset);
+        }
+        
 
         _rdr.Position = centralDirectoryOffset;
 
@@ -61,17 +131,41 @@ public class ZipReader : IAsyncDisposable
             var compressionMethod = await _rdr.ReadInt16();
             _rdr.Position += 4;
             var crc = await _rdr.ReadUInt32();
-            var compressedSize = await _rdr.ReadUInt32();
+            long compressedSize = await _rdr.ReadUInt32();
             
-            var uncompressedSize = await _rdr.ReadUInt32();
+            long uncompressedSize = await _rdr.ReadUInt32();
             var fileNameLength = await _rdr.ReadUInt16();
             var extraFieldLength = await _rdr.ReadUInt16();
             var fileCommentLength = await _rdr.ReadUInt16();
             _rdr.Position += 8;
-            var fileOffset = await _rdr.ReadUInt32();
+            long fileOffset = await _rdr.ReadUInt32();
+
             var fileName = await _rdr.ReadFixedSizeString(fileNameLength, Encoding.UTF8);
             
-            _rdr.Position += extraFieldLength + fileCommentLength;
+
+            _rdr.Position += fileCommentLength;
+
+            if (compressedSize == uint.MaxValue || uncompressedSize == uint.MaxValue || fileOffset == uint.MaxValue)
+            {
+                if (await _rdr.ReadUInt16() != 0x1)
+                {
+                    throw new Exception("Non Zip64 extra fields not implemented");
+                }
+                var size = await _rdr.ReadUInt16();
+                for (var x = 0; x < size / 8; x++)
+                {
+                    var value = await _rdr.ReadUInt64();
+                    if (compressedSize == uint.MaxValue)
+                        compressedSize = (long)value;
+                    else if (uncompressedSize == uint.MaxValue)
+                        uncompressedSize = (long) value;
+                    else if (fileOffset == (long) uint.MaxValue)
+                        fileOffset = (long) value;
+                    else
+                        throw new Exception("Bad zip format");
+                    
+                }
+            }
 
             entries[i] = new ExtractedEntry
             {
@@ -93,15 +187,14 @@ public class ZipReader : IAsyncDisposable
         _stream.Position = entry.FileOffset;
         _stream.Position += 6;
         var flags = await _rdr.ReadUInt16();
-        _stream.Position += 18;
+        bool isZip64 = ((flags & (0x1 << 3)) != 0);
+
+        
+        _stream.Position += (isZip64 ? 18 : 18);
         var fnLength = await _rdr.ReadUInt16();
         var efLength = await _rdr.ReadUInt16();
         _stream.Position += fnLength + efLength;
 
-        if (flags != 0)
-            throw new NotImplementedException("Don't know how to handle flags yet");
-        
-        
         switch (entry.CompressionMethod)
         {
             case 0: