diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index 9d27a593..937e642e 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -71,6 +71,7 @@ internal class Program services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); }).Build(); diff --git a/Wabbajack.CLI/Verbs/DumpZipInfo.cs b/Wabbajack.CLI/Verbs/DumpZipInfo.cs new file mode 100644 index 00000000..3bd7b5d1 --- /dev/null +++ b/Wabbajack.CLI/Verbs/DumpZipInfo.cs @@ -0,0 +1,57 @@ +using System; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.Common; +using Wabbajack.Compression.Zip; +using Wabbajack.Downloaders; +using Wabbajack.DTOs; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; + +namespace Wabbajack.CLI.Verbs; + +public class DumpZipInfo : IVerb +{ + private readonly ILogger _logger; + + public DumpZipInfo(ILogger logger) + { + _logger = logger; + } + + public Command MakeCommand() + { + var command = new Command("dump-zip-info"); + command.Add(new Option(new[] {"-i", "-input"}, "Zip file ot parse")); + command.Add(new Option(new[] {"-t", "-test"}, "Test extracting each file")); + command.Description = "Dumps the contents of a zip file to the console, for use in debugging wabbajack files"; + command.Handler = CommandHandler.Create(Run); + return command; + } + + private async Task Run(AbsolutePath input, bool test) + { + await using var ar = new ZipReader(input.Open(FileMode.Open), false); + foreach (var value in (await ar.GetFiles())) + { + if (test) + { + _logger.LogInformation("Extracting {File}", value.FileName); + await ar.Extract(value, new MemoryStream(), CancellationToken.None); + } + else + { + + _logger.LogInformation("Read {File}", value.FileName); + } + } + + return 0; + + } +} diff --git a/Wabbajack.Compression.Zip/ExtractedEntry.cs b/Wabbajack.Compression.Zip/ExtractedEntry.cs index 3a96e767..fed4d841 100644 --- a/Wabbajack.Compression.Zip/ExtractedEntry.cs +++ b/Wabbajack.Compression.Zip/ExtractedEntry.cs @@ -5,6 +5,6 @@ public class ExtractedEntry public long FileOffset { get; set; } public string FileName { get; set; } public long CompressedSize { get; set; } - public uint UncompressedSize { get; set; } + public long UncompressedSize { get; set; } public short CompressionMethod { get; set; } } \ No newline at end of file diff --git a/Wabbajack.Compression.Zip/ZipReader.cs b/Wabbajack.Compression.Zip/ZipReader.cs index 87b1b798..cbd09d8f 100644 --- a/Wabbajack.Compression.Zip/ZipReader.cs +++ b/Wabbajack.Compression.Zip/ZipReader.cs @@ -1,6 +1,7 @@ using System.IO.Compression; using System.Text; using Wabbajack.IO.Async; +using static System.UInt32; namespace Wabbajack.Compression.Zip; @@ -11,7 +12,9 @@ public class ZipReader : IAsyncDisposable private readonly bool _leaveOpen; private const uint EndOfCentralDirectoryRecordSignature = 0x06054b50; + private const uint EndOfCentralDirectoryOffsetSignature64 = 0x7064b50; private const uint CentralDirectoryFileHeaderSignature = 0x02014b50; + private const uint EndOfCentralDirectorySignature64 = 0x6064b50; public ZipReader(Stream s, bool leaveOpen = false) { @@ -20,9 +23,8 @@ public class ZipReader : IAsyncDisposable _rdr = new AsyncBinaryReader(s); } - public async Task GetFiles() + public async Task<(long sigOffset, uint TotalRecords, long CDOffset)> ReadZip32EODR(long sigOffset) { - var sigOffset = 0; while (true) { _rdr.Position = _rdr.Length - 22 - sigOffset; @@ -31,6 +33,8 @@ public class ZipReader : IAsyncDisposable sigOffset++; } + var hdrOffset = _rdr.Position - 4; + if (await _rdr.ReadUInt16() != 0) { throw new NotImplementedException("Only single disk archives are supported"); @@ -47,6 +51,72 @@ public class ZipReader : IAsyncDisposable var sizeOfCentralDirectory = await _rdr.ReadUInt32(); var centralDirectoryOffset = await _rdr.ReadUInt32(); + return (hdrOffset, totalCentralDirectoryRecords, centralDirectoryOffset); + } + + public async Task<(long sigOffset, uint TotalRecords, long CDOffset)> ReadZip64EODR(long sigOffset) + { + while (true) + { + _rdr.Position = sigOffset; + if (await _rdr.ReadUInt32() == EndOfCentralDirectoryOffsetSignature64) + break; + sigOffset--; + } + + var hdrOffset = sigOffset - 4; + + if (await _rdr.ReadUInt32() != 0) + { + throw new NotImplementedException("Only single disk archives are supported"); + } + + var ecodOffset = await _rdr.ReadUInt64(); + + _rdr.Position = (long)ecodOffset; + if (await _rdr.ReadUInt32() != EndOfCentralDirectorySignature64) + throw new Exception("Corrupt Zip64 structure, can't find EOCD"); + + var sizeOfECDR = await _rdr.ReadUInt64(); + + _rdr.Position += 4; // Skip version info + + if (await _rdr.ReadUInt32() != 0) + { + throw new NotImplementedException("Only single disk archives are supported"); + } + + if (await _rdr.ReadInt32() != 0) + { + throw new NotImplementedException("Only single disk archives are supported"); + } + + var recordsOnDisk = await _rdr.ReadUInt64(); + var totalRecords = await _rdr.ReadUInt64(); + + if (recordsOnDisk != totalRecords) + { + throw new NotImplementedException("Only single disk archives are supported"); + } + + var sizeOfCD = await _rdr.ReadUInt64(); + var cdOffset = await _rdr.ReadUInt64(); + + + + return (hdrOffset, (uint)totalRecords, (long)cdOffset); + } + + public async Task GetFiles() + { + var (sigOffset, totalCentralDirectoryRecords, centralDirectoryOffset) = await ReadZip32EODR(0); + var isZip64 = false; + if (centralDirectoryOffset == uint.MaxValue) + { + isZip64 = true; + (sigOffset, totalCentralDirectoryRecords, centralDirectoryOffset) = await ReadZip64EODR(sigOffset); + } + _rdr.Position = centralDirectoryOffset; @@ -61,17 +131,41 @@ public class ZipReader : IAsyncDisposable var compressionMethod = await _rdr.ReadInt16(); _rdr.Position += 4; var crc = await _rdr.ReadUInt32(); - var compressedSize = await _rdr.ReadUInt32(); + long compressedSize = await _rdr.ReadUInt32(); - var uncompressedSize = await _rdr.ReadUInt32(); + long uncompressedSize = await _rdr.ReadUInt32(); var fileNameLength = await _rdr.ReadUInt16(); var extraFieldLength = await _rdr.ReadUInt16(); var fileCommentLength = await _rdr.ReadUInt16(); _rdr.Position += 8; - var fileOffset = await _rdr.ReadUInt32(); - var fileName = await _rdr.ReadFixedSizeString(fileNameLength, Encoding.UTF8); + long fileOffset = await _rdr.ReadUInt32(); - _rdr.Position += extraFieldLength + fileCommentLength; + var fileName = await _rdr.ReadFixedSizeString(fileNameLength, Encoding.UTF8); + + + _rdr.Position += fileCommentLength; + + if (compressedSize == uint.MaxValue || uncompressedSize == uint.MaxValue || fileOffset == uint.MaxValue) + { + if (await _rdr.ReadUInt16() != 0x1) + { + throw new Exception("Non Zip64 extra fields not implemented"); + } + var size = await _rdr.ReadUInt16(); + for (var x = 0; x < size / 8; x++) + { + var value = await _rdr.ReadUInt64(); + if (compressedSize == uint.MaxValue) + compressedSize = (long)value; + else if (uncompressedSize == uint.MaxValue) + uncompressedSize = (long) value; + else if (fileOffset == (long) uint.MaxValue) + fileOffset = (long) value; + else + throw new Exception("Bad zip format"); + + } + } entries[i] = new ExtractedEntry { @@ -93,15 +187,14 @@ public class ZipReader : IAsyncDisposable _stream.Position = entry.FileOffset; _stream.Position += 6; var flags = await _rdr.ReadUInt16(); - _stream.Position += 18; + bool isZip64 = ((flags & (0x1 << 3)) != 0); + + + _stream.Position += (isZip64 ? 18 : 18); var fnLength = await _rdr.ReadUInt16(); var efLength = await _rdr.ReadUInt16(); _stream.Position += fnLength + efLength; - if (flags != 0) - throw new NotImplementedException("Don't know how to handle flags yet"); - - switch (entry.CompressionMethod) { case 0: