Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add basic zip64 support so we can handle zips larger than 4GB

Browse Source
This commit is contained in:
Timothy Baldridge 2022-05-14 08:18:03 -06:00
parent 6c4bfdfdb6
commit 087c33da6d
4 changed files with 164 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Wabbajack.CLI/Program.cs
View File

@ -71,6 +71,7 @@ internal class Program
services.AddSingleton<IVerb, ListCreationClubContent>();
services.AddSingleton<IVerb, ListModlists>();
services.AddSingleton<IVerb, Extract>();
services.AddSingleton<IVerb, DumpZipInfo>();
services.AddSingleton<IUserInterventionHandler, UserInterventionHandler>();
}).Build();

57
Wabbajack.CLI/Verbs/DumpZipInfo.cs Normal file
View File

@ -0,0 +1,57 @@
using System;
using System.CommandLine;
using System.CommandLine.Invocation;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Compression.Zip;
using Wabbajack.Downloaders;
using Wabbajack.DTOs;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
namespace Wabbajack.CLI.Verbs;
public class DumpZipInfo : IVerb
{
private readonly ILogger<DumpZipInfo> _logger;
public DumpZipInfo(ILogger<DumpZipInfo> logger)
{
_logger = logger;
}
public Command MakeCommand()
{
var command = new Command("dump-zip-info");
command.Add(new Option<AbsolutePath>(new[] {"-i", "-input"}, "Zip file ot parse"));
command.Add(new Option<bool>(new[] {"-t", "-test"}, "Test extracting each file"));
command.Description = "Dumps the contents of a zip file to the console, for use in debugging wabbajack files";
command.Handler = CommandHandler.Create(Run);
return command;
}
private async Task<int> Run(AbsolutePath input, bool test)
{
await using var ar = new ZipReader(input.Open(FileMode.Open), false);
foreach (var value in (await ar.GetFiles()))
{
if (test)
{
_logger.LogInformation("Extracting {File}", value.FileName);
await ar.Extract(value, new MemoryStream(), CancellationToken.None);
}
else
{
_logger.LogInformation("Read {File}", value.FileName);
}
}
return 0;
}
}

2
Wabbajack.Compression.Zip/ExtractedEntry.cs
View File

@ -5,6 +5,6 @@ public class ExtractedEntry
public long FileOffset { get; set; }
public string FileName { get; set; }
public long CompressedSize { get; set; }
public uint UncompressedSize { get; set; }
public long UncompressedSize { get; set; }
public short CompressionMethod { get; set; }
}

115
Wabbajack.Compression.Zip/ZipReader.cs
View File

@ -1,6 +1,7 @@
using System.IO.Compression;
using System.Text;
using Wabbajack.IO.Async;
using static System.UInt32;
namespace Wabbajack.Compression.Zip;
@ -11,7 +12,9 @@ public class ZipReader : IAsyncDisposable
private readonly bool _leaveOpen;
private const uint EndOfCentralDirectoryRecordSignature = 0x06054b50;
private const uint EndOfCentralDirectoryOffsetSignature64 = 0x7064b50;
private const uint CentralDirectoryFileHeaderSignature = 0x02014b50;
private const uint EndOfCentralDirectorySignature64 = 0x6064b50;
public ZipReader(Stream s, bool leaveOpen = false)
{
@ -20,9 +23,8 @@ public class ZipReader : IAsyncDisposable
_rdr = new AsyncBinaryReader(s);
}
public async Task<ExtractedEntry[]> GetFiles()
public async Task<(long sigOffset, uint TotalRecords, long CDOffset)> ReadZip32EODR(long sigOffset)
{
var sigOffset = 0;
while (true)
{
_rdr.Position = _rdr.Length - 22 - sigOffset;
@ -31,6 +33,8 @@ public class ZipReader : IAsyncDisposable
sigOffset++;
}
var hdrOffset = _rdr.Position - 4;
if (await _rdr.ReadUInt16() != 0)
{
throw new NotImplementedException("Only single disk archives are supported");
@ -47,6 +51,72 @@ public class ZipReader : IAsyncDisposable
var sizeOfCentralDirectory = await _rdr.ReadUInt32();
var centralDirectoryOffset = await _rdr.ReadUInt32();
return (hdrOffset, totalCentralDirectoryRecords, centralDirectoryOffset);
}
public async Task<(long sigOffset, uint TotalRecords, long CDOffset)> ReadZip64EODR(long sigOffset)
{
while (true)
{
_rdr.Position = sigOffset;
if (await _rdr.ReadUInt32() == EndOfCentralDirectoryOffsetSignature64)
break;
sigOffset--;
}
var hdrOffset = sigOffset - 4;
if (await _rdr.ReadUInt32() != 0)
{
throw new NotImplementedException("Only single disk archives are supported");
}
var ecodOffset = await _rdr.ReadUInt64();
_rdr.Position = (long)ecodOffset;
if (await _rdr.ReadUInt32() != EndOfCentralDirectorySignature64)
throw new Exception("Corrupt Zip64 structure, can't find EOCD");
var sizeOfECDR = await _rdr.ReadUInt64();
_rdr.Position += 4; // Skip version info
if (await _rdr.ReadUInt32() != 0)
{
throw new NotImplementedException("Only single disk archives are supported");
}
if (await _rdr.ReadInt32() != 0)
{
throw new NotImplementedException("Only single disk archives are supported");
}
var recordsOnDisk = await _rdr.ReadUInt64();
var totalRecords = await _rdr.ReadUInt64();
if (recordsOnDisk != totalRecords)
{
throw new NotImplementedException("Only single disk archives are supported");
}
var sizeOfCD = await _rdr.ReadUInt64();
var cdOffset = await _rdr.ReadUInt64();
return (hdrOffset, (uint)totalRecords, (long)cdOffset);
}
public async Task<ExtractedEntry[]> GetFiles()
{
var (sigOffset, totalCentralDirectoryRecords, centralDirectoryOffset) = await ReadZip32EODR(0);
var isZip64 = false;
if (centralDirectoryOffset == uint.MaxValue)
{
isZip64 = true;
(sigOffset, totalCentralDirectoryRecords, centralDirectoryOffset) = await ReadZip64EODR(sigOffset);
}
_rdr.Position = centralDirectoryOffset;
@ -61,17 +131,41 @@ public class ZipReader : IAsyncDisposable
var compressionMethod = await _rdr.ReadInt16();
_rdr.Position += 4;
var crc = await _rdr.ReadUInt32();
var compressedSize = await _rdr.ReadUInt32();
long compressedSize = await _rdr.ReadUInt32();
var uncompressedSize = await _rdr.ReadUInt32();
long uncompressedSize = await _rdr.ReadUInt32();
var fileNameLength = await _rdr.ReadUInt16();
var extraFieldLength = await _rdr.ReadUInt16();
var fileCommentLength = await _rdr.ReadUInt16();
_rdr.Position += 8;
var fileOffset = await _rdr.ReadUInt32();
long fileOffset = await _rdr.ReadUInt32();
var fileName = await _rdr.ReadFixedSizeString(fileNameLength, Encoding.UTF8);
_rdr.Position += extraFieldLength + fileCommentLength;
_rdr.Position += fileCommentLength;
if (compressedSize == uint.MaxValue || uncompressedSize == uint.MaxValue || fileOffset == uint.MaxValue)
{
if (await _rdr.ReadUInt16() != 0x1)
{
throw new Exception("Non Zip64 extra fields not implemented");
}
var size = await _rdr.ReadUInt16();
for (var x = 0; x < size / 8; x++)
{
var value = await _rdr.ReadUInt64();
if (compressedSize == uint.MaxValue)
compressedSize = (long)value;
else if (uncompressedSize == uint.MaxValue)
uncompressedSize = (long) value;
else if (fileOffset == (long) uint.MaxValue)
fileOffset = (long) value;
else
throw new Exception("Bad zip format");
}
}
entries[i] = new ExtractedEntry
{
@ -93,15 +187,14 @@ public class ZipReader : IAsyncDisposable
_stream.Position = entry.FileOffset;
_stream.Position += 6;
var flags = await _rdr.ReadUInt16();
_stream.Position += 18;
bool isZip64 = ((flags & (0x1 << 3)) != 0);
_stream.Position += (isZip64 ? 18 : 18);
var fnLength = await _rdr.ReadUInt16();
var efLength = await _rdr.ReadUInt16();
_stream.Position += fnLength + efLength;
if (flags != 0)
throw new NotImplementedException("Don't know how to handle flags yet");
switch (entry.CompressionMethod)
{
case 0: