Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rework file extraction to combine the old and new methods

Browse Source
This commit is contained in:
Timothy Baldridge 2020-10-09 21:02:58 -06:00
parent e17b577e5a
commit e557e46556
24 changed files with 267 additions and 1343 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Compression.BSA.Test/BSATests.cs
View File

@ -83,7 +83,7 @@ namespace Compression.BSA.Test
var folder = _bsaFolder.Combine(game.ToString(), modid.ToString());
await folder.DeleteDirectory();
folder.CreateDirectory();
await FileExtractor2.ExtractAll(filename, folder);
await FileExtractor2.ExtractAll(Queue, filename, folder);
foreach (var bsa in folder.EnumerateFiles().Where(f => Consts.SupportedBSAs.Contains(f.Extension)))
{

2
Wabbajack.Common/IStreamFactory.cs
View File

@ -16,7 +16,7 @@ namespace Wabbajack.Common
}
public class NativeFileStreamFactory : IStreamFactory
{
private AbsolutePath _file;
protected AbsolutePath _file;
public NativeFileStreamFactory(AbsolutePath file, IPath path)
{

2
Wabbajack.Common/Paths/AbsolutePath.cs
View File

@ -171,7 +171,7 @@ namespace Wabbajack.Common
/// </summary>
/// <param name="otherPath"></param>
/// <param name="overwrite">Replace the destination file if it exists</param>
public async Task MoveToAsync(AbsolutePath otherPath, bool overwrite = false)
public async ValueTask MoveToAsync(AbsolutePath otherPath, bool overwrite = false)
{
if (Root != otherPath.Root)
{

5
Wabbajack.Common/Util/TempFile.cs
View File

@ -18,6 +18,11 @@ namespace Wabbajack.Common
: this(new FileInfo((string)GetTempFilePath()))
{
}
public TempFile(AbsolutePath path, bool deleteAfter = true, bool createFolder = true)
: this(new FileInfo((string)path))
{
}
private static AbsolutePath GetTempFilePath()
{

6
Wabbajack.Common/Util/TempFolder.cs
View File

@ -13,9 +13,11 @@ namespace Wabbajack.Common
public bool DeleteAfter = true;
private static Task _cleanTask;
public static AbsolutePath BaseFolder => AbsolutePath.EntryPoint.Combine("tmp_files");
static TempFolder()
{
_cleanTask = Task.Run(() => "tmp_files".RelativeTo(AbsolutePath.EntryPoint).DeleteDirectory());
_cleanTask = Task.Run(() => BaseFolder.DeleteDirectory());
}
/// <summary>
@ -28,7 +30,7 @@ namespace Wabbajack.Common
private TempFolder(bool deleteAfter = true)
{
Dir = Path.Combine("tmp_files", Guid.NewGuid().ToString()).RelativeTo(AbsolutePath.EntryPoint);
Dir = BaseFolder.Combine(Guid.NewGuid().ToString());
if (!Dir.Exists)
Dir.CreateDirectory();
DeleteAfter = deleteAfter;

1
Wabbajack.Common/Utils.cs
View File

@ -477,6 +477,7 @@ namespace Wabbajack.Common
await Task.WhenAll(tasks);
}
public static async Task PMap<TI>(this IEnumerable<TI> coll, WorkQueue queue, Action<TI> f)
{

35
Wabbajack.Lib/AInstaller.cs
View File

@ -3,16 +3,10 @@ using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
using Wabbajack.VirtualFileSystem.SevenZipExtractor;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Lib
@ -54,19 +48,9 @@ namespace Wabbajack.Lib
public async Task ExtractModlist()
{
ExtractedModlistFolder = await TempFolder.Create();
await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(ModListArchive), _ => true,
async (path, sfn) =>
{
await using var s = await sfn.GetStream();
var fp = ExtractedModlistFolder.Dir.Combine(path);
fp.Parent.CreateDirectory();
await fp.WriteAllAsync(s);
return 0;
});
await FileExtractor2.ExtractAll(Queue, ModListArchive, ExtractedModlistFolder.Dir);
}
public void Info(string msg)
{
Utils.Log(msg);
@ -138,16 +122,16 @@ namespace Wabbajack.Lib
await VFS.Extract(Queue, grouped.Keys.ToHashSet(), async (vf, sf) =>
{
await using var s = await sf.GetStream();
foreach (var directive in grouped[vf])
{
var file = directive.Directive;
s.Position = 0;
switch (file)
{
case PatchedFromArchive pfa:
{
await using var s = await sf.GetStream();
s.Position = 0;
var patchData = await LoadBytesFromPath(pfa.PatchID);
var toFile = file.To.RelativeTo(OutputFolder);
{
@ -165,7 +149,16 @@ namespace Wabbajack.Lib
break;
case FromArchive _:
await directive.Directive.To.RelativeTo(OutputFolder).WriteAllAsync(s, false);
if (grouped[vf].Count() == 1)
{
await sf.Move(directive.Directive.To.RelativeTo(OutputFolder));
}
else
{
await using var s = await sf.GetStream();
await directive.Directive.To.RelativeTo(OutputFolder).WriteAllAsync(s, false);
}
break;
default:
throw new Exception($"No handler for {directive}");
@ -188,7 +181,7 @@ namespace Wabbajack.Lib
await file.To.RelativeTo(OutputFolder).Compact(FileCompaction.Algorithm.XPRESS16K);
}
}
});
}, tempFolder: OutputFolder);
}
public async Task DownloadArchives()

4
Wabbajack.Test/EndToEndTests.cs
View File

@ -111,7 +111,7 @@ namespace Wabbajack.Test
await src.CopyToAsync(destFile);
var modFolder = modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName);
await FileExtractor2.ExtractAll(src, modFolder);
await FileExtractor2.ExtractAll(Queue, src, modFolder);
return (destFile, modFolder);
}
@ -146,7 +146,7 @@ namespace Wabbajack.Test
await src.CopyToAsync(dest);
var modFolder = utils.ModsFolder.Combine(modName);
await FileExtractor2.ExtractAll(src, modFolder);
await FileExtractor2.ExtractAll(Queue, src, modFolder);
await dest.WithExtension(Consts.MetaFileExtension).WriteAllTextAsync(ini);
return (dest, modFolder);

14
Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs
View File

@ -15,6 +15,7 @@ namespace Wabbajack.VirtualFileSystem.Test
{
private ITestOutputHelper _helper;
private IDisposable _unsub;
private WorkQueue _queue;
public FileExtractorTests(ITestOutputHelper helper)
{
@ -30,6 +31,7 @@ namespace Wabbajack.VirtualFileSystem.Test
// ignored
}
});
_queue = new WorkQueue();
}
public async Task InitializeAsync()
@ -53,7 +55,7 @@ namespace Wabbajack.VirtualFileSystem.Test
await ZipUpFolder(temp.Dir, archive.Path, false);
var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path),
var results = await FileExtractor2.GatheringExtract(_queue, new NativeFileStreamFactory(archive.Path),
_ => true,
async (path, sfn) =>
{
@ -80,7 +82,7 @@ namespace Wabbajack.VirtualFileSystem.Test
await ZipUpFolder(temp.Dir, archive.Path, false);
var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path),
var results = await FileExtractor2.GatheringExtract(_queue, new NativeFileStreamFactory(archive.Path),
_ => true,
async (path, sfn) =>
{
@ -131,7 +133,7 @@ namespace Wabbajack.VirtualFileSystem.Test
await ZipUpFolder(temp.Dir, archive.Path, false);
var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path),
var results = await FileExtractor2.GatheringExtract(_queue, new NativeFileStreamFactory(archive.Path),
_ => true,
async (path, sfn) =>
{
@ -154,10 +156,10 @@ namespace Wabbajack.VirtualFileSystem.Test
{
var src = await DownloadMod(Game.Oblivion, 18498);
await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(src),
await FileExtractor2.GatheringExtract(_queue, new NativeFileStreamFactory(src),
p => p.Extension == OMODExtension, async (path, sfn) =>
{
await FileExtractor2.GatheringExtract(sfn, _ => true, async (ipath, isfn) => {
await FileExtractor2.GatheringExtract(_queue, sfn, _ => true, async (ipath, isfn) => {
// We shouldn't have any .crc files because this file should be recognized as a OMOD and extracted correctly
Assert.NotEqual(CRCExtension, ipath.Extension);
return 0;
@ -171,7 +173,7 @@ namespace Wabbajack.VirtualFileSystem.Test
{
var src = await DownloadMod(Game.Fallout4, 29596, 120918);
await using var tmpFolder = await TempFolder.Create();
await FileExtractor2.ExtractAll(src, tmpFolder.Dir);
await FileExtractor2.ExtractAll(_queue, src, tmpFolder.Dir);
}

16
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -9,6 +9,7 @@ using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.VirtualFileSystem.ExtractedFiles;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = System.IO.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
@ -206,7 +207,7 @@ namespace Wabbajack.VirtualFileSystem
/// <param name="files"></param>
/// <param name="callback"></param>
/// <returns></returns>
public async Task Extract(WorkQueue queue, HashSet<VirtualFile> files, Func<VirtualFile, IStreamFactory, ValueTask> callback)
public async Task Extract(WorkQueue queue, HashSet<VirtualFile> files, Func<VirtualFile, IExtractedFile, ValueTask> callback, AbsolutePath? tempFolder = null)
{
var top = new VirtualFile();
var filesByParent = files.SelectMany(f => f.FilesInFullPath)
@ -214,23 +215,28 @@ namespace Wabbajack.VirtualFileSystem
.GroupBy(f => f.Parent ?? top)
.ToDictionary(f => f.Key);
async Task HandleFile(VirtualFile file, IStreamFactory sfn)
async Task HandleFile(VirtualFile file, IExtractedFile sfn)
{
if (filesByParent.ContainsKey(file))
sfn.CanMove = false;
if (files.Contains(file)) await callback(file, sfn);
if (filesByParent.TryGetValue(file, out var children))
{
var fileNames = children.ToDictionary(c => c.RelativeName);
await FileExtractor2.GatheringExtract(sfn,
await FileExtractor2.GatheringExtract(queue, sfn,
r => fileNames.ContainsKey(r),
async (rel, csf) =>
{
await HandleFile(fileNames[rel], csf);
return 0;
});
},
tempFolder: tempFolder,
onlyFiles: fileNames.Keys.ToHashSet());
}
}
await filesByParent[top].PMap(queue, async file => await HandleFile(file, new NativeFileStreamFactory(file.AbsoluteName)));
await filesByParent[top].PMap(queue, async file => await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName)));
}
#region KnownFiles

33
Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/ExtractedMemoryfile.cs Normal file
View File

@ -0,0 +1,33 @@
using System;
using System.IO;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.ExtractedFiles
{
public class ExtractedMemoryFile : IExtractedFile
{
private IStreamFactory _factory;
public ExtractedMemoryFile(IStreamFactory factory)
{
_factory = factory;
}
public ValueTask<Stream> GetStream()
{
return _factory.GetStream();
}
public DateTime LastModifiedUtc => _factory.LastModifiedUtc;
public IPath Name => _factory.Name;
public async ValueTask Move(AbsolutePath newPath)
{
await using var stream = await _factory.GetStream();
await newPath.WriteAllAsync(stream);
}
public bool CanMove { get; set; } = true;
}
}

26
Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/ExtractedNativeFile.cs Normal file
View File

@ -0,0 +1,26 @@
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.ExtractedFiles
{
public class ExtractedNativeFile : NativeFileStreamFactory, IExtractedFile
{
public bool CanMove { get; set; } = true;
public ExtractedNativeFile(AbsolutePath file, IPath path) : base(file, path)
{
}
public ExtractedNativeFile(AbsolutePath file) : base(file)
{
}
public async ValueTask Move(AbsolutePath newPath)
{
if (CanMove)
await _file.MoveToAsync(newPath, overwrite: true);
else
await _file.CopyToAsync(newPath);
}
}
}

19
Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/IExtractedFile.cs Normal file
View File

@ -0,0 +1,19 @@
using System.IO;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.ExtractedFiles
{
public interface IExtractedFile : IStreamFactory
{
/// <summary>
/// Possibly destructive move operation. Should greatly optimize file copies when the file
/// exists on the same disk as the newPath. Performs a copy if a move is not possible.
/// </summary>
/// <param name="newPath">destination to move the entry to</param>
/// <returns></returns>
public ValueTask Move(AbsolutePath newPath);
public bool CanMove { get; set; }
}
}

161
Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs
View File

@ -1,15 +1,16 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reactive.Linq;
using System.Threading.Tasks;
using Compression.BSA;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using OMODFramework;
using SharpCompress.Archives.SevenZip;
using SharpCompress.Readers;
using Wabbajack.Common;
using Wabbajack.Common.FileSignatures;
using Wabbajack.VirtualFileSystem.SevenZipExtractor;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Common.StatusFeed.Errors;
using Wabbajack.VirtualFileSystem.ExtractedFiles;
using Utils = Wabbajack.Common.Utils;
namespace Wabbajack.VirtualFileSystem
@ -47,9 +48,14 @@ namespace Wabbajack.VirtualFileSystem
public static bool FavorPerfOverRAM { get; set; }
public static async Task<Dictionary<RelativePath, T>> GatheringExtract<T>(IStreamFactory sFn,
Predicate<RelativePath> shouldExtract, Func<RelativePath, IStreamFactory, ValueTask<T>> mapfn)
public static async Task<Dictionary<RelativePath, T>> GatheringExtract<T>(WorkQueue queue, IStreamFactory sFn,
Predicate<RelativePath> shouldExtract, Func<RelativePath, IExtractedFile, ValueTask<T>> mapfn,
AbsolutePath? tempFolder = null,
HashSet<RelativePath> onlyFiles = null)
{
if (tempFolder == null)
tempFolder = TempFolder.BaseFolder;
if (sFn is NativeFileStreamFactory)
{
Utils.Log($"Extracting {sFn.Name}");
@ -58,6 +64,8 @@ namespace Wabbajack.VirtualFileSystem
var sig = await ArchiveSigs.MatchesAsync(archive);
archive.Position = 0;
Dictionary<RelativePath, T> results = new Dictionary<RelativePath, T>();
switch (sig)
{
case Definitions.FileType.RAR_OLD:
@ -67,33 +75,42 @@ namespace Wabbajack.VirtualFileSystem
{
if (sFn.Name.FileName.Extension == OMODExtension)
{
return await GatheringExtractWithOMOD(archive, shouldExtract, mapfn);
results = await GatheringExtractWithOMOD(archive, shouldExtract, mapfn);
}
else
{
return await GatheringExtractWith7Zip<T>(sFn, (Definitions.FileType)sig, shouldExtract,
mapfn);
results = await GatheringExtractWith7Zip<T>(queue, sFn, (Definitions.FileType)sig, shouldExtract,
mapfn, tempFolder.Value, onlyFiles);
}
break;
}
case Definitions.FileType.BSA:
case Definitions.FileType.BA2:
return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
results = await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
break;
case Definitions.FileType.TES3:
if (sFn.Name.FileName.Extension == BSAExtension)
return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
results = await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
else
throw new Exception($"Invalid file format {sFn.Name}");
break;
default:
throw new Exception($"Invalid file format {sFn.Name}");
}
if (onlyFiles != null && onlyFiles.Count != results.Count)
{
throw new Exception(
$"Sanity check error extracting {sFn.Name} - {results.Count} results, expected {onlyFiles.Count}");
}
return results;
}
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithOMOD<T>(Stream archive, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithOMOD<T>(Stream archive, Predicate<RelativePath> shouldExtract, Func<RelativePath,IExtractedFile,ValueTask<T>> mapfn)
{
var tmpFile = new TempFile();
await tmpFile.Path.WriteAllAsync(archive);
@ -113,7 +130,7 @@ namespace Wabbajack.VirtualFileSystem
var path = file.RelativeTo(dest.Dir);
if (!shouldExtract(path)) continue;
var result = await mapfn(path, new NativeFileStreamFactory(file, path));
var result = await mapfn(path, new ExtractedNativeFile(file, path));
results.Add(path, result);
}
@ -141,7 +158,7 @@ namespace Wabbajack.VirtualFileSystem
}
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithBSA<T>(IStreamFactory sFn, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithBSA<T>(IStreamFactory sFn, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IExtractedFile,ValueTask<T>> mapfn)
{
var archive = await BSADispatch.OpenRead(sFn, sig);
var results = new Dictionary<RelativePath, T>();
@ -150,21 +167,123 @@ namespace Wabbajack.VirtualFileSystem
if (!shouldExtract(entry.Path))
continue;
var result = await mapfn(entry.Path, await entry.GetStreamFactory());
var result = await mapfn(entry.Path, new ExtractedMemoryFile(await entry.GetStreamFactory()));
results.Add(entry.Path, result);
}
return results;
}
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(IStreamFactory sf, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(WorkQueue queue, IStreamFactory sf, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IExtractedFile,ValueTask<T>> mapfn,
AbsolutePath tempPath, HashSet<RelativePath> onlyFiles)
{
return await new GatheringExtractor<T>(sf, sig, shouldExtract, mapfn).Extract();
TempFile tmpFile = null;
var dest = tempPath.Combine(Guid.NewGuid().ToString());
dest.CreateDirectory();
TempFile spoolFile = null;
AbsolutePath source;
try
{
if (sf.Name is AbsolutePath abs)
{
source = abs;
}
else
{
spoolFile = new TempFile(tempPath.Combine(Guid.NewGuid().ToString())
.WithExtension(source.Extension));
await using var s = await sf.GetStream();
await spoolFile.Path.WriteAllAsync(s);
}
Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}",
$"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe"));
var process = new ProcessHelper {Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),};
if (onlyFiles != null)
{
//It's stupid that we have to do this, but 7zip's file pattern matching isn't very fuzzy
IEnumerable<string> AllVariants(string input)
{
yield return $"\"{input}\"";
yield return $"\"\\{input}\"";
}
tmpFile = new TempFile();
await tmpFile.Path.WriteAllLinesAsync(onlyFiles.SelectMany(f => AllVariants((string)f)).ToArray());
process.Arguments = new object[]
{
"x", "-bsp1", "-y", $"-o\"{dest}\"", source, $"@\"{tmpFile.Path}\"", "-mmt=off"
};
}
else
{
process.Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest}\"", source, "-mmt=off"};
}
var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p =>
{
var (_, line) = p;
if (line == null)
return;
if (line.Length <= 4 || line[3] != '%') return;
int.TryParse(line.Substring(0, 3), out var percentInt);
Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}",
Percent.FactoryPutInRange(percentInt / 100d));
});
var exitCode = await process.Start();
if (exitCode != 0)
{
Utils.ErrorThrow(new _7zipReturnError(exitCode, source, dest, ""));
}
else
{
Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true);
}
var results = await dest.EnumerateFiles()
.PMap(queue, async f =>
{
var path = f.RelativeTo(dest);
if (!shouldExtract(path)) return ((RelativePath, T))default;
var file = new ExtractedNativeFile(f);
var result = await mapfn(path, file);
await f.DeleteAsync();
return (path, result);
});
return results.Where(d => d.Item1 != default)
.ToDictionary(d => d.Item1, d => d.Item2);
}
finally
{
await dest.DeleteDirectory();
if (tmpFile != null)
{
await tmpFile.DisposeAsync();
}
if (spoolFile != null)
{
await spoolFile.DisposeAsync();
}
}
}
public static async Task ExtractAll(AbsolutePath src, AbsolutePath dest)
public static async Task ExtractAll(WorkQueue queue, AbsolutePath src, AbsolutePath dest)
{
await GatheringExtract(new NativeFileStreamFactory(src), _ => true, async (path, factory) =>
await GatheringExtract(queue, new NativeFileStreamFactory(src), _ => true, async (path, factory) =>
{
var abs = path.RelativeTo(dest);
abs.Parent.CreateDirectory();

329
Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs
View File

@ -1,329 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reactive.Linq;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Compression.BSA;
using Wabbajack.Common;
using Wabbajack.Common.FileSignatures;
using Wabbajack.Common.StatusFeed.Errors;
using Wabbajack.VirtualFileSystem.SevenZipExtractor;
namespace Wabbajack.VirtualFileSystem
{
public class GatheringExtractor<T> : IArchiveExtractCallback
{
private ArchiveFile _archive;
private Predicate<RelativePath> _shouldExtract;
private Func<RelativePath, IStreamFactory, ValueTask<T>> _mapFn;
private Dictionary<RelativePath, T> _results;
private Definitions.FileType _sig;
private Exception _killException;
private uint _itemsCount;
private IStreamFactory _streamFactory;
public GatheringExtractor(IStreamFactory sF, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory, ValueTask<T>> mapfn)
{
_shouldExtract = shouldExtract;
_mapFn = mapfn;
_results = new Dictionary<RelativePath, T>();
_streamFactory = sF;
_sig = sig;
}
public async Task<Dictionary<RelativePath, T>> Extract()
{
var source = new TaskCompletionSource<bool>();
var th = new Thread(() =>
{
try
{
using var stream = _streamFactory.GetStream().Result;
_archive = ArchiveFile.Open(stream, _sig).Result;
ulong checkPos = (ulong)stream.Length;
var oresult = _archive._archive.Open(_archive._archiveStream, ref checkPos, new ArchiveCallback());
// Can't read this with the COM interface for some reason
if (oresult != 0)
{
var _ = ExtractSlow(source, _streamFactory);
return;
}
_itemsCount = _archive._archive.GetNumberOfItems();
var result = _archive._archive.Extract(null, 0xFFFFFFFF, 0, this);
_archive.Dispose();
if (_killException != null)
{
source.SetException(_killException);
}
else
{
source.SetResult(true);
}
}
catch (Exception ex)
{
source.SetException(ex);
}
}) {Priority = ThreadPriority.BelowNormal, Name = "7Zip Extraction Worker Thread"};
th.Start();
await source.Task;
return _results;
}
private async Task ExtractSlow(TaskCompletionSource<bool> tcs, IStreamFactory streamFactory)
{
try
{
TempFile tempFile = null;
AbsolutePath source;
if (streamFactory is NativeFileStreamFactory nsf)
{
source = (AbsolutePath)nsf.Name;
}
else
{
await using var stream = await streamFactory.GetStream();
tempFile = new TempFile();
await tempFile.Path.WriteAllAsync(stream);
}
var dest = await TempFolder.Create();
Utils.Log(
$"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe");
var process = new ProcessHelper {Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),};
process.Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, "-mmt=off"};
var _ = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p =>
{
var (_, line) = p;
if (line == null)
return;
if (line.Length <= 4 || line[3] != '%') return;
int.TryParse(line.Substring(0, 3), out var percentInt);
Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}",
Percent.FactoryPutInRange(percentInt / 100d));
});
var exitCode = await process.Start();
if (exitCode != 0)
{
Utils.ErrorThrow(new _7zipReturnError(exitCode, source, dest.Dir, ""));
}
else
{
Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true);
}
if (tempFile != null)
{
await tempFile.DisposeAsync();
}
foreach (var file in dest.Dir.EnumerateFiles())
{
var relPath = file.RelativeTo(dest.Dir);
if (!_shouldExtract(relPath)) continue;
var result = await _mapFn(relPath, new NativeFileStreamFactory(file));
_results[relPath] = result;
await file.DeleteAsync();
}
tcs.SetResult(true);
}
catch (Exception ex)
{
tcs.SetException(ex);
}
}
public void SetTotal(ulong total)
{
}
public void SetCompleted(ref ulong completeValue)
{
}
public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode)
{
var entry = _archive.GetEntry(index);
var path = (RelativePath)entry.FileName;
if (entry.IsFolder || !_shouldExtract(path))
{
outStream = null;
return 0;
}
Utils.Status($"Extracting {path}", Percent.FactoryPutInRange(_results.Count, _itemsCount));
// Empty files are never extracted via a write call, so we have to fake that now
if (entry.Size == 0)
{
var result = _mapFn(path, new MemoryStreamFactory(new MemoryStream(), path)).Result;
_results.Add(path, result);
}
outStream = new GatheringExtractorStream<T>(this, entry, path);
return 0;
}
public void PrepareOperation(AskMode askExtractMode)
{
}
public void SetOperationResult(OperationResult resultEOperationResult)
{
}
private class GatheringExtractorStream<T> : ISequentialOutStream, IOutStream
{
private GatheringExtractor<T> _extractor;
private ulong _totalSize;
private Stream _tmpStream;
private TempFile _tmpFile;
private bool _diskCached;
private RelativePath _path;
public GatheringExtractorStream(GatheringExtractor<T> extractor, Entry entry, RelativePath path)
{
_path = path;
_extractor = extractor;
_totalSize = entry.Size;
_diskCached = _totalSize >= int.MaxValue - 1024;
}
private IPath GetPath()
{
return _path;
}
public int Write(byte[] data, uint size, IntPtr processedSize)
{
try
{
if (size == _totalSize)
WriteSingleCall(data, size);
else if (_diskCached)
WriteDiskCached(data, size);
else
WriteMemoryCached(data, size);
if (processedSize != IntPtr.Zero)
{
Marshal.WriteInt32(processedSize, (int)size);
}
return 0;
}
catch (Exception ex)
{
Utils.Log($"Error during extraction {ex}");
_extractor.Kill(ex);
return 1;
}
}
private void WriteSingleCall(byte[] data, in uint size)
{
var result = _extractor._mapFn(_path, new MemoryBufferFactory(data, (int)size, GetPath())).Result;
AddResult(result);
Cleanup();
}
private void Cleanup()
{
_tmpStream?.Dispose();
_tmpFile?.DisposeAsync().AsTask().Wait();
}
private void AddResult(T result)
{
_extractor._results.Add(_path, result);
}
private void WriteMemoryCached(byte[] data, in uint size)
{
if (_tmpStream == null)
_tmpStream = new MemoryStream();
_tmpStream.Write(data, 0, (int)size);
if (_tmpStream.Length != (long)_totalSize) return;
_tmpStream.Flush();
_tmpStream.Position = 0;
var result = _extractor._mapFn(_path, new MemoryStreamFactory((MemoryStream)_tmpStream, GetPath())).Result;
AddResult(result);
Cleanup();
}
private void WriteDiskCached(byte[] data, in uint size)
{
if (_tmpFile == null)
{
_tmpFile = new TempFile();
_tmpStream = _tmpFile.Path.Create().Result;
}
_tmpStream.Write(data, 0, (int)size);
if (_tmpStream.Length != (long)_totalSize) return;
_tmpStream.Flush();
_tmpStream.Close();
var result = _extractor._mapFn(_path, new NativeFileStreamFactory(_tmpFile.Path, GetPath())).Result;
AddResult(result);
Cleanup();
}
public void Seek(long offset, uint seekOrigin, IntPtr newPosition)
{
}
public int SetSize(long newSize)
{
return 0;
}
}
private void Kill(Exception ex)
{
_killException = ex;
}
class ArchiveCallback : IArchiveOpenCallback
{
public void SetTotal(IntPtr files, IntPtr bytes)
{
}
public void SetCompleted(IntPtr files, IntPtr bytes)
{
}
}
}
}

255
Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs
View File

@ -1,255 +0,0 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.FileSignatures;
namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
{
public class ArchiveFile : IDisposable
{
private SevenZipHandle _sevenZipHandle;
internal IInArchive _archive;
public InStreamWrapper _archiveStream;
private IList<Entry> _entries;
private static readonly AbsolutePath LibraryFilePath = @"Extractors\7z.dll".RelativeTo(AbsolutePath.EntryPoint);
private static SignatureChecker _checker = new SignatureChecker(Formats.FileTypeGuidMapping.Keys.ToArray());
public static async Task<ArchiveFile> Open(Stream archiveStream, Definitions.FileType format)
{
var self = new ArchiveFile();
self.InitializeAndValidateLibrary();
self._archive = self._sevenZipHandle.CreateInArchive(Formats.FileTypeGuidMapping[format]);
if (!FileExtractor2.FavorPerfOverRAM)
{
self.SetCompressionProperties(new Dictionary<string, string>() {{"mt", "off"}});
}
self._archiveStream = new InStreamWrapper(archiveStream);
return self;
}
/// <summary>
/// Sets the compression properties
/// </summary>
private void SetCompressionProperties(Dictionary<string, string> CustomParameters)
{
{
ISetProperties setter;
try
{
setter = (ISetProperties)_archive;
}
catch (InvalidCastException)
{
return;
}
var names = new List<IntPtr>(1 + CustomParameters.Count);
var values = new List<PropVariant>(1 + CustomParameters.Count);
//var sp = new SecurityPermission(SecurityPermissionFlag.UnmanagedCode);
//sp.Demand();
#region Initialize compression properties
names.Add(Marshal.StringToBSTR("x"));
values.Add(new PropVariant());
foreach (var pair in CustomParameters)
{
names.Add(Marshal.StringToBSTR(pair.Key));
var pv = new PropVariant();
#region List of parameters to cast as integers
var integerParameters = new HashSet<string>
{
"fb",
"pass",
"o",
"yx",
"a",
"mc",
"lc",
"lp",
"pb",
"cp"
};
#endregion
if (integerParameters.Contains(pair.Key))
{
pv.VarType = VarEnum.VT_UI4;
pv.UInt32Value = Convert.ToUInt32(pair.Value, CultureInfo.InvariantCulture);
}
else
{
pv.VarType = VarEnum.VT_BSTR;
pv.pointerValue = Marshal.StringToBSTR(pair.Value);
}
values.Add(pv);
}
#endregion
#region Set compression level
var clpv = values[0];
clpv.VarType = VarEnum.VT_UI4;
clpv.UInt32Value = 0;
values[0] = clpv;
#endregion
var namesHandle = GCHandle.Alloc(names.ToArray(), GCHandleType.Pinned);
var valuesHandle = GCHandle.Alloc(values.ToArray(), GCHandleType.Pinned);
try
{
setter?.SetProperties(namesHandle.AddrOfPinnedObject(), valuesHandle.AddrOfPinnedObject(),
names.Count);
}
finally
{
namesHandle.Free();
valuesHandle.Free();
}
}
}
internal Entry GetEntry(uint fileIndex)
{
string fileName = this.GetProperty<string>(fileIndex, ItemPropId.kpidPath);
bool isFolder = this.GetProperty<bool>(fileIndex, ItemPropId.kpidIsFolder);
bool isEncrypted = this.GetProperty<bool>(fileIndex, ItemPropId.kpidEncrypted);
ulong size = this.GetProperty<ulong>(fileIndex, ItemPropId.kpidSize);
ulong packedSize = this.GetProperty<ulong>(fileIndex, ItemPropId.kpidPackedSize);
DateTime creationTime = this.GetPropertySafe<DateTime>(fileIndex, ItemPropId.kpidCreationTime);
DateTime lastWriteTime = this.GetPropertySafe<DateTime>(fileIndex, ItemPropId.kpidLastWriteTime);
DateTime lastAccessTime = this.GetPropertySafe<DateTime>(fileIndex, ItemPropId.kpidLastAccessTime);
uint crc = this.GetPropertySafe<uint>(fileIndex, ItemPropId.kpidCRC);
uint attributes = this.GetPropertySafe<uint>(fileIndex, ItemPropId.kpidAttributes);
string comment = this.GetPropertySafe<string>(fileIndex, ItemPropId.kpidComment);
string hostOS = this.GetPropertySafe<string>(fileIndex, ItemPropId.kpidHostOS);
string method = this.GetPropertySafe<string>(fileIndex, ItemPropId.kpidMethod);
bool isSplitBefore = this.GetPropertySafe<bool>(fileIndex, ItemPropId.kpidSplitBefore);
bool isSplitAfter = this.GetPropertySafe<bool>(fileIndex, ItemPropId.kpidSplitAfter);
var entry = new Entry(this._archive, fileIndex)
{
FileName = fileName,
IsFolder = isFolder,
IsEncrypted = isEncrypted,
Size = size,
PackedSize = packedSize,
CreationTime = creationTime,
LastWriteTime = lastWriteTime,
LastAccessTime = lastAccessTime,
CRC = crc,
Attributes = attributes,
Comment = comment,
HostOS = hostOS,
Method = method,
IsSplitBefore = isSplitBefore,
IsSplitAfter = isSplitAfter
};
return entry;
}
private T GetPropertySafe<T>(uint fileIndex, ItemPropId name)
{
try
{
return this.GetProperty<T>(fileIndex, name);
}
catch (InvalidCastException)
{
return default(T);
}
}
private T GetProperty<T>(uint fileIndex, ItemPropId name)
{
PropVariant propVariant = new PropVariant();
this._archive.GetProperty(fileIndex, name, ref propVariant);
object value = propVariant.GetObject();
if (propVariant.VarType == VarEnum.VT_EMPTY)
{
propVariant.Clear();
return default(T);
}
propVariant.Clear();
if (value == null)
{
return default(T);
}
Type type = typeof(T);
bool isNullable = type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>);
Type underlyingType = isNullable ? Nullable.GetUnderlyingType(type) : type;
T result = (T)Convert.ChangeType(value.ToString(), underlyingType);
return result;
}
private void InitializeAndValidateLibrary()
{
try
{
this._sevenZipHandle = new SevenZipHandle((string)LibraryFilePath);
}
catch (Exception e)
{
throw new Exception("Unable to initialize SevenZipHandle", e);
}
}
~ArchiveFile()
{
this.Dispose(false);
}
protected void Dispose(bool disposing)
{
if (this._archiveStream != null)
{
this._archiveStream.Dispose();
}
if (this._archive != null)
{
Marshal.ReleaseComObject(this._archive);
}
if (this._sevenZipHandle != null)
{
this._sevenZipHandle.Dispose();
}
}
public void Dispose()
{
this.Dispose(true);
GC.SuppressFinalize(this);
}
}
}

89
Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs
View File

@ -1,89 +0,0 @@
using System;
using System.IO;
namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
{
public class Entry
{
private readonly IInArchive archive;
private readonly uint index;
internal Entry(IInArchive archive, uint index)
{
this.archive = archive;
this.index = index;
}
/// <summary>
/// Name of the file with its relative path within the archive
/// </summary>
public string FileName { get; internal set; }
/// <summary>
/// True if entry is a folder, false if it is a file
/// </summary>
public bool IsFolder { get; internal set; }
/// <summary>
/// Original entry size
/// </summary>
public ulong Size { get; internal set; }
/// <summary>
/// Entry size in a archived state
/// </summary>
public ulong PackedSize { get; internal set; }
/// <summary>
/// Date and time of the file (entry) creation
/// </summary>
public DateTime CreationTime { get; internal set; }
/// <summary>
/// Date and time of the last change of the file (entry)
/// </summary>
public DateTime LastWriteTime { get; internal set; }
/// <summary>
/// Date and time of the last access of the file (entry)
/// </summary>
public DateTime LastAccessTime { get; internal set; }
/// <summary>
/// CRC hash of the entry
/// </summary>
public UInt32 CRC { get; internal set; }
/// <summary>
/// Attributes of the entry
/// </summary>
public UInt32 Attributes { get; internal set; }
/// <summary>
/// True if entry is encrypted, otherwise false
/// </summary>
public bool IsEncrypted { get; internal set; }
/// <summary>
/// Comment of the entry
/// </summary>
public string Comment { get; internal set; }
/// <summary>
/// Compression method of the entry
/// </summary>
public string Method { get; internal set; }
/// <summary>
/// Host operating system of the entry
/// </summary>
public string HostOS { get; internal set; }
/// <summary>
/// True if there are parts of this file in previous split archive parts
/// </summary>
public bool IsSplitBefore { get; set; }
/// <summary>
/// True if there are parts of this file in next split archive parts
/// </summary>
public bool IsSplitAfter { get; set; }
}
}

20
Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs
View File

@ -1,20 +0,0 @@
using System;
using System.Collections.Generic;
using Wabbajack.Common.FileSignatures;
namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
{
public class Formats
{
internal static Dictionary<Definitions.FileType, Guid> FileTypeGuidMapping = new Dictionary<Definitions.FileType, Guid>
{
{Definitions.FileType._7Z, new Guid("23170f69-40c1-278a-1000-000110070000")},
{Definitions.FileType.BZ2, new Guid("23170f69-40c1-278a-1000-000110020000")},
{Definitions.FileType.RAR_OLD, new Guid("23170f69-40c1-278a-1000-000110030000")},
{Definitions.FileType.RAR_NEW, new Guid("23170f69-40c1-278a-1000-000110CC0000")},
{Definitions.FileType.ZIP, new Guid("23170f69-40c1-278a-1000-000110010000")},
};
}
}

23
Wabbajack.VirtualFileSystem/SevenZipExtractor/IArchiveExtractCallback.cs
View File

@ -1,23 +0,0 @@
using System.Runtime.InteropServices;
namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
{
[ComImport]
[Guid("23170F69-40C1-278A-0000-000600200000")]
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
internal interface IArchiveExtractCallback //: IProgress
{
void SetTotal(ulong total);
void SetCompleted([In] ref ulong completeValue);
[PreserveSig]
int GetStream(
uint index,
[MarshalAs(UnmanagedType.Interface)] out ISequentialOutStream outStream,
AskMode askExtractMode);
// GetStream OUT: S_OK - OK, S_FALSE - skeep this file
void PrepareOperation(AskMode askExtractMode);
void SetOperationResult(OperationResult resultEOperationResult);
}
}

20
Wabbajack.VirtualFileSystem/SevenZipExtractor/Kernel32Dll.cs
View File

@ -1,20 +0,0 @@
using System;
using System.Runtime.InteropServices;
using System.Security;
namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
{
internal static class Kernel32Dll
{
[DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
internal static extern SafeLibraryHandle LoadLibrary([MarshalAs(UnmanagedType.LPTStr)] string lpFileName);
[DllImport("kernel32.dll", CharSet = CharSet.Ansi, SetLastError = true)]
internal static extern IntPtr GetProcAddress(SafeLibraryHandle hModule, [MarshalAs(UnmanagedType.LPStr)] string procName);
[SuppressUnmanagedCodeSecurity]
[DllImport("kernel32.dll")]
[return: MarshalAs(UnmanagedType.Bool)]
internal static extern bool FreeLibrary(IntPtr hModule);
}
}

21
Wabbajack.VirtualFileSystem/SevenZipExtractor/SafeLibraryHandle.cs
View File

@ -1,21 +0,0 @@
using System;
using System.Runtime.ConstrainedExecution;
using Microsoft.Win32.SafeHandles;
namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
{
internal sealed class SafeLibraryHandle : SafeHandleZeroOrMinusOneIsInvalid
{
public SafeLibraryHandle() : base(true)
{
}
/// <summary>Release library handle</summary>
/// <returns>true if the handle was released</returns>
[ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success)]
protected override bool ReleaseHandle()
{
return Kernel32Dll.FreeLibrary(this.handle);
}
}
}

68
Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipHandle.cs
View File

@ -1,68 +0,0 @@
using System;
using System.ComponentModel;
using System.Runtime.InteropServices;
namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
{
internal class SevenZipHandle : IDisposable
{
private SafeLibraryHandle sevenZipSafeHandle;
public SevenZipHandle(string sevenZipLibPath)
{
this.sevenZipSafeHandle = Kernel32Dll.LoadLibrary(sevenZipLibPath);
if (this.sevenZipSafeHandle.IsInvalid)
{
throw new Win32Exception();
}
IntPtr functionPtr = Kernel32Dll.GetProcAddress(this.sevenZipSafeHandle, "GetHandlerProperty");
// Not valid dll
if (functionPtr == IntPtr.Zero)
{
this.sevenZipSafeHandle.Close();
throw new ArgumentException();
}
}
~SevenZipHandle()
{
this.Dispose(false);
}
protected void Dispose(bool disposing)
{
if ((this.sevenZipSafeHandle != null) && !this.sevenZipSafeHandle.IsClosed)
{
this.sevenZipSafeHandle.Close();
}
this.sevenZipSafeHandle = null;
}
public void Dispose()
{
this.Dispose(true);
GC.SuppressFinalize(this);
}
public IInArchive CreateInArchive(Guid classId)
{
if (this.sevenZipSafeHandle == null)
{
throw new ObjectDisposedException("SevenZipHandle");
}
IntPtr procAddress = Kernel32Dll.GetProcAddress(this.sevenZipSafeHandle, "CreateObject");
CreateObjectDelegate createObject = (CreateObjectDelegate) Marshal.GetDelegateForFunctionPointer(procAddress, typeof (CreateObjectDelegate));
object result;
Guid interfaceId = typeof (IInArchive).GUID;
createObject(ref classId, ref interfaceId, out result);
return result as IInArchive;
}
}
}

457
Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs
View File

@ -1,457 +0,0 @@
// Version 1.5
using System;
using System.Globalization;
using System.IO;
using System.Runtime.InteropServices;
using System.Security.Permissions;
using System.Threading;
namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
{
[StructLayout(LayoutKind.Sequential)]
internal struct PropArray
{
uint length;
IntPtr pointerValues;
}
[StructLayout(LayoutKind.Explicit)]
internal struct PropVariant
{
[DllImport("ole32.dll")]
private static extern int PropVariantClear(ref PropVariant pvar);
[FieldOffset(0)] public ushort vt;
[FieldOffset(8)] public IntPtr pointerValue;
[FieldOffset(8)] public byte byteValue;
[FieldOffset(8)] public long longValue;
[FieldOffset(8)] public UInt32 UInt32Value;
[FieldOffset(8)] public System.Runtime.InteropServices.ComTypes.FILETIME filetime;
[FieldOffset(8)] public PropArray propArray;
public VarEnum VarType
{
get
{
return (VarEnum) this.vt;
}
set
{
vt = (ushort)value;
}
}
public void Clear()
{
switch (this.VarType)
{
case VarEnum.VT_EMPTY:
break;
case VarEnum.VT_NULL:
case VarEnum.VT_I2:
case VarEnum.VT_I4:
case VarEnum.VT_R4:
case VarEnum.VT_R8:
case VarEnum.VT_CY:
case VarEnum.VT_DATE:
case VarEnum.VT_ERROR:
case VarEnum.VT_BOOL:
//case VarEnum.VT_DECIMAL:
case VarEnum.VT_I1:
case VarEnum.VT_UI1:
case VarEnum.VT_UI2:
case VarEnum.VT_UI4:
case VarEnum.VT_I8:
case VarEnum.VT_UI8:
case VarEnum.VT_INT:
case VarEnum.VT_UINT:
case VarEnum.VT_HRESULT:
case VarEnum.VT_FILETIME:
this.vt = 0;
break;
default:
PropVariantClear(ref this);
break;
}
}
public object GetObject()
{
switch (this.VarType)
{
case VarEnum.VT_EMPTY:
return null;
case VarEnum.VT_FILETIME:
return DateTime.FromFileTime(this.longValue);
default:
GCHandle PropHandle = GCHandle.Alloc(this, GCHandleType.Pinned);
try
{
return Marshal.GetObjectForNativeVariant(PropHandle.AddrOfPinnedObject());
}
finally
{
PropHandle.Free();
}
}
}
}
[ComImport]
[Guid("23170F69-40C1-278A-0000-000000050000")]
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
internal interface IProgress
{
void SetTotal(ulong total);
void SetCompleted([In] ref ulong completeValue);
}
[ComImport]
[Guid("23170F69-40C1-278A-0000-000600100000")]
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
internal interface IArchiveOpenCallback
{
// ref ulong replaced with IntPtr because handlers ofter pass null value
// read actual value with Marshal.ReadInt64
void SetTotal(
IntPtr files, // [In] ref ulong files, can use 'ulong* files' but it is unsafe
IntPtr bytes); // [In] ref ulong bytes
void SetCompleted(
IntPtr files, // [In] ref ulong files
IntPtr bytes); // [In] ref ulong bytes
}
[ComImport]
[Guid("23170F69-40C1-278A-0000-000500100000")]
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
internal interface ICryptoGetTextPassword
{
[PreserveSig]
int CryptoGetTextPassword(
[MarshalAs(UnmanagedType.BStr)] out string password);
//[return : MarshalAs(UnmanagedType.BStr)]
//string CryptoGetTextPassword();
}
public enum AskMode : int
{
kExtract = 0,
kTest,
kSkip
}
public enum OperationResult : int
{
kOK = 0,
kUnSupportedMethod,
kDataError,
kCRCError
}
[ComImport]
[Guid("23170F69-40C1-278A-0000-000600300000")]
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
internal interface IArchiveOpenVolumeCallback
{
void GetProperty(
ItemPropId propID, // PROPID
IntPtr value); // PROPVARIANT
[PreserveSig]
int GetStream(
[MarshalAs(UnmanagedType.LPWStr)] string name,
[MarshalAs(UnmanagedType.Interface)] out IInStream inStream);
}
[ComImport]
[Guid("23170F69-40C1-278A-0000-000600400000")]
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
internal interface IInArchiveGetStream
{
[return: MarshalAs(UnmanagedType.Interface)]
ISequentialInStream GetStream(uint index);
}
[ComImport]
[Guid("23170F69-40C1-278A-0000-000300010000")]
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
internal interface ISequentialInStream
{
//[PreserveSig]
//int Read(
// [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
// uint size,
// IntPtr processedSize); // ref uint processedSize
uint Read(
[Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
uint size);
/*
Out: if size != 0, return_value = S_OK and (*processedSize == 0),
then there are no more bytes in stream.
if (size > 0) && there are bytes in stream,
this function must read at least 1 byte.
This function is allowed to read less than number of remaining bytes in stream.
You must call Read function in loop, if you need exact amount of data
*/
}
[ComImport]
[Guid("23170F69-40C1-278A-0000-000300020000")]
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
public interface ISequentialOutStream
{
[PreserveSig]
int Write(
[In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)]byte[] data,
uint size,
IntPtr processedSize); // ref uint processedSize
/*
if (size > 0) this function must write at least 1 byte.
This function is allowed to write less than "size".
You must call Write function in loop, if you need to write exact amount of data
*/
}
[ComImport]
[Guid("23170F69-40C1-278A-0000-000300030000")]
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
internal interface IInStream //: ISequentialInStream
{
//[PreserveSig]
//int Read(
// [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
// uint size,
// IntPtr processedSize); // ref uint processedSize
uint Read(
[Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
uint size);
//[PreserveSig]
void Seek(
long offset,
uint seekOrigin,
IntPtr newPosition); // ref long newPosition
}
[ComImport]
[Guid("23170F69-40C1-278A-0000-000300040000")]
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
internal interface IOutStream //: ISequentialOutStream
{
[PreserveSig]
int Write(
[Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data,
uint size,
IntPtr processedSize); // ref uint processedSize
//[PreserveSig]
void Seek(
long offset,
uint seekOrigin,
IntPtr newPosition); // ref long newPosition
[PreserveSig]
int SetSize(long newSize);
}
internal enum ItemPropId : uint
{
kpidNoProperty = 0,
kpidHandlerItemIndex = 2,
kpidPath,
kpidName,
kpidExtension,
kpidIsFolder,
kpidSize,
kpidPackedSize,
kpidAttributes,
kpidCreationTime,
kpidLastAccessTime,
kpidLastWriteTime,
kpidSolid,
kpidCommented,
kpidEncrypted,
kpidSplitBefore,
kpidSplitAfter,
kpidDictionarySize,
kpidCRC,
kpidType,
kpidIsAnti,
kpidMethod,
kpidHostOS,
kpidFileSystem,
kpidUser,
kpidGroup,
kpidBlock,
kpidComment,
kpidPosition,
kpidPrefix,
kpidTotalSize = 0x1100,
kpidFreeSpace,
kpidClusterSize,
kpidVolumeName,
kpidLocalName = 0x1200,
kpidProvider,
kpidUserDefined = 0x10000
}
/// <summary>
/// 7-zip ISetProperties interface for setting various archive properties
/// </summary>
[ComImport]
[Guid("23170F69-40C1-278A-0000-000600030000")]
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
internal interface ISetProperties
{
/// <summary>
/// Sets the archive properties
/// </summary>
/// <param name="names">The names of the properties</param>
/// <param name="values">The values of the properties</param>
/// <param name="numProperties">The properties count</param>
/// <returns></returns>
int SetProperties(IntPtr names, IntPtr values, int numProperties);
}
[ComImport]
[Guid("23170F69-40C1-278A-0000-000600600000")]
[InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
//[AutomationProxy(true)]
internal interface IInArchive
{
[PreserveSig]
int Open(
IInStream stream,
/*[MarshalAs(UnmanagedType.U8)]*/ [In] ref ulong maxCheckStartPosition,
[MarshalAs(UnmanagedType.Interface)] IArchiveOpenCallback openArchiveCallback);
void Close();
//void GetNumberOfItems([In] ref uint numItem);
uint GetNumberOfItems();
void GetProperty(
uint index,
ItemPropId propID, // PROPID
ref PropVariant value); // PROPVARIANT
[PreserveSig]
int Extract(
[MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] uint[] indices, //[In] ref uint indices,
uint numItems,
int testMode,
[MarshalAs(UnmanagedType.Interface)] IArchiveExtractCallback extractCallback);
// indices must be sorted
// numItems = 0xFFFFFFFF means all files
// testMode != 0 means "test files operation"
void GetArchiveProperty(
uint propID, // PROPID
ref PropVariant value); // PROPVARIANT
//void GetNumberOfProperties([In] ref uint numProperties);
uint GetNumberOfProperties();
void GetPropertyInfo(
uint index,
[MarshalAs(UnmanagedType.BStr)] out string name,
out ItemPropId propID, // PROPID
out ushort varType); //VARTYPE
//void GetNumberOfArchiveProperties([In] ref uint numProperties);
uint GetNumberOfArchiveProperties();
void GetArchivePropertyInfo(
uint index,
[MarshalAs(UnmanagedType.BStr)] string name,
ref uint propID, // PROPID
ref ushort varType); //VARTYPE
}
internal enum ArchivePropId : uint
{
kName = 0,
kClassID,
kExtension,
kAddExtension,
kUpdate,
kKeepName,
kStartSignature,
kFinishSignature,
kAssociate
}
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
internal delegate int CreateObjectDelegate(
[In] ref Guid classID,
[In] ref Guid interfaceID,
//out IntPtr outObject);
[MarshalAs(UnmanagedType.Interface)] out object outObject);
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
internal delegate int GetHandlerPropertyDelegate(
ArchivePropId propID,
ref PropVariant value); // PROPVARIANT
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
internal delegate int GetNumberOfFormatsDelegate(out uint numFormats);
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
internal delegate int GetHandlerProperty2Delegate(
uint formatIndex,
ArchivePropId propID,
ref PropVariant value); // PROPVARIANT
public class StreamWrapper : IDisposable
{
protected Stream BaseStream;
protected StreamWrapper(Stream baseStream)
{
this.BaseStream = baseStream;
}
public void Dispose()
{
this.BaseStream.Close();
}
public virtual void Seek(long offset, uint seekOrigin, IntPtr newPosition)
{
long Position = (uint) this.BaseStream.Seek(offset, (SeekOrigin) seekOrigin);
if (newPosition != IntPtr.Zero)
{
Marshal.WriteInt64(newPosition, Position);
}
}
}
public class InStreamWrapper : StreamWrapper, ISequentialInStream, IInStream
{
public InStreamWrapper(Stream baseStream) : base(baseStream)
{
}
public uint Read(byte[] data, uint size)
{
return (uint) this.BaseStream.Read(data, 0, (int) size);
}
}
}

2
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -218,7 +218,7 @@ namespace Wabbajack.VirtualFileSystem
try
{
var list = await FileExtractor2.GatheringExtract(extractedFile,
var list = await FileExtractor2.GatheringExtract(context.Queue, extractedFile,
_ => true,
async (path, sfactory) => await Analyze(context, self, sfactory, path, depth + 1));