From e557e46556cbefb873ffc38fa0df201959034dcc Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 9 Oct 2020 21:02:58 -0600 Subject: [PATCH] Rework file extraction to combine the old and new methods --- Compression.BSA.Test/BSATests.cs | 2 +- Wabbajack.Common/IStreamFactory.cs | 2 +- Wabbajack.Common/Paths/AbsolutePath.cs | 2 +- Wabbajack.Common/Util/TempFile.cs | 5 + Wabbajack.Common/Util/TempFolder.cs | 6 +- Wabbajack.Common/Utils.cs | 1 + Wabbajack.Lib/AInstaller.cs | 35 +- Wabbajack.Test/EndToEndTests.cs | 4 +- .../FileExtractorTests.cs | 14 +- Wabbajack.VirtualFileSystem/Context.cs | 16 +- .../ExtractedFiles/ExtractedMemoryfile.cs | 33 ++ .../ExtractedFiles/ExtractedNativeFile.cs | 26 + .../ExtractedFiles/IExtractedFile.cs | 19 + .../FileExtractor2/FileExtractor.cs | 161 +++++- .../FileExtractor2/GatheringExtractor.cs | 329 ------------- .../SevenZipExtractor/ArchiveFile.cs | 255 ---------- .../SevenZipExtractor/Entry.cs | 89 ---- .../SevenZipExtractor/Formats.cs | 20 - .../IArchiveExtractCallback.cs | 23 - .../SevenZipExtractor/Kernel32Dll.cs | 20 - .../SevenZipExtractor/SafeLibraryHandle.cs | 21 - .../SevenZipExtractor/SevenZipHandle.cs | 68 --- .../SevenZipExtractor/SevenZipInterface.cs | 457 ------------------ Wabbajack.VirtualFileSystem/VirtualFile.cs | 2 +- 24 files changed, 267 insertions(+), 1343 deletions(-) create mode 100644 Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/ExtractedMemoryfile.cs create mode 100644 Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/ExtractedNativeFile.cs create mode 100644 Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/IExtractedFile.cs delete mode 100644 Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/IArchiveExtractCallback.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/Kernel32Dll.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/SafeLibraryHandle.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipHandle.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs diff --git a/Compression.BSA.Test/BSATests.cs b/Compression.BSA.Test/BSATests.cs index 73a16434..cc457666 100644 --- a/Compression.BSA.Test/BSATests.cs +++ b/Compression.BSA.Test/BSATests.cs @@ -83,7 +83,7 @@ namespace Compression.BSA.Test var folder = _bsaFolder.Combine(game.ToString(), modid.ToString()); await folder.DeleteDirectory(); folder.CreateDirectory(); - await FileExtractor2.ExtractAll(filename, folder); + await FileExtractor2.ExtractAll(Queue, filename, folder); foreach (var bsa in folder.EnumerateFiles().Where(f => Consts.SupportedBSAs.Contains(f.Extension))) { diff --git a/Wabbajack.Common/IStreamFactory.cs b/Wabbajack.Common/IStreamFactory.cs index 4fc2d541..7dc4cc4d 100644 --- a/Wabbajack.Common/IStreamFactory.cs +++ b/Wabbajack.Common/IStreamFactory.cs @@ -16,7 +16,7 @@ namespace Wabbajack.Common } public class NativeFileStreamFactory : IStreamFactory { - private AbsolutePath _file; + protected AbsolutePath _file; public NativeFileStreamFactory(AbsolutePath file, IPath path) { diff --git a/Wabbajack.Common/Paths/AbsolutePath.cs b/Wabbajack.Common/Paths/AbsolutePath.cs index 8f56a644..90297883 100644 --- a/Wabbajack.Common/Paths/AbsolutePath.cs +++ b/Wabbajack.Common/Paths/AbsolutePath.cs @@ -171,7 +171,7 @@ namespace Wabbajack.Common /// /// /// Replace the destination file if it exists - public async Task MoveToAsync(AbsolutePath otherPath, bool overwrite = false) + public async ValueTask MoveToAsync(AbsolutePath otherPath, bool overwrite = false) { if (Root != otherPath.Root) { diff --git a/Wabbajack.Common/Util/TempFile.cs b/Wabbajack.Common/Util/TempFile.cs index 92717eb8..5105dcea 100644 --- a/Wabbajack.Common/Util/TempFile.cs +++ b/Wabbajack.Common/Util/TempFile.cs @@ -18,6 +18,11 @@ namespace Wabbajack.Common : this(new FileInfo((string)GetTempFilePath())) { } + + public TempFile(AbsolutePath path, bool deleteAfter = true, bool createFolder = true) + : this(new FileInfo((string)path)) + { + } private static AbsolutePath GetTempFilePath() { diff --git a/Wabbajack.Common/Util/TempFolder.cs b/Wabbajack.Common/Util/TempFolder.cs index 70862216..2b9d8e08 100644 --- a/Wabbajack.Common/Util/TempFolder.cs +++ b/Wabbajack.Common/Util/TempFolder.cs @@ -13,9 +13,11 @@ namespace Wabbajack.Common public bool DeleteAfter = true; private static Task _cleanTask; + public static AbsolutePath BaseFolder => AbsolutePath.EntryPoint.Combine("tmp_files"); + static TempFolder() { - _cleanTask = Task.Run(() => "tmp_files".RelativeTo(AbsolutePath.EntryPoint).DeleteDirectory()); + _cleanTask = Task.Run(() => BaseFolder.DeleteDirectory()); } /// @@ -28,7 +30,7 @@ namespace Wabbajack.Common private TempFolder(bool deleteAfter = true) { - Dir = Path.Combine("tmp_files", Guid.NewGuid().ToString()).RelativeTo(AbsolutePath.EntryPoint); + Dir = BaseFolder.Combine(Guid.NewGuid().ToString()); if (!Dir.Exists) Dir.CreateDirectory(); DeleteAfter = deleteAfter; diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 6e88a707..e1459e84 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -477,6 +477,7 @@ namespace Wabbajack.Common await Task.WhenAll(tasks); } + public static async Task PMap(this IEnumerable coll, WorkQueue queue, Action f) { diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 3654784e..c2ec0b4b 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -3,16 +3,10 @@ using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; -using System.Threading; using System.Threading.Tasks; -using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; using Wabbajack.VirtualFileSystem; -using Wabbajack.VirtualFileSystem.SevenZipExtractor; -using Directory = Alphaleonis.Win32.Filesystem.Directory; -using File = Alphaleonis.Win32.Filesystem.File; -using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib @@ -54,19 +48,9 @@ namespace Wabbajack.Lib public async Task ExtractModlist() { ExtractedModlistFolder = await TempFolder.Create(); - await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(ModListArchive), _ => true, - async (path, sfn) => - { - await using var s = await sfn.GetStream(); - var fp = ExtractedModlistFolder.Dir.Combine(path); - fp.Parent.CreateDirectory(); - await fp.WriteAllAsync(s); - return 0; - }); + await FileExtractor2.ExtractAll(Queue, ModListArchive, ExtractedModlistFolder.Dir); } - - public void Info(string msg) { Utils.Log(msg); @@ -138,16 +122,16 @@ namespace Wabbajack.Lib await VFS.Extract(Queue, grouped.Keys.ToHashSet(), async (vf, sf) => { - await using var s = await sf.GetStream(); foreach (var directive in grouped[vf]) { var file = directive.Directive; - s.Position = 0; switch (file) { case PatchedFromArchive pfa: { + await using var s = await sf.GetStream(); + s.Position = 0; var patchData = await LoadBytesFromPath(pfa.PatchID); var toFile = file.To.RelativeTo(OutputFolder); { @@ -165,7 +149,16 @@ namespace Wabbajack.Lib break; case FromArchive _: - await directive.Directive.To.RelativeTo(OutputFolder).WriteAllAsync(s, false); + if (grouped[vf].Count() == 1) + { + await sf.Move(directive.Directive.To.RelativeTo(OutputFolder)); + } + else + { + await using var s = await sf.GetStream(); + await directive.Directive.To.RelativeTo(OutputFolder).WriteAllAsync(s, false); + } + break; default: throw new Exception($"No handler for {directive}"); @@ -188,7 +181,7 @@ namespace Wabbajack.Lib await file.To.RelativeTo(OutputFolder).Compact(FileCompaction.Algorithm.XPRESS16K); } } - }); + }, tempFolder: OutputFolder); } public async Task DownloadArchives() diff --git a/Wabbajack.Test/EndToEndTests.cs b/Wabbajack.Test/EndToEndTests.cs index 2fa7d9c1..914a8935 100644 --- a/Wabbajack.Test/EndToEndTests.cs +++ b/Wabbajack.Test/EndToEndTests.cs @@ -111,7 +111,7 @@ namespace Wabbajack.Test await src.CopyToAsync(destFile); var modFolder = modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName); - await FileExtractor2.ExtractAll(src, modFolder); + await FileExtractor2.ExtractAll(Queue, src, modFolder); return (destFile, modFolder); } @@ -146,7 +146,7 @@ namespace Wabbajack.Test await src.CopyToAsync(dest); var modFolder = utils.ModsFolder.Combine(modName); - await FileExtractor2.ExtractAll(src, modFolder); + await FileExtractor2.ExtractAll(Queue, src, modFolder); await dest.WithExtension(Consts.MetaFileExtension).WriteAllTextAsync(ini); return (dest, modFolder); diff --git a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs index dab6cfa0..e10230a1 100644 --- a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs @@ -15,6 +15,7 @@ namespace Wabbajack.VirtualFileSystem.Test { private ITestOutputHelper _helper; private IDisposable _unsub; + private WorkQueue _queue; public FileExtractorTests(ITestOutputHelper helper) { @@ -30,6 +31,7 @@ namespace Wabbajack.VirtualFileSystem.Test // ignored } }); + _queue = new WorkQueue(); } public async Task InitializeAsync() @@ -53,7 +55,7 @@ namespace Wabbajack.VirtualFileSystem.Test await ZipUpFolder(temp.Dir, archive.Path, false); - var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path), + var results = await FileExtractor2.GatheringExtract(_queue, new NativeFileStreamFactory(archive.Path), _ => true, async (path, sfn) => { @@ -80,7 +82,7 @@ namespace Wabbajack.VirtualFileSystem.Test await ZipUpFolder(temp.Dir, archive.Path, false); - var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path), + var results = await FileExtractor2.GatheringExtract(_queue, new NativeFileStreamFactory(archive.Path), _ => true, async (path, sfn) => { @@ -131,7 +133,7 @@ namespace Wabbajack.VirtualFileSystem.Test await ZipUpFolder(temp.Dir, archive.Path, false); - var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path), + var results = await FileExtractor2.GatheringExtract(_queue, new NativeFileStreamFactory(archive.Path), _ => true, async (path, sfn) => { @@ -154,10 +156,10 @@ namespace Wabbajack.VirtualFileSystem.Test { var src = await DownloadMod(Game.Oblivion, 18498); - await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(src), + await FileExtractor2.GatheringExtract(_queue, new NativeFileStreamFactory(src), p => p.Extension == OMODExtension, async (path, sfn) => { - await FileExtractor2.GatheringExtract(sfn, _ => true, async (ipath, isfn) => { + await FileExtractor2.GatheringExtract(_queue, sfn, _ => true, async (ipath, isfn) => { // We shouldn't have any .crc files because this file should be recognized as a OMOD and extracted correctly Assert.NotEqual(CRCExtension, ipath.Extension); return 0; @@ -171,7 +173,7 @@ namespace Wabbajack.VirtualFileSystem.Test { var src = await DownloadMod(Game.Fallout4, 29596, 120918); await using var tmpFolder = await TempFolder.Create(); - await FileExtractor2.ExtractAll(src, tmpFolder.Dir); + await FileExtractor2.ExtractAll(_queue, src, tmpFolder.Dir); } diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index dd6015b3..fa3e0cc3 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -9,6 +9,7 @@ using System.Text; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; +using Wabbajack.VirtualFileSystem.ExtractedFiles; using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = System.IO.File; using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; @@ -206,7 +207,7 @@ namespace Wabbajack.VirtualFileSystem /// /// /// - public async Task Extract(WorkQueue queue, HashSet files, Func callback) + public async Task Extract(WorkQueue queue, HashSet files, Func callback, AbsolutePath? tempFolder = null) { var top = new VirtualFile(); var filesByParent = files.SelectMany(f => f.FilesInFullPath) @@ -214,23 +215,28 @@ namespace Wabbajack.VirtualFileSystem .GroupBy(f => f.Parent ?? top) .ToDictionary(f => f.Key); - async Task HandleFile(VirtualFile file, IStreamFactory sfn) + async Task HandleFile(VirtualFile file, IExtractedFile sfn) { + if (filesByParent.ContainsKey(file)) + sfn.CanMove = false; + if (files.Contains(file)) await callback(file, sfn); if (filesByParent.TryGetValue(file, out var children)) { var fileNames = children.ToDictionary(c => c.RelativeName); - await FileExtractor2.GatheringExtract(sfn, + await FileExtractor2.GatheringExtract(queue, sfn, r => fileNames.ContainsKey(r), async (rel, csf) => { await HandleFile(fileNames[rel], csf); return 0; - }); + }, + tempFolder: tempFolder, + onlyFiles: fileNames.Keys.ToHashSet()); } } - await filesByParent[top].PMap(queue, async file => await HandleFile(file, new NativeFileStreamFactory(file.AbsoluteName))); + await filesByParent[top].PMap(queue, async file => await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName))); } #region KnownFiles diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/ExtractedMemoryfile.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/ExtractedMemoryfile.cs new file mode 100644 index 00000000..1d069ca1 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/ExtractedMemoryfile.cs @@ -0,0 +1,33 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem.ExtractedFiles +{ + public class ExtractedMemoryFile : IExtractedFile + { + private IStreamFactory _factory; + + public ExtractedMemoryFile(IStreamFactory factory) + { + _factory = factory; + } + + + public ValueTask GetStream() + { + return _factory.GetStream(); + } + + public DateTime LastModifiedUtc => _factory.LastModifiedUtc; + public IPath Name => _factory.Name; + public async ValueTask Move(AbsolutePath newPath) + { + await using var stream = await _factory.GetStream(); + await newPath.WriteAllAsync(stream); + } + + public bool CanMove { get; set; } = true; + } +} diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/ExtractedNativeFile.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/ExtractedNativeFile.cs new file mode 100644 index 00000000..9e3267d1 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/ExtractedNativeFile.cs @@ -0,0 +1,26 @@ +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem.ExtractedFiles +{ + public class ExtractedNativeFile : NativeFileStreamFactory, IExtractedFile + { + public bool CanMove { get; set; } = true; + + public ExtractedNativeFile(AbsolutePath file, IPath path) : base(file, path) + { + } + + public ExtractedNativeFile(AbsolutePath file) : base(file) + { + } + + public async ValueTask Move(AbsolutePath newPath) + { + if (CanMove) + await _file.MoveToAsync(newPath, overwrite: true); + else + await _file.CopyToAsync(newPath); + } + } +} diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/IExtractedFile.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/IExtractedFile.cs new file mode 100644 index 00000000..1cdc6131 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/ExtractedFiles/IExtractedFile.cs @@ -0,0 +1,19 @@ +using System.IO; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem.ExtractedFiles +{ + public interface IExtractedFile : IStreamFactory + { + /// + /// Possibly destructive move operation. Should greatly optimize file copies when the file + /// exists on the same disk as the newPath. Performs a copy if a move is not possible. + /// + /// destination to move the entry to + /// + public ValueTask Move(AbsolutePath newPath); + + public bool CanMove { get; set; } + } +} diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs index 5ea14e0a..6809867c 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs @@ -1,15 +1,16 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Reactive.Linq; using System.Threading.Tasks; using Compression.BSA; -using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using OMODFramework; -using SharpCompress.Archives.SevenZip; -using SharpCompress.Readers; using Wabbajack.Common; using Wabbajack.Common.FileSignatures; -using Wabbajack.VirtualFileSystem.SevenZipExtractor; +using Wabbajack.Common.StatusFeed; +using Wabbajack.Common.StatusFeed.Errors; +using Wabbajack.VirtualFileSystem.ExtractedFiles; using Utils = Wabbajack.Common.Utils; namespace Wabbajack.VirtualFileSystem @@ -47,9 +48,14 @@ namespace Wabbajack.VirtualFileSystem public static bool FavorPerfOverRAM { get; set; } - public static async Task> GatheringExtract(IStreamFactory sFn, - Predicate shouldExtract, Func> mapfn) + public static async Task> GatheringExtract(WorkQueue queue, IStreamFactory sFn, + Predicate shouldExtract, Func> mapfn, + AbsolutePath? tempFolder = null, + HashSet onlyFiles = null) { + if (tempFolder == null) + tempFolder = TempFolder.BaseFolder; + if (sFn is NativeFileStreamFactory) { Utils.Log($"Extracting {sFn.Name}"); @@ -58,6 +64,8 @@ namespace Wabbajack.VirtualFileSystem var sig = await ArchiveSigs.MatchesAsync(archive); archive.Position = 0; + Dictionary results = new Dictionary(); + switch (sig) { case Definitions.FileType.RAR_OLD: @@ -67,33 +75,42 @@ namespace Wabbajack.VirtualFileSystem { if (sFn.Name.FileName.Extension == OMODExtension) { - return await GatheringExtractWithOMOD(archive, shouldExtract, mapfn); + results = await GatheringExtractWithOMOD(archive, shouldExtract, mapfn); } else { - return await GatheringExtractWith7Zip(sFn, (Definitions.FileType)sig, shouldExtract, - mapfn); + results = await GatheringExtractWith7Zip(queue, sFn, (Definitions.FileType)sig, shouldExtract, + mapfn, tempFolder.Value, onlyFiles); } + + break; } case Definitions.FileType.BSA: case Definitions.FileType.BA2: - return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn); + results = await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn); + break; case Definitions.FileType.TES3: if (sFn.Name.FileName.Extension == BSAExtension) - return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn); + results = await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn); else throw new Exception($"Invalid file format {sFn.Name}"); - - + break; default: throw new Exception($"Invalid file format {sFn.Name}"); } + + if (onlyFiles != null && onlyFiles.Count != results.Count) + { + throw new Exception( + $"Sanity check error extracting {sFn.Name} - {results.Count} results, expected {onlyFiles.Count}"); + } + return results; } - private static async Task> GatheringExtractWithOMOD(Stream archive, Predicate shouldExtract, Func> mapfn) + private static async Task> GatheringExtractWithOMOD(Stream archive, Predicate shouldExtract, Func> mapfn) { var tmpFile = new TempFile(); await tmpFile.Path.WriteAllAsync(archive); @@ -113,7 +130,7 @@ namespace Wabbajack.VirtualFileSystem var path = file.RelativeTo(dest.Dir); if (!shouldExtract(path)) continue; - var result = await mapfn(path, new NativeFileStreamFactory(file, path)); + var result = await mapfn(path, new ExtractedNativeFile(file, path)); results.Add(path, result); } @@ -141,7 +158,7 @@ namespace Wabbajack.VirtualFileSystem } - private static async Task> GatheringExtractWithBSA(IStreamFactory sFn, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) + private static async Task> GatheringExtractWithBSA(IStreamFactory sFn, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) { var archive = await BSADispatch.OpenRead(sFn, sig); var results = new Dictionary(); @@ -150,21 +167,123 @@ namespace Wabbajack.VirtualFileSystem if (!shouldExtract(entry.Path)) continue; - var result = await mapfn(entry.Path, await entry.GetStreamFactory()); + var result = await mapfn(entry.Path, new ExtractedMemoryFile(await entry.GetStreamFactory())); results.Add(entry.Path, result); } return results; } - private static async Task> GatheringExtractWith7Zip(IStreamFactory sf, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) + private static async Task> GatheringExtractWith7Zip(WorkQueue queue, IStreamFactory sf, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn, + AbsolutePath tempPath, HashSet onlyFiles) { - return await new GatheringExtractor(sf, sig, shouldExtract, mapfn).Extract(); + TempFile tmpFile = null; + var dest = tempPath.Combine(Guid.NewGuid().ToString()); + dest.CreateDirectory(); + + TempFile spoolFile = null; + AbsolutePath source; + + try + { + if (sf.Name is AbsolutePath abs) + { + source = abs; + } + else + { + spoolFile = new TempFile(tempPath.Combine(Guid.NewGuid().ToString()) + .WithExtension(source.Extension)); + await using var s = await sf.GetStream(); + await spoolFile.Path.WriteAllAsync(s); + } + + Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}", + $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe")); + + var process = new ProcessHelper {Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),}; + + if (onlyFiles != null) + { + //It's stupid that we have to do this, but 7zip's file pattern matching isn't very fuzzy + IEnumerable AllVariants(string input) + { + yield return $"\"{input}\""; + yield return $"\"\\{input}\""; + } + + tmpFile = new TempFile(); + await tmpFile.Path.WriteAllLinesAsync(onlyFiles.SelectMany(f => AllVariants((string)f)).ToArray()); + process.Arguments = new object[] + { + "x", "-bsp1", "-y", $"-o\"{dest}\"", source, $"@\"{tmpFile.Path}\"", "-mmt=off" + }; + } + else + { + process.Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest}\"", source, "-mmt=off"}; + } + + + var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output) + .ForEachAsync(p => + { + var (_, line) = p; + if (line == null) + return; + + if (line.Length <= 4 || line[3] != '%') return; + + int.TryParse(line.Substring(0, 3), out var percentInt); + Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}", + Percent.FactoryPutInRange(percentInt / 100d)); + }); + + var exitCode = await process.Start(); + + + if (exitCode != 0) + { + Utils.ErrorThrow(new _7zipReturnError(exitCode, source, dest, "")); + } + else + { + Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true); + } + + var results = await dest.EnumerateFiles() + .PMap(queue, async f => + { + var path = f.RelativeTo(dest); + if (!shouldExtract(path)) return ((RelativePath, T))default; + var file = new ExtractedNativeFile(f); + var result = await mapfn(path, file); + await f.DeleteAsync(); + return (path, result); + }); + + return results.Where(d => d.Item1 != default) + .ToDictionary(d => d.Item1, d => d.Item2); + } + finally + { + await dest.DeleteDirectory(); + + if (tmpFile != null) + { + await tmpFile.DisposeAsync(); + } + + if (spoolFile != null) + { + await spoolFile.DisposeAsync(); + } + } } - public static async Task ExtractAll(AbsolutePath src, AbsolutePath dest) + public static async Task ExtractAll(WorkQueue queue, AbsolutePath src, AbsolutePath dest) { - await GatheringExtract(new NativeFileStreamFactory(src), _ => true, async (path, factory) => + await GatheringExtract(queue, new NativeFileStreamFactory(src), _ => true, async (path, factory) => { var abs = path.RelativeTo(dest); abs.Parent.CreateDirectory(); diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs deleted file mode 100644 index 4cfda9eb..00000000 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs +++ /dev/null @@ -1,329 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reactive.Linq; -using System.Runtime.InteropServices; -using System.Threading; -using System.Threading.Tasks; -using Compression.BSA; -using Wabbajack.Common; -using Wabbajack.Common.FileSignatures; -using Wabbajack.Common.StatusFeed.Errors; -using Wabbajack.VirtualFileSystem.SevenZipExtractor; - -namespace Wabbajack.VirtualFileSystem -{ - public class GatheringExtractor : IArchiveExtractCallback - { - private ArchiveFile _archive; - private Predicate _shouldExtract; - private Func> _mapFn; - private Dictionary _results; - private Definitions.FileType _sig; - private Exception _killException; - private uint _itemsCount; - private IStreamFactory _streamFactory; - - public GatheringExtractor(IStreamFactory sF, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) - { - _shouldExtract = shouldExtract; - _mapFn = mapfn; - _results = new Dictionary(); - _streamFactory = sF; - _sig = sig; - } - - public async Task> Extract() - { - var source = new TaskCompletionSource(); - - var th = new Thread(() => - { - try - { - using var stream = _streamFactory.GetStream().Result; - _archive = ArchiveFile.Open(stream, _sig).Result; - ulong checkPos = (ulong)stream.Length; - var oresult = _archive._archive.Open(_archive._archiveStream, ref checkPos, new ArchiveCallback()); - // Can't read this with the COM interface for some reason - if (oresult != 0) - { - var _ = ExtractSlow(source, _streamFactory); - return; - } - _itemsCount = _archive._archive.GetNumberOfItems(); - var result = _archive._archive.Extract(null, 0xFFFFFFFF, 0, this); - _archive.Dispose(); - if (_killException != null) - { - source.SetException(_killException); - } - else - { - source.SetResult(true); - } - } - catch (Exception ex) - { - source.SetException(ex); - } - - }) {Priority = ThreadPriority.BelowNormal, Name = "7Zip Extraction Worker Thread"}; - th.Start(); - - - - await source.Task; - return _results; - } - - private async Task ExtractSlow(TaskCompletionSource tcs, IStreamFactory streamFactory) - { - try - { - TempFile tempFile = null; - AbsolutePath source; - if (streamFactory is NativeFileStreamFactory nsf) - { - source = (AbsolutePath)nsf.Name; - } - else - { - await using var stream = await streamFactory.GetStream(); - tempFile = new TempFile(); - await tempFile.Path.WriteAllAsync(stream); - } - - var dest = await TempFolder.Create(); - Utils.Log( - $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe"); - - var process = new ProcessHelper {Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),}; - - - process.Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, "-mmt=off"}; - - - var _ = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output) - .ForEachAsync(p => - { - var (_, line) = p; - if (line == null) - return; - - if (line.Length <= 4 || line[3] != '%') return; - - int.TryParse(line.Substring(0, 3), out var percentInt); - Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}", - Percent.FactoryPutInRange(percentInt / 100d)); - }); - - var exitCode = await process.Start(); - - - if (exitCode != 0) - { - Utils.ErrorThrow(new _7zipReturnError(exitCode, source, dest.Dir, "")); - } - else - { - Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true); - } - - if (tempFile != null) - { - await tempFile.DisposeAsync(); - } - - foreach (var file in dest.Dir.EnumerateFiles()) - { - var relPath = file.RelativeTo(dest.Dir); - if (!_shouldExtract(relPath)) continue; - - var result = await _mapFn(relPath, new NativeFileStreamFactory(file)); - _results[relPath] = result; - await file.DeleteAsync(); - } - - tcs.SetResult(true); - } - catch (Exception ex) - { - tcs.SetException(ex); - } - } - - public void SetTotal(ulong total) - { - - } - - public void SetCompleted(ref ulong completeValue) - { - - } - - public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode) - { - var entry = _archive.GetEntry(index); - var path = (RelativePath)entry.FileName; - if (entry.IsFolder || !_shouldExtract(path)) - { - outStream = null; - return 0; - } - - Utils.Status($"Extracting {path}", Percent.FactoryPutInRange(_results.Count, _itemsCount)); - // Empty files are never extracted via a write call, so we have to fake that now - if (entry.Size == 0) - { - var result = _mapFn(path, new MemoryStreamFactory(new MemoryStream(), path)).Result; - _results.Add(path, result); - } - outStream = new GatheringExtractorStream(this, entry, path); - return 0; - } - - public void PrepareOperation(AskMode askExtractMode) - { - - } - - public void SetOperationResult(OperationResult resultEOperationResult) - { - - } - - private class GatheringExtractorStream : ISequentialOutStream, IOutStream - { - private GatheringExtractor _extractor; - private ulong _totalSize; - private Stream _tmpStream; - private TempFile _tmpFile; - private bool _diskCached; - private RelativePath _path; - - public GatheringExtractorStream(GatheringExtractor extractor, Entry entry, RelativePath path) - { - _path = path; - _extractor = extractor; - _totalSize = entry.Size; - _diskCached = _totalSize >= int.MaxValue - 1024; - } - - private IPath GetPath() - { - return _path; - } - - public int Write(byte[] data, uint size, IntPtr processedSize) - { - try - { - if (size == _totalSize) - WriteSingleCall(data, size); - else if (_diskCached) - WriteDiskCached(data, size); - else - WriteMemoryCached(data, size); - - if (processedSize != IntPtr.Zero) - { - Marshal.WriteInt32(processedSize, (int)size); - } - - return 0; - } - catch (Exception ex) - { - Utils.Log($"Error during extraction {ex}"); - _extractor.Kill(ex); - - return 1; - } - } - - private void WriteSingleCall(byte[] data, in uint size) - { - var result = _extractor._mapFn(_path, new MemoryBufferFactory(data, (int)size, GetPath())).Result; - AddResult(result); - Cleanup(); - } - - private void Cleanup() - { - _tmpStream?.Dispose(); - _tmpFile?.DisposeAsync().AsTask().Wait(); - } - - private void AddResult(T result) - { - _extractor._results.Add(_path, result); - } - - private void WriteMemoryCached(byte[] data, in uint size) - { - if (_tmpStream == null) - _tmpStream = new MemoryStream(); - _tmpStream.Write(data, 0, (int)size); - - if (_tmpStream.Length != (long)_totalSize) return; - - _tmpStream.Flush(); - _tmpStream.Position = 0; - var result = _extractor._mapFn(_path, new MemoryStreamFactory((MemoryStream)_tmpStream, GetPath())).Result; - AddResult(result); - Cleanup(); - } - - private void WriteDiskCached(byte[] data, in uint size) - { - if (_tmpFile == null) - { - _tmpFile = new TempFile(); - _tmpStream = _tmpFile.Path.Create().Result; - } - - _tmpStream.Write(data, 0, (int)size); - - if (_tmpStream.Length != (long)_totalSize) return; - - _tmpStream.Flush(); - _tmpStream.Close(); - - var result = _extractor._mapFn(_path, new NativeFileStreamFactory(_tmpFile.Path, GetPath())).Result; - AddResult(result); - Cleanup(); - } - - public void Seek(long offset, uint seekOrigin, IntPtr newPosition) - { - - } - - public int SetSize(long newSize) - { - return 0; - } - } - - private void Kill(Exception ex) - { - _killException = ex; - } - - class ArchiveCallback : IArchiveOpenCallback - { - public void SetTotal(IntPtr files, IntPtr bytes) - { - - } - - public void SetCompleted(IntPtr files, IntPtr bytes) - { - - } - } - } -} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs deleted file mode 100644 index 8cdcf977..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs +++ /dev/null @@ -1,255 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using System.Threading.Tasks; -using Wabbajack.Common; -using Wabbajack.Common.FileSignatures; - -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - public class ArchiveFile : IDisposable - { - private SevenZipHandle _sevenZipHandle; - internal IInArchive _archive; - public InStreamWrapper _archiveStream; - private IList _entries; - - private static readonly AbsolutePath LibraryFilePath = @"Extractors\7z.dll".RelativeTo(AbsolutePath.EntryPoint); - private static SignatureChecker _checker = new SignatureChecker(Formats.FileTypeGuidMapping.Keys.ToArray()); - - public static async Task Open(Stream archiveStream, Definitions.FileType format) - { - var self = new ArchiveFile(); - self.InitializeAndValidateLibrary(); - self._archive = self._sevenZipHandle.CreateInArchive(Formats.FileTypeGuidMapping[format]); - - if (!FileExtractor2.FavorPerfOverRAM) - { - self.SetCompressionProperties(new Dictionary() {{"mt", "off"}}); - - } - - - self._archiveStream = new InStreamWrapper(archiveStream); - return self; - } - - /// - /// Sets the compression properties - /// - private void SetCompressionProperties(Dictionary CustomParameters) - { - - - { - ISetProperties setter; - try - { - setter = (ISetProperties)_archive; - } - catch (InvalidCastException) - { - return; - } - - var names = new List(1 + CustomParameters.Count); - var values = new List(1 + CustomParameters.Count); - //var sp = new SecurityPermission(SecurityPermissionFlag.UnmanagedCode); - //sp.Demand(); - - #region Initialize compression properties - - names.Add(Marshal.StringToBSTR("x")); - values.Add(new PropVariant()); - - foreach (var pair in CustomParameters) - { - - names.Add(Marshal.StringToBSTR(pair.Key)); - var pv = new PropVariant(); - - #region List of parameters to cast as integers - - var integerParameters = new HashSet - { - "fb", - "pass", - "o", - "yx", - "a", - "mc", - "lc", - "lp", - "pb", - "cp" - }; - - #endregion - - if (integerParameters.Contains(pair.Key)) - { - pv.VarType = VarEnum.VT_UI4; - pv.UInt32Value = Convert.ToUInt32(pair.Value, CultureInfo.InvariantCulture); - } - else - { - pv.VarType = VarEnum.VT_BSTR; - pv.pointerValue = Marshal.StringToBSTR(pair.Value); - } - - values.Add(pv); - } - - #endregion - - #region Set compression level - - var clpv = values[0]; - clpv.VarType = VarEnum.VT_UI4; - clpv.UInt32Value = 0; - - values[0] = clpv; - - #endregion - - var namesHandle = GCHandle.Alloc(names.ToArray(), GCHandleType.Pinned); - var valuesHandle = GCHandle.Alloc(values.ToArray(), GCHandleType.Pinned); - - try - { - setter?.SetProperties(namesHandle.AddrOfPinnedObject(), valuesHandle.AddrOfPinnedObject(), - names.Count); - } - finally - { - namesHandle.Free(); - valuesHandle.Free(); - } - - } - } - internal Entry GetEntry(uint fileIndex) - { - string fileName = this.GetProperty(fileIndex, ItemPropId.kpidPath); - bool isFolder = this.GetProperty(fileIndex, ItemPropId.kpidIsFolder); - bool isEncrypted = this.GetProperty(fileIndex, ItemPropId.kpidEncrypted); - ulong size = this.GetProperty(fileIndex, ItemPropId.kpidSize); - ulong packedSize = this.GetProperty(fileIndex, ItemPropId.kpidPackedSize); - DateTime creationTime = this.GetPropertySafe(fileIndex, ItemPropId.kpidCreationTime); - DateTime lastWriteTime = this.GetPropertySafe(fileIndex, ItemPropId.kpidLastWriteTime); - DateTime lastAccessTime = this.GetPropertySafe(fileIndex, ItemPropId.kpidLastAccessTime); - uint crc = this.GetPropertySafe(fileIndex, ItemPropId.kpidCRC); - uint attributes = this.GetPropertySafe(fileIndex, ItemPropId.kpidAttributes); - string comment = this.GetPropertySafe(fileIndex, ItemPropId.kpidComment); - string hostOS = this.GetPropertySafe(fileIndex, ItemPropId.kpidHostOS); - string method = this.GetPropertySafe(fileIndex, ItemPropId.kpidMethod); - - bool isSplitBefore = this.GetPropertySafe(fileIndex, ItemPropId.kpidSplitBefore); - bool isSplitAfter = this.GetPropertySafe(fileIndex, ItemPropId.kpidSplitAfter); - - var entry = new Entry(this._archive, fileIndex) - { - FileName = fileName, - IsFolder = isFolder, - IsEncrypted = isEncrypted, - Size = size, - PackedSize = packedSize, - CreationTime = creationTime, - LastWriteTime = lastWriteTime, - LastAccessTime = lastAccessTime, - CRC = crc, - Attributes = attributes, - Comment = comment, - HostOS = hostOS, - Method = method, - IsSplitBefore = isSplitBefore, - IsSplitAfter = isSplitAfter - }; - return entry; - } - - private T GetPropertySafe(uint fileIndex, ItemPropId name) - { - try - { - return this.GetProperty(fileIndex, name); - } - catch (InvalidCastException) - { - return default(T); - } - } - - private T GetProperty(uint fileIndex, ItemPropId name) - { - PropVariant propVariant = new PropVariant(); - this._archive.GetProperty(fileIndex, name, ref propVariant); - object value = propVariant.GetObject(); - - if (propVariant.VarType == VarEnum.VT_EMPTY) - { - propVariant.Clear(); - return default(T); - } - - propVariant.Clear(); - - if (value == null) - { - return default(T); - } - - Type type = typeof(T); - bool isNullable = type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>); - Type underlyingType = isNullable ? Nullable.GetUnderlyingType(type) : type; - - T result = (T)Convert.ChangeType(value.ToString(), underlyingType); - - return result; - } - - private void InitializeAndValidateLibrary() - { - try - { - this._sevenZipHandle = new SevenZipHandle((string)LibraryFilePath); - } - catch (Exception e) - { - throw new Exception("Unable to initialize SevenZipHandle", e); - } - } - - ~ArchiveFile() - { - this.Dispose(false); - } - - protected void Dispose(bool disposing) - { - if (this._archiveStream != null) - { - this._archiveStream.Dispose(); - } - - if (this._archive != null) - { - Marshal.ReleaseComObject(this._archive); - } - - if (this._sevenZipHandle != null) - { - this._sevenZipHandle.Dispose(); - } - } - - public void Dispose() - { - this.Dispose(true); - GC.SuppressFinalize(this); - } - } -} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs deleted file mode 100644 index 15a960c0..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs +++ /dev/null @@ -1,89 +0,0 @@ -using System; -using System.IO; - -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - public class Entry - { - private readonly IInArchive archive; - private readonly uint index; - - internal Entry(IInArchive archive, uint index) - { - this.archive = archive; - this.index = index; - } - - /// - /// Name of the file with its relative path within the archive - /// - public string FileName { get; internal set; } - /// - /// True if entry is a folder, false if it is a file - /// - public bool IsFolder { get; internal set; } - /// - /// Original entry size - /// - public ulong Size { get; internal set; } - /// - /// Entry size in a archived state - /// - public ulong PackedSize { get; internal set; } - - /// - /// Date and time of the file (entry) creation - /// - public DateTime CreationTime { get; internal set; } - - /// - /// Date and time of the last change of the file (entry) - /// - public DateTime LastWriteTime { get; internal set; } - - /// - /// Date and time of the last access of the file (entry) - /// - public DateTime LastAccessTime { get; internal set; } - - /// - /// CRC hash of the entry - /// - public UInt32 CRC { get; internal set; } - - /// - /// Attributes of the entry - /// - public UInt32 Attributes { get; internal set; } - - /// - /// True if entry is encrypted, otherwise false - /// - public bool IsEncrypted { get; internal set; } - - /// - /// Comment of the entry - /// - public string Comment { get; internal set; } - - /// - /// Compression method of the entry - /// - public string Method { get; internal set; } - - /// - /// Host operating system of the entry - /// - public string HostOS { get; internal set; } - - /// - /// True if there are parts of this file in previous split archive parts - /// - public bool IsSplitBefore { get; set; } - - /// - /// True if there are parts of this file in next split archive parts - /// - public bool IsSplitAfter { get; set; } - } -} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs deleted file mode 100644 index bc82e4a8..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.Collections.Generic; -using Wabbajack.Common.FileSignatures; - -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - public class Formats - { - - internal static Dictionary FileTypeGuidMapping = new Dictionary - { - {Definitions.FileType._7Z, new Guid("23170f69-40c1-278a-1000-000110070000")}, - {Definitions.FileType.BZ2, new Guid("23170f69-40c1-278a-1000-000110020000")}, - {Definitions.FileType.RAR_OLD, new Guid("23170f69-40c1-278a-1000-000110030000")}, - {Definitions.FileType.RAR_NEW, new Guid("23170f69-40c1-278a-1000-000110CC0000")}, - {Definitions.FileType.ZIP, new Guid("23170f69-40c1-278a-1000-000110010000")}, - }; - - } -} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/IArchiveExtractCallback.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/IArchiveExtractCallback.cs deleted file mode 100644 index 8f95cc80..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/IArchiveExtractCallback.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Runtime.InteropServices; - -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - [ComImport] - [Guid("23170F69-40C1-278A-0000-000600200000")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - internal interface IArchiveExtractCallback //: IProgress - { - void SetTotal(ulong total); - void SetCompleted([In] ref ulong completeValue); - - [PreserveSig] - int GetStream( - uint index, - [MarshalAs(UnmanagedType.Interface)] out ISequentialOutStream outStream, - AskMode askExtractMode); - // GetStream OUT: S_OK - OK, S_FALSE - skeep this file - - void PrepareOperation(AskMode askExtractMode); - void SetOperationResult(OperationResult resultEOperationResult); - } -} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Kernel32Dll.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Kernel32Dll.cs deleted file mode 100644 index 79d62a90..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Kernel32Dll.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.Runtime.InteropServices; -using System.Security; - -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - internal static class Kernel32Dll - { - [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] - internal static extern SafeLibraryHandle LoadLibrary([MarshalAs(UnmanagedType.LPTStr)] string lpFileName); - - [DllImport("kernel32.dll", CharSet = CharSet.Ansi, SetLastError = true)] - internal static extern IntPtr GetProcAddress(SafeLibraryHandle hModule, [MarshalAs(UnmanagedType.LPStr)] string procName); - - [SuppressUnmanagedCodeSecurity] - [DllImport("kernel32.dll")] - [return: MarshalAs(UnmanagedType.Bool)] - internal static extern bool FreeLibrary(IntPtr hModule); - } -} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SafeLibraryHandle.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SafeLibraryHandle.cs deleted file mode 100644 index ed38dd21..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SafeLibraryHandle.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System; -using System.Runtime.ConstrainedExecution; -using Microsoft.Win32.SafeHandles; - -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - internal sealed class SafeLibraryHandle : SafeHandleZeroOrMinusOneIsInvalid - { - public SafeLibraryHandle() : base(true) - { - } - - /// Release library handle - /// true if the handle was released - [ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success)] - protected override bool ReleaseHandle() - { - return Kernel32Dll.FreeLibrary(this.handle); - } - } -} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipHandle.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipHandle.cs deleted file mode 100644 index f8fb73a7..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipHandle.cs +++ /dev/null @@ -1,68 +0,0 @@ -using System; -using System.ComponentModel; -using System.Runtime.InteropServices; - -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - internal class SevenZipHandle : IDisposable - { - private SafeLibraryHandle sevenZipSafeHandle; - - public SevenZipHandle(string sevenZipLibPath) - { - this.sevenZipSafeHandle = Kernel32Dll.LoadLibrary(sevenZipLibPath); - - if (this.sevenZipSafeHandle.IsInvalid) - { - throw new Win32Exception(); - } - - IntPtr functionPtr = Kernel32Dll.GetProcAddress(this.sevenZipSafeHandle, "GetHandlerProperty"); - - // Not valid dll - if (functionPtr == IntPtr.Zero) - { - this.sevenZipSafeHandle.Close(); - throw new ArgumentException(); - } - } - - ~SevenZipHandle() - { - this.Dispose(false); - } - - protected void Dispose(bool disposing) - { - if ((this.sevenZipSafeHandle != null) && !this.sevenZipSafeHandle.IsClosed) - { - this.sevenZipSafeHandle.Close(); - } - - this.sevenZipSafeHandle = null; - } - - public void Dispose() - { - this.Dispose(true); - GC.SuppressFinalize(this); - } - - public IInArchive CreateInArchive(Guid classId) - { - if (this.sevenZipSafeHandle == null) - { - throw new ObjectDisposedException("SevenZipHandle"); - } - - IntPtr procAddress = Kernel32Dll.GetProcAddress(this.sevenZipSafeHandle, "CreateObject"); - CreateObjectDelegate createObject = (CreateObjectDelegate) Marshal.GetDelegateForFunctionPointer(procAddress, typeof (CreateObjectDelegate)); - - object result; - Guid interfaceId = typeof (IInArchive).GUID; - createObject(ref classId, ref interfaceId, out result); - - return result as IInArchive; - } - } -} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs deleted file mode 100644 index dfd5dcd4..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs +++ /dev/null @@ -1,457 +0,0 @@ -// Version 1.5 - -using System; -using System.Globalization; -using System.IO; -using System.Runtime.InteropServices; -using System.Security.Permissions; -using System.Threading; - -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - [StructLayout(LayoutKind.Sequential)] - internal struct PropArray - { - uint length; - IntPtr pointerValues; - } - - [StructLayout(LayoutKind.Explicit)] - internal struct PropVariant - { - [DllImport("ole32.dll")] - private static extern int PropVariantClear(ref PropVariant pvar); - - [FieldOffset(0)] public ushort vt; - [FieldOffset(8)] public IntPtr pointerValue; - [FieldOffset(8)] public byte byteValue; - [FieldOffset(8)] public long longValue; - [FieldOffset(8)] public UInt32 UInt32Value; - [FieldOffset(8)] public System.Runtime.InteropServices.ComTypes.FILETIME filetime; - [FieldOffset(8)] public PropArray propArray; - - public VarEnum VarType - { - get - { - return (VarEnum) this.vt; - } - set - { - vt = (ushort)value; - } - } - - public void Clear() - { - switch (this.VarType) - { - case VarEnum.VT_EMPTY: - break; - - case VarEnum.VT_NULL: - case VarEnum.VT_I2: - case VarEnum.VT_I4: - case VarEnum.VT_R4: - case VarEnum.VT_R8: - case VarEnum.VT_CY: - case VarEnum.VT_DATE: - case VarEnum.VT_ERROR: - case VarEnum.VT_BOOL: - //case VarEnum.VT_DECIMAL: - case VarEnum.VT_I1: - case VarEnum.VT_UI1: - case VarEnum.VT_UI2: - case VarEnum.VT_UI4: - case VarEnum.VT_I8: - case VarEnum.VT_UI8: - case VarEnum.VT_INT: - case VarEnum.VT_UINT: - case VarEnum.VT_HRESULT: - case VarEnum.VT_FILETIME: - this.vt = 0; - break; - - default: - PropVariantClear(ref this); - break; - } - } - - public object GetObject() - { - switch (this.VarType) - { - case VarEnum.VT_EMPTY: - return null; - - case VarEnum.VT_FILETIME: - return DateTime.FromFileTime(this.longValue); - - default: - GCHandle PropHandle = GCHandle.Alloc(this, GCHandleType.Pinned); - - try - { - return Marshal.GetObjectForNativeVariant(PropHandle.AddrOfPinnedObject()); - } - finally - { - PropHandle.Free(); - } - } - } - } - - [ComImport] - [Guid("23170F69-40C1-278A-0000-000000050000")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - internal interface IProgress - { - void SetTotal(ulong total); - void SetCompleted([In] ref ulong completeValue); - } - - [ComImport] - [Guid("23170F69-40C1-278A-0000-000600100000")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - internal interface IArchiveOpenCallback - { - // ref ulong replaced with IntPtr because handlers ofter pass null value - // read actual value with Marshal.ReadInt64 - void SetTotal( - IntPtr files, // [In] ref ulong files, can use 'ulong* files' but it is unsafe - IntPtr bytes); // [In] ref ulong bytes - - void SetCompleted( - IntPtr files, // [In] ref ulong files - IntPtr bytes); // [In] ref ulong bytes - } - - [ComImport] - [Guid("23170F69-40C1-278A-0000-000500100000")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - internal interface ICryptoGetTextPassword - { - [PreserveSig] - int CryptoGetTextPassword( - [MarshalAs(UnmanagedType.BStr)] out string password); - - //[return : MarshalAs(UnmanagedType.BStr)] - //string CryptoGetTextPassword(); - } - - public enum AskMode : int - { - kExtract = 0, - kTest, - kSkip - } - - public enum OperationResult : int - { - kOK = 0, - kUnSupportedMethod, - kDataError, - kCRCError - } - - [ComImport] - [Guid("23170F69-40C1-278A-0000-000600300000")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - internal interface IArchiveOpenVolumeCallback - { - void GetProperty( - ItemPropId propID, // PROPID - IntPtr value); // PROPVARIANT - - [PreserveSig] - int GetStream( - [MarshalAs(UnmanagedType.LPWStr)] string name, - [MarshalAs(UnmanagedType.Interface)] out IInStream inStream); - } - - [ComImport] - [Guid("23170F69-40C1-278A-0000-000600400000")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - internal interface IInArchiveGetStream - { - [return: MarshalAs(UnmanagedType.Interface)] - ISequentialInStream GetStream(uint index); - } - - [ComImport] - [Guid("23170F69-40C1-278A-0000-000300010000")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - internal interface ISequentialInStream - { - //[PreserveSig] - //int Read( - // [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, - // uint size, - // IntPtr processedSize); // ref uint processedSize - - uint Read( - [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, - uint size); - - /* - Out: if size != 0, return_value = S_OK and (*processedSize == 0), - then there are no more bytes in stream. - if (size > 0) && there are bytes in stream, - this function must read at least 1 byte. - This function is allowed to read less than number of remaining bytes in stream. - You must call Read function in loop, if you need exact amount of data - */ - } - - [ComImport] - [Guid("23170F69-40C1-278A-0000-000300020000")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - public interface ISequentialOutStream - { - [PreserveSig] - int Write( - [In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)]byte[] data, - uint size, - IntPtr processedSize); // ref uint processedSize - /* - if (size > 0) this function must write at least 1 byte. - This function is allowed to write less than "size". - You must call Write function in loop, if you need to write exact amount of data - */ - } - - [ComImport] - [Guid("23170F69-40C1-278A-0000-000300030000")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - internal interface IInStream //: ISequentialInStream - { - //[PreserveSig] - //int Read( - // [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, - // uint size, - // IntPtr processedSize); // ref uint processedSize - - uint Read( - [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, - uint size); - - //[PreserveSig] - void Seek( - long offset, - uint seekOrigin, - IntPtr newPosition); // ref long newPosition - } - - [ComImport] - [Guid("23170F69-40C1-278A-0000-000300040000")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - internal interface IOutStream //: ISequentialOutStream - { - [PreserveSig] - int Write( - [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, - uint size, - IntPtr processedSize); // ref uint processedSize - - //[PreserveSig] - void Seek( - long offset, - uint seekOrigin, - IntPtr newPosition); // ref long newPosition - - [PreserveSig] - int SetSize(long newSize); - } - - internal enum ItemPropId : uint - { - kpidNoProperty = 0, - - kpidHandlerItemIndex = 2, - kpidPath, - kpidName, - kpidExtension, - kpidIsFolder, - kpidSize, - kpidPackedSize, - kpidAttributes, - kpidCreationTime, - kpidLastAccessTime, - kpidLastWriteTime, - kpidSolid, - kpidCommented, - kpidEncrypted, - kpidSplitBefore, - kpidSplitAfter, - kpidDictionarySize, - kpidCRC, - kpidType, - kpidIsAnti, - kpidMethod, - kpidHostOS, - kpidFileSystem, - kpidUser, - kpidGroup, - kpidBlock, - kpidComment, - kpidPosition, - kpidPrefix, - - kpidTotalSize = 0x1100, - kpidFreeSpace, - kpidClusterSize, - kpidVolumeName, - - kpidLocalName = 0x1200, - kpidProvider, - - kpidUserDefined = 0x10000 - } - - /// - /// 7-zip ISetProperties interface for setting various archive properties - /// - [ComImport] - [Guid("23170F69-40C1-278A-0000-000600030000")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - internal interface ISetProperties - { - /// - /// Sets the archive properties - /// - /// The names of the properties - /// The values of the properties - /// The properties count - /// - int SetProperties(IntPtr names, IntPtr values, int numProperties); - } - - - [ComImport] - [Guid("23170F69-40C1-278A-0000-000600600000")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - //[AutomationProxy(true)] - internal interface IInArchive - { - [PreserveSig] - int Open( - IInStream stream, - /*[MarshalAs(UnmanagedType.U8)]*/ [In] ref ulong maxCheckStartPosition, - [MarshalAs(UnmanagedType.Interface)] IArchiveOpenCallback openArchiveCallback); - - void Close(); - //void GetNumberOfItems([In] ref uint numItem); - uint GetNumberOfItems(); - - void GetProperty( - uint index, - ItemPropId propID, // PROPID - ref PropVariant value); // PROPVARIANT - - [PreserveSig] - int Extract( - [MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] uint[] indices, //[In] ref uint indices, - uint numItems, - int testMode, - [MarshalAs(UnmanagedType.Interface)] IArchiveExtractCallback extractCallback); - - // indices must be sorted - // numItems = 0xFFFFFFFF means all files - // testMode != 0 means "test files operation" - - void GetArchiveProperty( - uint propID, // PROPID - ref PropVariant value); // PROPVARIANT - - //void GetNumberOfProperties([In] ref uint numProperties); - uint GetNumberOfProperties(); - - void GetPropertyInfo( - uint index, - [MarshalAs(UnmanagedType.BStr)] out string name, - out ItemPropId propID, // PROPID - out ushort varType); //VARTYPE - - //void GetNumberOfArchiveProperties([In] ref uint numProperties); - uint GetNumberOfArchiveProperties(); - - void GetArchivePropertyInfo( - uint index, - [MarshalAs(UnmanagedType.BStr)] string name, - ref uint propID, // PROPID - ref ushort varType); //VARTYPE - } - - internal enum ArchivePropId : uint - { - kName = 0, - kClassID, - kExtension, - kAddExtension, - kUpdate, - kKeepName, - kStartSignature, - kFinishSignature, - kAssociate - } - - [UnmanagedFunctionPointer(CallingConvention.StdCall)] - internal delegate int CreateObjectDelegate( - [In] ref Guid classID, - [In] ref Guid interfaceID, - //out IntPtr outObject); - [MarshalAs(UnmanagedType.Interface)] out object outObject); - - [UnmanagedFunctionPointer(CallingConvention.StdCall)] - internal delegate int GetHandlerPropertyDelegate( - ArchivePropId propID, - ref PropVariant value); // PROPVARIANT - - [UnmanagedFunctionPointer(CallingConvention.StdCall)] - internal delegate int GetNumberOfFormatsDelegate(out uint numFormats); - - [UnmanagedFunctionPointer(CallingConvention.StdCall)] - internal delegate int GetHandlerProperty2Delegate( - uint formatIndex, - ArchivePropId propID, - ref PropVariant value); // PROPVARIANT - - public class StreamWrapper : IDisposable - { - protected Stream BaseStream; - - protected StreamWrapper(Stream baseStream) - { - this.BaseStream = baseStream; - } - - public void Dispose() - { - this.BaseStream.Close(); - } - - public virtual void Seek(long offset, uint seekOrigin, IntPtr newPosition) - { - long Position = (uint) this.BaseStream.Seek(offset, (SeekOrigin) seekOrigin); - - if (newPosition != IntPtr.Zero) - { - Marshal.WriteInt64(newPosition, Position); - } - } - } - - public class InStreamWrapper : StreamWrapper, ISequentialInStream, IInStream - { - public InStreamWrapper(Stream baseStream) : base(baseStream) - { - } - - public uint Read(byte[] data, uint size) - { - return (uint) this.BaseStream.Read(data, 0, (int) size); - } - } -} diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index ef852831..224be3af 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -218,7 +218,7 @@ namespace Wabbajack.VirtualFileSystem try { - var list = await FileExtractor2.GatheringExtract(extractedFile, + var list = await FileExtractor2.GatheringExtract(context.Queue, extractedFile, _ => true, async (path, sfactory) => await Analyze(context, self, sfactory, path, depth + 1));