From 27f3571951b5d9b8fafc9450940a790ba9bff8ee Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 2 Sep 2020 16:14:56 -0600 Subject: [PATCH 01/13] 7zip extraction improvements --- .../FileSignatures/Definitions/bsasigs.txt | 4 +- Wabbajack.Common/FileSignatures/Signatures.cs | 8 + Wabbajack.Common/FileSignatures/Signatures.tt | 10 +- Wabbajack.Server/Services/DiscordWebHook.cs | 2 +- Wabbajack.VirtualFileSystem/Context.cs | 2 +- Wabbajack.VirtualFileSystem/FileExtractor.cs | 61 +-- .../SevenZipExtractor/ArchiveFile.cs | 257 ++++++++++ .../SevenZipExtractor/ArchiveFileCallback.cs | 57 +++ .../ArchiveStreamCallback.cs | 45 ++ .../ArchiveStreamsCallback.cs | 58 +++ .../SevenZipExtractor/Entry.cs | 119 +++++ .../SevenZipExtractor/Formats.cs | 122 +++++ .../IArchiveExtractCallback.cs | 23 + .../SevenZipExtractor/Kernel32Dll.cs | 20 + .../SevenZipExtractor/SafeLibraryHandle.cs | 21 + .../SevenZipExtractor/SevenZipException.cs | 24 + .../SevenZipExtractor/SevenZipFormat.cs | 285 +++++++++++ .../SevenZipExtractor/SevenZipHandle.cs | 68 +++ .../SevenZipExtractor/SevenZipInterface.cs | 459 ++++++++++++++++++ 19 files changed, 1588 insertions(+), 57 deletions(-) create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFileCallback.cs create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamCallback.cs create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamsCallback.cs create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/IArchiveExtractCallback.cs create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/Kernel32Dll.cs create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/SafeLibraryHandle.cs create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipException.cs create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipFormat.cs create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipHandle.cs create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs diff --git a/Wabbajack.Common/FileSignatures/Definitions/bsasigs.txt b/Wabbajack.Common/FileSignatures/Definitions/bsasigs.txt index 44262d68..ecf143d8 100644 --- a/Wabbajack.Common/FileSignatures/Definitions/bsasigs.txt +++ b/Wabbajack.Common/FileSignatures/Definitions/bsasigs.txt @@ -1,4 +1,6 @@ Morrowind BSA,00 01 00 00,TES3, null, null, 0, null TES 4-5 and FO 3 BSA,42 53 41 00,BSA FO4 BSA,42 54 44 58,BA2 -Relaxed RAR format,52 61 72 21,RAR \ No newline at end of file +Relaxed RAR format,52 61 72 21,RAR +RAR5 or newer, 52 61 72 21 1A 07 01 00,RAR_NEW +RAR4 or older, 52 61 72 21 1A 07 00,RAR_OLD \ No newline at end of file diff --git a/Wabbajack.Common/FileSignatures/Signatures.cs b/Wabbajack.Common/FileSignatures/Signatures.cs index b21247b9..cd9049b8 100644 --- a/Wabbajack.Common/FileSignatures/Signatures.cs +++ b/Wabbajack.Common/FileSignatures/Signatures.cs @@ -307,6 +307,8 @@ namespace Wabbajack.Common.FileSignatures { RA, RAM, RAR, + RAR_NEW, + RAR_OLD, RBI, RDATA, REG, @@ -443,6 +445,12 @@ namespace Wabbajack.Common.FileSignatures { // Relaxed RAR format (FileType.RAR, new byte[] {0x52, 0x61, 0x72, 0x21}), + // RAR5 or newer + (FileType.RAR_NEW, new byte[] {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00}), + + // RAR4 or older + (FileType.RAR_OLD, new byte[] {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00}), + // JPEG2000 image files (FileType.JP2, new byte[] {0x00, 0x00, 0x00, 0x0C, 0x6A, 0x50, 0x20, 0x20}), diff --git a/Wabbajack.Common/FileSignatures/Signatures.tt b/Wabbajack.Common/FileSignatures/Signatures.tt index e273a80e..57720e82 100644 --- a/Wabbajack.Common/FileSignatures/Signatures.tt +++ b/Wabbajack.Common/FileSignatures/Signatures.tt @@ -7,10 +7,12 @@ <# - byte[] StringToByteArray(string hex) { - return Enumerable.Range(0, hex.Length) - .Where(x => x % 2 == 0) - .Select(x => Convert.ToByte(hex.Substring(x, 2), 16)) + byte[] StringToByteArray(string hex) + { + return Enumerable.Range(0, hex.Length) + .Where(x => x % 2 == 0) + .Select(x => Convert.ToByte(hex.Substring(x, 2), 16)) + .ToArray(); } var files = new string[] {"bsasigs.txt", "file_sigs_RAW.txt"}; diff --git a/Wabbajack.Server/Services/DiscordWebHook.cs b/Wabbajack.Server/Services/DiscordWebHook.cs index 1c7b794b..8571b544 100644 --- a/Wabbajack.Server/Services/DiscordWebHook.cs +++ b/Wabbajack.Server/Services/DiscordWebHook.cs @@ -54,7 +54,7 @@ namespace Wabbajack.Server.Services } catch (Exception ex) { - _logger.LogError(ex, ex.ToJson()); + _logger.LogError(ex, ex.ToString()); } } diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index b3d63d02..eb1a29dc 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -212,7 +212,7 @@ namespace Wabbajack.VirtualFileSystem foreach (var group in grouped) { var only = group.Select(f => f.RelativeName); - var extracted = await group.Key.StagedFile.ExtractAll(Queue, only); + var extracted = await group.Key.StagedFile.ExtractAll(Queue, only, true); paths.Add(extracted); foreach (var file in group) file.StagedFile = extracted[file.RelativeName]; diff --git a/Wabbajack.VirtualFileSystem/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor.cs index da5ab02a..c78d95c4 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor.cs @@ -11,6 +11,7 @@ using Wabbajack.Common.StatusFeed; using Wabbajack.Common.StatusFeed.Errors; using Wabbajack.Common; using Wabbajack.Common.FileSignatures; +using Wabbajack.VirtualFileSystem.SevenZipExtractor; using Utils = Wabbajack.Common.Utils; @@ -29,6 +30,8 @@ namespace Wabbajack.VirtualFileSystem public static async Task ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable OnlyFiles = null, bool throwOnError = true) { + OnlyFiles ??= new RelativePath[0]; + try { var sig = await archiveSigs.MatchesAsync(source); @@ -56,6 +59,7 @@ namespace Wabbajack.VirtualFileSystem if (!throwOnError) return new ExtractedFiles(await TempFolder.Create()); + Utils.Log(ex.ToString()); Utils.ErrorThrow(ex, $"Error while extracting {source}"); throw new Exception(); } @@ -150,66 +154,23 @@ namespace Wabbajack.VirtualFileSystem private static async Task ExtractAllWith7Zip(AbsolutePath source, IEnumerable onlyFiles) { - TempFile tmpFile = null; var dest = await TempFolder.Create(); Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}", $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe")); - var process = new ProcessHelper - { - Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint), - - }; + var files = onlyFiles.ToHashSet(); - if (onlyFiles != null) + using var archive = await ArchiveFile.Open(source); + if (files.Count > 0) { - //It's stupid that we have to do this, but 7zip's file pattern matching isn't very fuzzy - IEnumerable AllVariants(string input) + await archive.Extract(path => { - yield return $"\"{input}\""; - yield return $"\"\\{input}\""; - } - - tmpFile = new TempFile(); - await tmpFile.Path.WriteAllLinesAsync(onlyFiles.SelectMany(f => AllVariants((string)f)).ToArray()); - process.Arguments = new object[] - { - "x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, $"@\"{tmpFile.Path}\"", "-mmt=off" - }; - } - else - { - process.Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, "-mmt=off"}; - } - - - var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output) - .ForEachAsync(p => - { - var (_, line) = p; - if (line == null) - return; - - if (line.Length <= 4 || line[3] != '%') return; - - int.TryParse(line.Substring(0, 3), out var percentInt); - Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d)); + Utils.Log($"Extract file {path} {files.Contains(path)} {dest.Dir.Combine(path)}"); + return files.Contains(path) ? dest.Dir.Combine(path) : default; }); - - var exitCode = await process.Start(); - - - if (exitCode != 0) - { - Utils.ErrorThrow(new _7zipReturnError(exitCode, source, dest.Dir, "")); } else { - Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true); - } - - if (tmpFile != null) - { - await tmpFile.DisposeAsync(); + await archive.Extract(path => dest.Dir.Combine(path)); } return new ExtractedFiles(dest); diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs new file mode 100644 index 00000000..7de4369c --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs @@ -0,0 +1,257 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Common.FileSignatures; + +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + public class ArchiveFile : IDisposable + { + private SevenZipHandle _sevenZipHandle; + private IInArchive _archive; + private InStreamWrapper _archiveStream; + private IList _entries; + + private static readonly AbsolutePath LibraryFilePath = @"Extractors\7z.dll".RelativeTo(AbsolutePath.EntryPoint); + private static SignatureChecker _checker = new SignatureChecker(Formats.FileTypeGuidMapping.Keys.ToArray()); + + public static async Task Open(AbsolutePath archiveFilePath) + { + var self = new ArchiveFile(); + + self.InitializeAndValidateLibrary(); + + if (!archiveFilePath.IsFile) + { + throw new SevenZipException("Archive file not found"); + } + + var format = await _checker.MatchesAsync(archiveFilePath); + + if (format == null) + { + throw new SevenZipException($"Unknown format for {archiveFilePath}"); + } + + self._archive = self._sevenZipHandle.CreateInArchive(Formats.FileTypeGuidMapping[format.Value]); + self._archiveStream = new InStreamWrapper(await archiveFilePath.OpenRead()); + return self; + } + + public static async Task Open(Stream archiveStream, SevenZipFormat? format = null, string libraryFilePath = null) + { + var self = new ArchiveFile(); + self.InitializeAndValidateLibrary(); + self._archive = self._sevenZipHandle.CreateInArchive(Formats.FormatGuidMapping[SevenZipFormat.SevenZip]); + self._archiveStream = new InStreamWrapper(archiveStream); + return self; + } + + public async Task Extract(AbsolutePath outputFolder, bool overwrite = false) + { + await this.Extract(entry => + { + var fileName = outputFolder.Combine(entry.FileName); + + if (!fileName.Exists || overwrite) + { + return fileName; + } + + return default; + }); + } + + public async Task Extract(Func getOutputPath) + { + IList fileStreams = new List(); + + try + { + foreach (Entry entry in Entries) + { + + AbsolutePath outputPath = entry.IsFolder ? default : getOutputPath((RelativePath)entry.FileName); + + if (outputPath == default) // getOutputPath = null means SKIP + { + fileStreams.Add(null); + continue; + } + + if (entry.IsFolder) + { + outputPath.CreateDirectory(); + fileStreams.Add(null); + continue; + } + + var directoryName = outputPath.Parent; + directoryName.CreateDirectory(); + + fileStreams.Add(await outputPath.Create()); + } + + this._archive.Extract(null, 0xFFFFFFFF, 0, new ArchiveStreamsCallback(fileStreams)); + } + finally + { + foreach (Stream stream in fileStreams) + { + if (stream == null) continue; + var tsk = stream?.DisposeAsync(); + await tsk.Value; + } + } + } + + public IList Entries + { + get + { + if (this._entries != null) + { + return this._entries; + } + + ulong checkPos = 32 * 1024; + int open = this._archive.Open(this._archiveStream, ref checkPos, null); + + if (open != 0) + { + throw new SevenZipException("Unable to open archive"); + } + + uint itemsCount = this._archive.GetNumberOfItems(); + + this._entries = new List(); + + for (uint fileIndex = 0; fileIndex < itemsCount; fileIndex++) + { + string fileName = this.GetProperty(fileIndex, ItemPropId.kpidPath); + bool isFolder = this.GetProperty(fileIndex, ItemPropId.kpidIsFolder); + bool isEncrypted = this.GetProperty(fileIndex, ItemPropId.kpidEncrypted); + ulong size = this.GetProperty(fileIndex, ItemPropId.kpidSize); + ulong packedSize = this.GetProperty(fileIndex, ItemPropId.kpidPackedSize); + DateTime creationTime = this.GetPropertySafe(fileIndex, ItemPropId.kpidCreationTime); + DateTime lastWriteTime = this.GetPropertySafe(fileIndex, ItemPropId.kpidLastWriteTime); + DateTime lastAccessTime = this.GetPropertySafe(fileIndex, ItemPropId.kpidLastAccessTime); + uint crc = this.GetPropertySafe(fileIndex, ItemPropId.kpidCRC); + uint attributes = this.GetPropertySafe(fileIndex, ItemPropId.kpidAttributes); + string comment = this.GetPropertySafe(fileIndex, ItemPropId.kpidComment); + string hostOS = this.GetPropertySafe(fileIndex, ItemPropId.kpidHostOS); + string method = this.GetPropertySafe(fileIndex, ItemPropId.kpidMethod); + + bool isSplitBefore = this.GetPropertySafe(fileIndex, ItemPropId.kpidSplitBefore); + bool isSplitAfter = this.GetPropertySafe(fileIndex, ItemPropId.kpidSplitAfter); + + this._entries.Add(new Entry(this._archive, fileIndex) + { + FileName = fileName, + IsFolder = isFolder, + IsEncrypted = isEncrypted, + Size = size, + PackedSize = packedSize, + CreationTime = creationTime, + LastWriteTime = lastWriteTime, + LastAccessTime = lastAccessTime, + CRC = crc, + Attributes = attributes, + Comment = comment, + HostOS = hostOS, + Method = method, + IsSplitBefore = isSplitBefore, + IsSplitAfter = isSplitAfter + }); + } + + return this._entries; + } + } + + private T GetPropertySafe(uint fileIndex, ItemPropId name) + { + try + { + return this.GetProperty(fileIndex, name); + } + catch (InvalidCastException) + { + return default(T); + } + } + + private T GetProperty(uint fileIndex, ItemPropId name) + { + PropVariant propVariant = new PropVariant(); + this._archive.GetProperty(fileIndex, name, ref propVariant); + object value = propVariant.GetObject(); + + if (propVariant.VarType == VarEnum.VT_EMPTY) + { + propVariant.Clear(); + return default(T); + } + + propVariant.Clear(); + + if (value == null) + { + return default(T); + } + + Type type = typeof(T); + bool isNullable = type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>); + Type underlyingType = isNullable ? Nullable.GetUnderlyingType(type) : type; + + T result = (T)Convert.ChangeType(value.ToString(), underlyingType); + + return result; + } + + private void InitializeAndValidateLibrary() + { + try + { + this._sevenZipHandle = new SevenZipHandle((string)LibraryFilePath); + } + catch (Exception e) + { + throw new SevenZipException("Unable to initialize SevenZipHandle", e); + } + } + + ~ArchiveFile() + { + this.Dispose(false); + } + + protected void Dispose(bool disposing) + { + if (this._archiveStream != null) + { + this._archiveStream.Dispose(); + } + + if (this._archive != null) + { + Marshal.ReleaseComObject(this._archive); + } + + if (this._sevenZipHandle != null) + { + this._sevenZipHandle.Dispose(); + } + } + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + } +} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFileCallback.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFileCallback.cs new file mode 100644 index 00000000..18ae341e --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFileCallback.cs @@ -0,0 +1,57 @@ +using System; +using System.IO; + +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + internal class ArchiveFileCallback : IArchiveExtractCallback + { + private readonly string fileName; + private readonly uint fileNumber; + private OutStreamWrapper fileStream; // to be removed + + public ArchiveFileCallback(uint fileNumber, string fileName) + { + this.fileNumber = fileNumber; + this.fileName = fileName; + } + + public void SetTotal(ulong total) + { + } + + public void SetCompleted(ref ulong completeValue) + { + } + + public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode) + { + if ((index != this.fileNumber) || (askExtractMode != AskMode.kExtract)) + { + outStream = null; + return 0; + } + + string fileDir = Path.GetDirectoryName(this.fileName); + + if (!string.IsNullOrEmpty(fileDir)) + { + Directory.CreateDirectory(fileDir); + } + + this.fileStream = new OutStreamWrapper(File.Create(this.fileName)); + + outStream = this.fileStream; + + return 0; + } + + public void PrepareOperation(AskMode askExtractMode) + { + } + + public void SetOperationResult(OperationResult resultEOperationResult) + { + this.fileStream.Dispose(); + } + } +} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamCallback.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamCallback.cs new file mode 100644 index 00000000..ecba71e3 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamCallback.cs @@ -0,0 +1,45 @@ +using System.IO; + +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + internal class ArchiveStreamCallback : IArchiveExtractCallback + { + private readonly uint fileNumber; + private readonly Stream stream; + + public ArchiveStreamCallback(uint fileNumber, Stream stream) + { + this.fileNumber = fileNumber; + this.stream = stream; + } + + public void SetTotal(ulong total) + { + } + + public void SetCompleted(ref ulong completeValue) + { + } + + public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode) + { + if ((index != this.fileNumber) || (askExtractMode != AskMode.kExtract)) + { + outStream = null; + return 0; + } + + outStream = new OutStreamWrapper(this.stream); + + return 0; + } + + public void PrepareOperation(AskMode askExtractMode) + { + } + + public void SetOperationResult(OperationResult resultEOperationResult) + { + } + } +} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamsCallback.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamsCallback.cs new file mode 100644 index 00000000..1cf9feed --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamsCallback.cs @@ -0,0 +1,58 @@ +using System.Collections.Generic; +using System.IO; + +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + internal class ArchiveStreamsCallback : IArchiveExtractCallback + { + private readonly IList streams; + + public ArchiveStreamsCallback(IList streams) + { + this.streams = streams; + } + + public void SetTotal(ulong total) + { + } + + public void SetCompleted(ref ulong completeValue) + { + } + + public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode) + { + if (askExtractMode != AskMode.kExtract) + { + outStream = null; + return 0; + } + + if (this.streams == null) + { + outStream = null; + return 0; + } + + Stream stream = this.streams[(int) index]; + + if (stream == null) + { + outStream = null; + return 0; + } + + outStream = new OutStreamWrapper(stream); + + return 0; + } + + public void PrepareOperation(AskMode askExtractMode) + { + } + + public void SetOperationResult(OperationResult resultEOperationResult) + { + } + } +} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs new file mode 100644 index 00000000..ff6b8e20 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs @@ -0,0 +1,119 @@ +using System; +using System.IO; + +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + public class Entry + { + private readonly IInArchive archive; + private readonly uint index; + + internal Entry(IInArchive archive, uint index) + { + this.archive = archive; + this.index = index; + } + + /// + /// Name of the file with its relative path within the archive + /// + public string FileName { get; internal set; } + /// + /// True if entry is a folder, false if it is a file + /// + public bool IsFolder { get; internal set; } + /// + /// Original entry size + /// + public ulong Size { get; internal set; } + /// + /// Entry size in a archived state + /// + public ulong PackedSize { get; internal set; } + + /// + /// Date and time of the file (entry) creation + /// + public DateTime CreationTime { get; internal set; } + + /// + /// Date and time of the last change of the file (entry) + /// + public DateTime LastWriteTime { get; internal set; } + + /// + /// Date and time of the last access of the file (entry) + /// + public DateTime LastAccessTime { get; internal set; } + + /// + /// CRC hash of the entry + /// + public UInt32 CRC { get; internal set; } + + /// + /// Attributes of the entry + /// + public UInt32 Attributes { get; internal set; } + + /// + /// True if entry is encrypted, otherwise false + /// + public bool IsEncrypted { get; internal set; } + + /// + /// Comment of the entry + /// + public string Comment { get; internal set; } + + /// + /// Compression method of the entry + /// + public string Method { get; internal set; } + + /// + /// Host operating system of the entry + /// + public string HostOS { get; internal set; } + + /// + /// True if there are parts of this file in previous split archive parts + /// + public bool IsSplitBefore { get; set; } + + /// + /// True if there are parts of this file in next split archive parts + /// + public bool IsSplitAfter { get; set; } + + public void Extract(string fileName, bool preserveTimestamp = true) + { + if (this.IsFolder) + { + Directory.CreateDirectory(fileName); + return; + } + + string directoryName = Path.GetDirectoryName(fileName); + + if (!string.IsNullOrWhiteSpace(directoryName)) + { + Directory.CreateDirectory(directoryName); + } + + using (FileStream fileStream = File.Create(fileName)) + { + this.Extract(fileStream); + } + + if (preserveTimestamp) + { + File.SetLastWriteTime(fileName, this.LastWriteTime); + } + } + public void Extract(Stream stream) + { + this.archive.Extract(new[] { this.index }, 1, 0, new ArchiveStreamCallback(this.index, stream)); + } + } +} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs new file mode 100644 index 00000000..a44d180a --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using Wabbajack.Common.FileSignatures; + +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + public class Formats + { + internal static readonly Dictionary ExtensionFormatMapping = new Dictionary + { + {"7z", SevenZipFormat.SevenZip}, + {"gz", SevenZipFormat.GZip}, + {"tar", SevenZipFormat.Tar}, + {"rar", SevenZipFormat.Rar}, + {"zip", SevenZipFormat.Zip}, + {"lzma", SevenZipFormat.Lzma}, + {"lzh", SevenZipFormat.Lzh}, + {"arj", SevenZipFormat.Arj}, + {"bz2", SevenZipFormat.BZip2}, + {"cab", SevenZipFormat.Cab}, + {"chm", SevenZipFormat.Chm}, + {"deb", SevenZipFormat.Deb}, + {"iso", SevenZipFormat.Iso}, + {"rpm", SevenZipFormat.Rpm}, + {"wim", SevenZipFormat.Wim}, + {"udf", SevenZipFormat.Udf}, + {"mub", SevenZipFormat.Mub}, + {"xar", SevenZipFormat.Xar}, + {"hfs", SevenZipFormat.Hfs}, + {"dmg", SevenZipFormat.Dmg}, + {"z", SevenZipFormat.Lzw}, + {"xz", SevenZipFormat.XZ}, + {"flv", SevenZipFormat.Flv}, + {"swf", SevenZipFormat.Swf}, + {"exe", SevenZipFormat.PE}, + {"dll", SevenZipFormat.PE}, + {"vhd", SevenZipFormat.Vhd} + }; + + internal static Dictionary FormatGuidMapping = new Dictionary + { + {SevenZipFormat.SevenZip, new Guid("23170f69-40c1-278a-1000-000110070000")}, + {SevenZipFormat.Arj, new Guid("23170f69-40c1-278a-1000-000110040000")}, + {SevenZipFormat.BZip2, new Guid("23170f69-40c1-278a-1000-000110020000")}, + {SevenZipFormat.Cab, new Guid("23170f69-40c1-278a-1000-000110080000")}, + {SevenZipFormat.Chm, new Guid("23170f69-40c1-278a-1000-000110e90000")}, + {SevenZipFormat.Compound, new Guid("23170f69-40c1-278a-1000-000110e50000")}, + {SevenZipFormat.Cpio, new Guid("23170f69-40c1-278a-1000-000110ed0000")}, + {SevenZipFormat.Deb, new Guid("23170f69-40c1-278a-1000-000110ec0000")}, + {SevenZipFormat.GZip, new Guid("23170f69-40c1-278a-1000-000110ef0000")}, + {SevenZipFormat.Iso, new Guid("23170f69-40c1-278a-1000-000110e70000")}, + {SevenZipFormat.Lzh, new Guid("23170f69-40c1-278a-1000-000110060000")}, + {SevenZipFormat.Lzma, new Guid("23170f69-40c1-278a-1000-0001100a0000")}, + {SevenZipFormat.Nsis, new Guid("23170f69-40c1-278a-1000-000110090000")}, + {SevenZipFormat.Rar, new Guid("23170f69-40c1-278a-1000-000110030000")}, + {SevenZipFormat.Rar5, new Guid("23170f69-40c1-278a-1000-000110CC0000")}, + {SevenZipFormat.Rpm, new Guid("23170f69-40c1-278a-1000-000110eb0000")}, + {SevenZipFormat.Split, new Guid("23170f69-40c1-278a-1000-000110ea0000")}, + {SevenZipFormat.Tar, new Guid("23170f69-40c1-278a-1000-000110ee0000")}, + {SevenZipFormat.Wim, new Guid("23170f69-40c1-278a-1000-000110e60000")}, + {SevenZipFormat.Lzw, new Guid("23170f69-40c1-278a-1000-000110050000")}, + {SevenZipFormat.Zip, new Guid("23170f69-40c1-278a-1000-000110010000")}, + {SevenZipFormat.Udf, new Guid("23170f69-40c1-278a-1000-000110E00000")}, + {SevenZipFormat.Xar, new Guid("23170f69-40c1-278a-1000-000110E10000")}, + {SevenZipFormat.Mub, new Guid("23170f69-40c1-278a-1000-000110E20000")}, + {SevenZipFormat.Hfs, new Guid("23170f69-40c1-278a-1000-000110E30000")}, + {SevenZipFormat.Dmg, new Guid("23170f69-40c1-278a-1000-000110E40000")}, + {SevenZipFormat.XZ, new Guid("23170f69-40c1-278a-1000-0001100C0000")}, + {SevenZipFormat.Mslz, new Guid("23170f69-40c1-278a-1000-000110D50000")}, + {SevenZipFormat.PE, new Guid("23170f69-40c1-278a-1000-000110DD0000")}, + {SevenZipFormat.Elf, new Guid("23170f69-40c1-278a-1000-000110DE0000")}, + {SevenZipFormat.Swf, new Guid("23170f69-40c1-278a-1000-000110D70000")}, + {SevenZipFormat.Vhd, new Guid("23170f69-40c1-278a-1000-000110DC0000")}, + {SevenZipFormat.Flv, new Guid("23170f69-40c1-278a-1000-000110D60000")}, + {SevenZipFormat.SquashFS, new Guid("23170f69-40c1-278a-1000-000110D20000")}, + {SevenZipFormat.Lzma86, new Guid("23170f69-40c1-278a-1000-0001100B0000")}, + {SevenZipFormat.Ppmd, new Guid("23170f69-40c1-278a-1000-0001100D0000")}, + {SevenZipFormat.TE, new Guid("23170f69-40c1-278a-1000-000110CF0000")}, + {SevenZipFormat.UEFIc, new Guid("23170f69-40c1-278a-1000-000110D00000")}, + {SevenZipFormat.UEFIs, new Guid("23170f69-40c1-278a-1000-000110D10000")}, + {SevenZipFormat.CramFS, new Guid("23170f69-40c1-278a-1000-000110D30000")}, + {SevenZipFormat.APM, new Guid("23170f69-40c1-278a-1000-000110D40000")}, + {SevenZipFormat.Swfc, new Guid("23170f69-40c1-278a-1000-000110D80000")}, + {SevenZipFormat.Ntfs, new Guid("23170f69-40c1-278a-1000-000110D90000")}, + {SevenZipFormat.Fat, new Guid("23170f69-40c1-278a-1000-000110DA0000")}, + {SevenZipFormat.Mbr, new Guid("23170f69-40c1-278a-1000-000110DB0000")}, + {SevenZipFormat.MachO, new Guid("23170f69-40c1-278a-1000-000110DF0000")} + }; + + internal static Dictionary FileTypeGuidMapping = new Dictionary + { + {Definitions.FileType._7Z, new Guid("23170f69-40c1-278a-1000-000110070000")}, + {Definitions.FileType.BZ2, new Guid("23170f69-40c1-278a-1000-000110020000")}, + {Definitions.FileType.RAR_NEW, new Guid("23170f69-40c1-278a-1000-000110030000")}, + {Definitions.FileType.RAR_OLD, new Guid("23170f69-40c1-278a-1000-000110CC0000")}, + {Definitions.FileType.ZIP, new Guid("23170f69-40c1-278a-1000-000110010000")}, + }; + + internal static Dictionary FileSignatures = new Dictionary + { + {SevenZipFormat.Rar5, new byte[] {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00}}, + {SevenZipFormat.Rar, new byte[] { 0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00 }}, + {SevenZipFormat.Vhd, new byte[] { 0x63, 0x6F, 0x6E, 0x65, 0x63, 0x74, 0x69, 0x78 }}, + {SevenZipFormat.Deb, new byte[] { 0x21, 0x3C, 0x61, 0x72, 0x63, 0x68, 0x3E }}, + {SevenZipFormat.Dmg, new byte[] { 0x78, 0x01, 0x73, 0x0D, 0x62, 0x62, 0x60 }}, + {SevenZipFormat.SevenZip, new byte[] { 0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C }}, + {SevenZipFormat.Tar, new byte[] { 0x75, 0x73, 0x74, 0x61, 0x72 }}, + {SevenZipFormat.Iso, new byte[] { 0x43, 0x44, 0x30, 0x30, 0x31 }}, + {SevenZipFormat.Cab, new byte[] { 0x4D, 0x53, 0x43, 0x46 }}, + {SevenZipFormat.Rpm, new byte[] { 0xed, 0xab, 0xee, 0xdb }}, + {SevenZipFormat.Xar, new byte[] { 0x78, 0x61, 0x72, 0x21 }}, + {SevenZipFormat.Chm, new byte[] { 0x49, 0x54, 0x53, 0x46 }}, + {SevenZipFormat.BZip2, new byte[] { 0x42, 0x5A, 0x68 }}, + {SevenZipFormat.Flv, new byte[] { 0x46, 0x4C, 0x56 }}, + {SevenZipFormat.Swf, new byte[] { 0x46, 0x57, 0x53 }}, + {SevenZipFormat.GZip, new byte[] { 0x1f, 0x0b }}, + {SevenZipFormat.Zip, new byte[] { 0x50, 0x4b }}, + {SevenZipFormat.Arj, new byte[] { 0x60, 0xEA }}, + {SevenZipFormat.Lzh, new byte[] { 0x2D, 0x6C, 0x68 }} + }; + } +} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/IArchiveExtractCallback.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/IArchiveExtractCallback.cs new file mode 100644 index 00000000..8f95cc80 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/IArchiveExtractCallback.cs @@ -0,0 +1,23 @@ +using System.Runtime.InteropServices; + +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + [ComImport] + [Guid("23170F69-40C1-278A-0000-000600200000")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IArchiveExtractCallback //: IProgress + { + void SetTotal(ulong total); + void SetCompleted([In] ref ulong completeValue); + + [PreserveSig] + int GetStream( + uint index, + [MarshalAs(UnmanagedType.Interface)] out ISequentialOutStream outStream, + AskMode askExtractMode); + // GetStream OUT: S_OK - OK, S_FALSE - skeep this file + + void PrepareOperation(AskMode askExtractMode); + void SetOperationResult(OperationResult resultEOperationResult); + } +} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Kernel32Dll.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Kernel32Dll.cs new file mode 100644 index 00000000..79d62a90 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Kernel32Dll.cs @@ -0,0 +1,20 @@ +using System; +using System.Runtime.InteropServices; +using System.Security; + +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + internal static class Kernel32Dll + { + [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] + internal static extern SafeLibraryHandle LoadLibrary([MarshalAs(UnmanagedType.LPTStr)] string lpFileName); + + [DllImport("kernel32.dll", CharSet = CharSet.Ansi, SetLastError = true)] + internal static extern IntPtr GetProcAddress(SafeLibraryHandle hModule, [MarshalAs(UnmanagedType.LPStr)] string procName); + + [SuppressUnmanagedCodeSecurity] + [DllImport("kernel32.dll")] + [return: MarshalAs(UnmanagedType.Bool)] + internal static extern bool FreeLibrary(IntPtr hModule); + } +} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SafeLibraryHandle.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SafeLibraryHandle.cs new file mode 100644 index 00000000..ed38dd21 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SafeLibraryHandle.cs @@ -0,0 +1,21 @@ +using System; +using System.Runtime.ConstrainedExecution; +using Microsoft.Win32.SafeHandles; + +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + internal sealed class SafeLibraryHandle : SafeHandleZeroOrMinusOneIsInvalid + { + public SafeLibraryHandle() : base(true) + { + } + + /// Release library handle + /// true if the handle was released + [ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success)] + protected override bool ReleaseHandle() + { + return Kernel32Dll.FreeLibrary(this.handle); + } + } +} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipException.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipException.cs new file mode 100644 index 00000000..9fd9665c --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipException.cs @@ -0,0 +1,24 @@ +using System; +using System.Runtime.Serialization; + +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + public class SevenZipException : Exception + { + public SevenZipException() + { + } + + public SevenZipException(string message) : base(message) + { + } + + public SevenZipException(string message, Exception innerException) : base(message, innerException) + { + } + + protected SevenZipException(SerializationInfo info, StreamingContext context) : base(info, context) + { + } + } +} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipFormat.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipFormat.cs new file mode 100644 index 00000000..6ca1ed4b --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipFormat.cs @@ -0,0 +1,285 @@ +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + /// + /// + /// + public enum SevenZipFormat + { + // Default invalid format value + Undefined = 0, + + /// + /// Open 7-zip archive format. + /// + /// Wikipedia information + SevenZip, + + /// + /// Proprietary Arj archive format. + /// + /// Wikipedia information + Arj, + + /// + /// Open Bzip2 archive format. + /// + /// Wikipedia information + BZip2, + + /// + /// Microsoft cabinet archive format. + /// + /// Wikipedia information + Cab, + + /// + /// Microsoft Compiled HTML Help file format. + /// + /// Wikipedia information + Chm, + + /// + /// Microsoft Compound file format. + /// + /// Wikipedia information + Compound, + + /// + /// Open Cpio archive format. + /// + /// Wikipedia information + Cpio, + + /// + /// Open Debian software package format. + /// + /// Wikipedia information + Deb, + + /// + /// Open Gzip archive format. + /// + /// Wikipedia information + GZip, + + /// + /// Open ISO disk image format. + /// + /// Wikipedia information + Iso, + + /// + /// Open Lzh archive format. + /// + /// Wikipedia information + Lzh, + + /// + /// Open core 7-zip Lzma raw archive format. + /// + /// Wikipedia information + Lzma, + + /// + /// Nullsoft installation package format. + /// + /// Wikipedia information + Nsis, + + /// + /// RarLab Rar archive format. + /// + /// Wikipedia information + Rar, + + /// + /// RarLab Rar archive format, version 5. + /// + /// Wikipedia information + Rar5, + + /// + /// Open Rpm software package format. + /// + /// Wikipedia information + Rpm, + + /// + /// Open split file format. + /// + /// Wikipedia information + Split, + + /// + /// Open Tar archive format. + /// + /// Wikipedia information + Tar, + + /// + /// Microsoft Windows Imaging disk image format. + /// + /// Wikipedia information + Wim, + + /// + /// Open LZW archive format; implemented in "compress" program; also known as "Z" archive format. + /// + /// Wikipedia information + Lzw, + + /// + /// Open Zip archive format. + /// + /// Wikipedia information + Zip, + + /// + /// Open Udf disk image format. + /// + Udf, + + /// + /// Xar open source archive format. + /// + /// Wikipedia information + Xar, + + /// + /// Mub + /// + Mub, + + /// + /// Macintosh Disk Image on CD. + /// + /// Wikipedia information + Hfs, + + /// + /// Apple Mac OS X Disk Copy Disk Image format. + /// + Dmg, + + /// + /// Open Xz archive format. + /// + /// Wikipedia information + XZ, + + /// + /// MSLZ archive format. + /// + Mslz, + + /// + /// Flash video format. + /// + /// Wikipedia information + Flv, + + /// + /// Shockwave Flash format. + /// + /// Wikipedia information + Swf, + + /// + /// Windows PE executable format. + /// + /// Wikipedia information + PE, + + /// + /// Linux executable Elf format. + /// + /// Wikipedia information + Elf, + + /// + /// Windows Installer Database. + /// + /// Wikipedia information + Msi, + + /// + /// Microsoft virtual hard disk file format. + /// + /// Wikipedia information + Vhd, + + /// + /// SquashFS file system format. + /// + /// Wikipedia information + SquashFS, + + /// + /// Lzma86 file format. + /// + Lzma86, + + /// + /// Prediction by Partial Matching by Dmitry algorithm. + /// + /// Wikipedia information + Ppmd, + + /// + /// TE format. + /// + TE, + + /// + /// UEFIc format. + /// + /// Wikipedia information + UEFIc, + + /// + /// UEFIs format. + /// + /// Wikipedia information + UEFIs, + + /// + /// Compressed ROM file system format. + /// + /// Wikipedia information + CramFS, + + /// + /// APM format. + /// + APM, + + /// + /// Swfc format. + /// + Swfc, + + /// + /// NTFS file system format. + /// + /// Wikipedia information + Ntfs, + + /// + /// FAT file system format. + /// + /// Wikipedia information + Fat, + + /// + /// MBR format. + /// + /// Wikipedia information + Mbr, + + /// + /// Mach-O file format. + /// + /// Wikipedia information + MachO + } +} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipHandle.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipHandle.cs new file mode 100644 index 00000000..f8fb73a7 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipHandle.cs @@ -0,0 +1,68 @@ +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; + +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + internal class SevenZipHandle : IDisposable + { + private SafeLibraryHandle sevenZipSafeHandle; + + public SevenZipHandle(string sevenZipLibPath) + { + this.sevenZipSafeHandle = Kernel32Dll.LoadLibrary(sevenZipLibPath); + + if (this.sevenZipSafeHandle.IsInvalid) + { + throw new Win32Exception(); + } + + IntPtr functionPtr = Kernel32Dll.GetProcAddress(this.sevenZipSafeHandle, "GetHandlerProperty"); + + // Not valid dll + if (functionPtr == IntPtr.Zero) + { + this.sevenZipSafeHandle.Close(); + throw new ArgumentException(); + } + } + + ~SevenZipHandle() + { + this.Dispose(false); + } + + protected void Dispose(bool disposing) + { + if ((this.sevenZipSafeHandle != null) && !this.sevenZipSafeHandle.IsClosed) + { + this.sevenZipSafeHandle.Close(); + } + + this.sevenZipSafeHandle = null; + } + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + public IInArchive CreateInArchive(Guid classId) + { + if (this.sevenZipSafeHandle == null) + { + throw new ObjectDisposedException("SevenZipHandle"); + } + + IntPtr procAddress = Kernel32Dll.GetProcAddress(this.sevenZipSafeHandle, "CreateObject"); + CreateObjectDelegate createObject = (CreateObjectDelegate) Marshal.GetDelegateForFunctionPointer(procAddress, typeof (CreateObjectDelegate)); + + object result; + Guid interfaceId = typeof (IInArchive).GUID; + createObject(ref classId, ref interfaceId, out result); + + return result as IInArchive; + } + } +} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs new file mode 100644 index 00000000..91666547 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs @@ -0,0 +1,459 @@ +// Version 1.5 + +using System; +using System.Globalization; +using System.IO; +using System.Runtime.InteropServices; +using System.Security.Permissions; +using System.Threading; + +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + [StructLayout(LayoutKind.Sequential)] + internal struct PropArray + { + uint length; + IntPtr pointerValues; + } + + [StructLayout(LayoutKind.Explicit)] + internal struct PropVariant + { + [DllImport("ole32.dll")] + private static extern int PropVariantClear(ref PropVariant pvar); + + [FieldOffset(0)] public ushort vt; + [FieldOffset(8)] public IntPtr pointerValue; + [FieldOffset(8)] public byte byteValue; + [FieldOffset(8)] public long longValue; + [FieldOffset(8)] public System.Runtime.InteropServices.ComTypes.FILETIME filetime; + [FieldOffset(8)] public PropArray propArray; + + public VarEnum VarType + { + get + { + return (VarEnum) this.vt; + } + } + + public void Clear() + { + switch (this.VarType) + { + case VarEnum.VT_EMPTY: + break; + + case VarEnum.VT_NULL: + case VarEnum.VT_I2: + case VarEnum.VT_I4: + case VarEnum.VT_R4: + case VarEnum.VT_R8: + case VarEnum.VT_CY: + case VarEnum.VT_DATE: + case VarEnum.VT_ERROR: + case VarEnum.VT_BOOL: + //case VarEnum.VT_DECIMAL: + case VarEnum.VT_I1: + case VarEnum.VT_UI1: + case VarEnum.VT_UI2: + case VarEnum.VT_UI4: + case VarEnum.VT_I8: + case VarEnum.VT_UI8: + case VarEnum.VT_INT: + case VarEnum.VT_UINT: + case VarEnum.VT_HRESULT: + case VarEnum.VT_FILETIME: + this.vt = 0; + break; + + default: + PropVariantClear(ref this); + break; + } + } + + public object GetObject() + { + switch (this.VarType) + { + case VarEnum.VT_EMPTY: + return null; + + case VarEnum.VT_FILETIME: + return DateTime.FromFileTime(this.longValue); + + default: + GCHandle PropHandle = GCHandle.Alloc(this, GCHandleType.Pinned); + + try + { + return Marshal.GetObjectForNativeVariant(PropHandle.AddrOfPinnedObject()); + } + finally + { + PropHandle.Free(); + } + } + } + } + + [ComImport] + [Guid("23170F69-40C1-278A-0000-000000050000")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IProgress + { + void SetTotal(ulong total); + void SetCompleted([In] ref ulong completeValue); + } + + [ComImport] + [Guid("23170F69-40C1-278A-0000-000600100000")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IArchiveOpenCallback + { + // ref ulong replaced with IntPtr because handlers ofter pass null value + // read actual value with Marshal.ReadInt64 + void SetTotal( + IntPtr files, // [In] ref ulong files, can use 'ulong* files' but it is unsafe + IntPtr bytes); // [In] ref ulong bytes + + void SetCompleted( + IntPtr files, // [In] ref ulong files + IntPtr bytes); // [In] ref ulong bytes + } + + [ComImport] + [Guid("23170F69-40C1-278A-0000-000500100000")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface ICryptoGetTextPassword + { + [PreserveSig] + int CryptoGetTextPassword( + [MarshalAs(UnmanagedType.BStr)] out string password); + + //[return : MarshalAs(UnmanagedType.BStr)] + //string CryptoGetTextPassword(); + } + + internal enum AskMode : int + { + kExtract = 0, + kTest, + kSkip + } + + internal enum OperationResult : int + { + kOK = 0, + kUnSupportedMethod, + kDataError, + kCRCError + } + + [ComImport] + [Guid("23170F69-40C1-278A-0000-000600300000")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IArchiveOpenVolumeCallback + { + void GetProperty( + ItemPropId propID, // PROPID + IntPtr value); // PROPVARIANT + + [PreserveSig] + int GetStream( + [MarshalAs(UnmanagedType.LPWStr)] string name, + [MarshalAs(UnmanagedType.Interface)] out IInStream inStream); + } + + [ComImport] + [Guid("23170F69-40C1-278A-0000-000600400000")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IInArchiveGetStream + { + [return: MarshalAs(UnmanagedType.Interface)] + ISequentialInStream GetStream(uint index); + } + + [ComImport] + [Guid("23170F69-40C1-278A-0000-000300010000")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface ISequentialInStream + { + //[PreserveSig] + //int Read( + // [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, + // uint size, + // IntPtr processedSize); // ref uint processedSize + + uint Read( + [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, + uint size); + + /* + Out: if size != 0, return_value = S_OK and (*processedSize == 0), + then there are no more bytes in stream. + if (size > 0) && there are bytes in stream, + this function must read at least 1 byte. + This function is allowed to read less than number of remaining bytes in stream. + You must call Read function in loop, if you need exact amount of data + */ + } + + [ComImport] + [Guid("23170F69-40C1-278A-0000-000300020000")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface ISequentialOutStream + { + [PreserveSig] + int Write( + [In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, + uint size, + IntPtr processedSize); // ref uint processedSize + /* + if (size > 0) this function must write at least 1 byte. + This function is allowed to write less than "size". + You must call Write function in loop, if you need to write exact amount of data + */ + } + + [ComImport] + [Guid("23170F69-40C1-278A-0000-000300030000")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IInStream //: ISequentialInStream + { + //[PreserveSig] + //int Read( + // [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, + // uint size, + // IntPtr processedSize); // ref uint processedSize + + uint Read( + [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, + uint size); + + //[PreserveSig] + void Seek( + long offset, + uint seekOrigin, + IntPtr newPosition); // ref long newPosition + } + + [ComImport] + [Guid("23170F69-40C1-278A-0000-000300040000")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface IOutStream //: ISequentialOutStream + { + [PreserveSig] + int Write( + [In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, + uint size, + IntPtr processedSize); // ref uint processedSize + + //[PreserveSig] + void Seek( + long offset, + uint seekOrigin, + IntPtr newPosition); // ref long newPosition + + [PreserveSig] + int SetSize(long newSize); + } + + internal enum ItemPropId : uint + { + kpidNoProperty = 0, + + kpidHandlerItemIndex = 2, + kpidPath, + kpidName, + kpidExtension, + kpidIsFolder, + kpidSize, + kpidPackedSize, + kpidAttributes, + kpidCreationTime, + kpidLastAccessTime, + kpidLastWriteTime, + kpidSolid, + kpidCommented, + kpidEncrypted, + kpidSplitBefore, + kpidSplitAfter, + kpidDictionarySize, + kpidCRC, + kpidType, + kpidIsAnti, + kpidMethod, + kpidHostOS, + kpidFileSystem, + kpidUser, + kpidGroup, + kpidBlock, + kpidComment, + kpidPosition, + kpidPrefix, + + kpidTotalSize = 0x1100, + kpidFreeSpace, + kpidClusterSize, + kpidVolumeName, + + kpidLocalName = 0x1200, + kpidProvider, + + kpidUserDefined = 0x10000 + } + + + [ComImport] + [Guid("23170F69-40C1-278A-0000-000600600000")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + //[AutomationProxy(true)] + internal interface IInArchive + { + [PreserveSig] + int Open( + IInStream stream, + /*[MarshalAs(UnmanagedType.U8)]*/ [In] ref ulong maxCheckStartPosition, + [MarshalAs(UnmanagedType.Interface)] IArchiveOpenCallback openArchiveCallback); + + void Close(); + //void GetNumberOfItems([In] ref uint numItem); + uint GetNumberOfItems(); + + void GetProperty( + uint index, + ItemPropId propID, // PROPID + ref PropVariant value); // PROPVARIANT + + [PreserveSig] + int Extract( + [MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] uint[] indices, //[In] ref uint indices, + uint numItems, + int testMode, + [MarshalAs(UnmanagedType.Interface)] IArchiveExtractCallback extractCallback); + + // indices must be sorted + // numItems = 0xFFFFFFFF means all files + // testMode != 0 means "test files operation" + + void GetArchiveProperty( + uint propID, // PROPID + ref PropVariant value); // PROPVARIANT + + //void GetNumberOfProperties([In] ref uint numProperties); + uint GetNumberOfProperties(); + + void GetPropertyInfo( + uint index, + [MarshalAs(UnmanagedType.BStr)] out string name, + out ItemPropId propID, // PROPID + out ushort varType); //VARTYPE + + //void GetNumberOfArchiveProperties([In] ref uint numProperties); + uint GetNumberOfArchiveProperties(); + + void GetArchivePropertyInfo( + uint index, + [MarshalAs(UnmanagedType.BStr)] string name, + ref uint propID, // PROPID + ref ushort varType); //VARTYPE + } + + internal enum ArchivePropId : uint + { + kName = 0, + kClassID, + kExtension, + kAddExtension, + kUpdate, + kKeepName, + kStartSignature, + kFinishSignature, + kAssociate + } + + [UnmanagedFunctionPointer(CallingConvention.StdCall)] + internal delegate int CreateObjectDelegate( + [In] ref Guid classID, + [In] ref Guid interfaceID, + //out IntPtr outObject); + [MarshalAs(UnmanagedType.Interface)] out object outObject); + + [UnmanagedFunctionPointer(CallingConvention.StdCall)] + internal delegate int GetHandlerPropertyDelegate( + ArchivePropId propID, + ref PropVariant value); // PROPVARIANT + + [UnmanagedFunctionPointer(CallingConvention.StdCall)] + internal delegate int GetNumberOfFormatsDelegate(out uint numFormats); + + [UnmanagedFunctionPointer(CallingConvention.StdCall)] + internal delegate int GetHandlerProperty2Delegate( + uint formatIndex, + ArchivePropId propID, + ref PropVariant value); // PROPVARIANT + + internal class StreamWrapper : IDisposable + { + protected Stream BaseStream; + + protected StreamWrapper(Stream baseStream) + { + this.BaseStream = baseStream; + } + + public void Dispose() + { + this.BaseStream.Close(); + } + + public virtual void Seek(long offset, uint seekOrigin, IntPtr newPosition) + { + long Position = (uint) this.BaseStream.Seek(offset, (SeekOrigin) seekOrigin); + + if (newPosition != IntPtr.Zero) + { + Marshal.WriteInt64(newPosition, Position); + } + } + } + + internal class InStreamWrapper : StreamWrapper, ISequentialInStream, IInStream + { + public InStreamWrapper(Stream baseStream) : base(baseStream) + { + } + + public uint Read(byte[] data, uint size) + { + return (uint) this.BaseStream.Read(data, 0, (int) size); + } + } + + internal class OutStreamWrapper : StreamWrapper, ISequentialOutStream, IOutStream + { + public OutStreamWrapper(Stream baseStream) : base(baseStream) + { + } + + public int SetSize(long newSize) + { + this.BaseStream.SetLength(newSize); + return 0; + } + + public int Write(byte[] data, uint size, IntPtr processedSize) + { + this.BaseStream.Write(data, 0, (int) size); + + if (processedSize != IntPtr.Zero) + { + Marshal.WriteInt32(processedSize, (int) size); + } + + return 0; + } + } +} \ No newline at end of file From 8a365ac44551191b305ebcd43b3068ab33a49616 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 4 Sep 2020 15:00:29 -0600 Subject: [PATCH 02/13] WIP, massive rework of file extraction --- Compression.BSA/BSA/Reader/BSAReader.cs | 1 + Compression.BSA/BSADispatch.cs | 2 +- .../FileSignatures/SignatureChecker.cs | 10 +- Wabbajack.Lib/AInstaller.cs | 6 +- .../FileExtractorTests.cs | 63 +++++++ .../VirtualFileSystemTests.cs | 27 ++- Wabbajack.VirtualFileSystem/Context.cs | 44 ++++- Wabbajack.VirtualFileSystem/FileExtractor.cs | 71 +++++++- .../FileExtractor2/GatheringExtractor.cs | 139 ++++++++++++++++ .../FileExtractor2/IStreamFactory.cs | 52 ++++++ .../SevenZipExtractor/ArchiveFile.cs | 6 +- .../SevenZipExtractor/SevenZipInterface.cs | 18 +- .../SevenZipExtractor/TypedExtractor.cs | 64 +++++++ .../StagingPlan/ASubStage.cs | 25 +++ .../StagingPlan/CopyTo.cs | 23 +++ .../StagingPlan/DuplicateTo.cs | 26 +++ .../StagingPlan/IStagingPlan.cs | 156 ++++++++++++++++++ .../StagingPlan/IStagingSrc.cs | 9 + .../StagingPlan/ISubStage.cs | 10 ++ .../StagingPlan/NativeArchive.cs | 22 +++ .../StagingPlan/SubStage.cs | 21 +++ .../StagingPlan/TempSubStage.cs | 25 +++ Wabbajack.VirtualFileSystem/VirtualFile.cs | 47 +++--- .../Wabbajack.VirtualFileSystem.csproj | 1 + 24 files changed, 802 insertions(+), 66 deletions(-) create mode 100644 Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs create mode 100644 Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs create mode 100644 Wabbajack.VirtualFileSystem/FileExtractor2/IStreamFactory.cs create mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/TypedExtractor.cs create mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs create mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs create mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs create mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs create mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs create mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs create mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs create mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs create mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs diff --git a/Compression.BSA/BSA/Reader/BSAReader.cs b/Compression.BSA/BSA/Reader/BSAReader.cs index 57f23cf5..e67d1b1d 100644 --- a/Compression.BSA/BSA/Reader/BSAReader.cs +++ b/Compression.BSA/BSA/Reader/BSAReader.cs @@ -76,6 +76,7 @@ namespace Compression.BSA return bsa; } + public static BSAReader Load(AbsolutePath filename) { var bsa = new BSAReader { _fileName = filename }; diff --git a/Compression.BSA/BSADispatch.cs b/Compression.BSA/BSADispatch.cs index df8140fd..c3dcc60c 100644 --- a/Compression.BSA/BSADispatch.cs +++ b/Compression.BSA/BSADispatch.cs @@ -19,7 +19,7 @@ namespace Compression.BSA _ => throw new InvalidDataException("Filename is not a .bsa or .ba2") }; } - + private static SignatureChecker BSASignatures = new SignatureChecker(Definitions.FileType.BSA, Definitions.FileType.BA2, Definitions.FileType.TES3); public static async ValueTask MightBeBSA(AbsolutePath filename) { diff --git a/Wabbajack.Common/FileSignatures/SignatureChecker.cs b/Wabbajack.Common/FileSignatures/SignatureChecker.cs index 827b872a..b03454ec 100644 --- a/Wabbajack.Common/FileSignatures/SignatureChecker.cs +++ b/Wabbajack.Common/FileSignatures/SignatureChecker.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; @@ -21,9 +22,14 @@ namespace Wabbajack.Common.FileSignatures public async Task MatchesAsync(AbsolutePath path) { await using var fs = await path.OpenShared(); + return await MatchesAsync(fs); + } + + public async Task MatchesAsync(Stream stream) + { var buffer = new byte[_maxLength]; - fs.Position = 0; - await fs.ReadAsync(buffer); + stream.Position = 0; + await stream.ReadAsync(buffer); foreach (var (fileType, signature) in _signatures) { diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 88589be4..df948df3 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -113,6 +113,10 @@ namespace Wabbajack.Lib public async Task InstallArchives() { + await VFS.CopyTo(Queue, ModList.Directives + .OfType() + .Select(a => (VFS.Index.FileForArchiveHashPath(a.ArchiveHashPath), a.To.RelativeTo(OutputFolder)))); + /* Info("Installing Archives"); Info("Grouping Install Files"); var grouped = ModList.Directives @@ -125,7 +129,7 @@ namespace Wabbajack.Lib .ToList(); Info("Installing Archives"); - await archives.PMap(Queue, UpdateTracker,a => InstallArchive(Queue, a.Archive, a.AbsolutePath, grouped[a.Archive.Hash])); + await archives.PMap(Queue, UpdateTracker,a => InstallArchive(Queue, a.Archive, a.AbsolutePath, grouped[a.Archive.Hash]));*/ } private async Task InstallArchive(WorkQueue queue, Archive archive, AbsolutePath absolutePath, IGrouping grouping) diff --git a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs new file mode 100644 index 00000000..f166b7ae --- /dev/null +++ b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs @@ -0,0 +1,63 @@ +using System; +using System.IO.Compression; +using System.Threading.Tasks; +using Wabbajack.Common; +using Xunit; + +namespace Wabbajack.VirtualFileSystem.Test +{ + public class FileExtractorTests + { + [Fact] + public async Task CanGatherDataFromZipFiles() + { + await using var temp = await TempFolder.Create(); + await using var archive = new TempFile(); + for (int i = 0; i < 10; i ++) + { + await WriteRandomData(temp.Dir.Combine($"{i}.bin"), _rng.Next(10, 1024)); + } + + await ZipUpFolder(temp.Dir, archive.Path, false); + + var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(archive.Path), + _ => true, + async (path, sfn) => + { + await using var s = await sfn.GetStream(); + return await s.xxHashAsync(); + }); + + Assert.Equal(10, results.Count); + foreach (var (path, hash) in results) + { + Assert.Equal(await temp.Dir.Combine(path).FileHashAsync(), hash); + } + + + } + + + private static readonly Random _rng = new Random(); + private static async Task WriteRandomData(AbsolutePath path, int size) + { + var buff = new byte[size]; + _rng.NextBytes(buff); + await path.WriteAllBytesAsync(buff); + } + + private static async Task AddFile(AbsolutePath filename, string text) + { + filename.Parent.CreateDirectory(); + await filename.WriteAllTextAsync(text); + } + + private static async Task ZipUpFolder(AbsolutePath folder, AbsolutePath output, bool deleteSource = true) + { + ZipFile.CreateFromDirectory((string)folder, (string)output); + if (deleteSource) + await folder.DeleteDirectory(); + } + + } +} diff --git a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs index 9d19588e..b06dee3c 100644 --- a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs @@ -139,15 +139,15 @@ namespace Wabbajack.VirtualFileSystem.Test await AddTestRoot(); var res = new FullPath(TEST_ZIP, new[] {(RelativePath)"test.txt"}); - var file = context.Index.ByFullPath[res]; + var files = new [] {context.Index.ByFullPath[res]}; - var cleanup = await context.Stage(new List {file}); - - await using var stream = await file.StagedFile.OpenRead(); - - Assert.Equal("This is a test", await stream.ReadAllTextAsync()); + var queue = new WorkQueue(); + await context.Extract(queue, files.ToHashSet(), async (file, factory) => + { + await using var s = await factory.GetStream(); + Assert.Equal("This is a test", await s.ReadAllTextAsync()); + }); - await cleanup(); } [Fact] @@ -165,16 +165,13 @@ namespace Wabbajack.VirtualFileSystem.Test var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")]; - var cleanup = await context.Stage(files); - - foreach (var file in files) + var queue = new WorkQueue(); + await context.Extract(queue, files.ToHashSet(), async (file, factory) => { - await using var stream = await file.StagedFile.OpenRead(); - - Assert.Equal("This is a test", await stream.ReadAllTextAsync()); - } + await using var s = await factory.GetStream(); + Assert.Equal("This is a test", await s.ReadAllTextAsync()); + }); - await cleanup(); } private static async Task AddFile(AbsolutePath filename, string text) diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index eb1a29dc..f980d9bf 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -71,7 +71,7 @@ namespace Wabbajack.VirtualFileSystem return found; } - return await VirtualFile.Analyze(this, null, new RootDiskFile(f), f, 0); + return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, 0); }); var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList()); @@ -103,7 +103,7 @@ namespace Wabbajack.VirtualFileSystem return found; } - return await VirtualFile.Analyze(this, null, new RootDiskFile(f), f, 0); + return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, 0); }); var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList()); @@ -196,6 +196,40 @@ namespace Wabbajack.VirtualFileSystem } } + /// + /// Extract the set of files and call the callback for each, handing it a stream factory and the virtual file, + /// top level archives (native archives) will be processed in parallel. Duplicate files will not be + /// + /// + /// + /// + public async Task Extract(WorkQueue queue, HashSet files, Func callback) + { + var top = new VirtualFile(); + var filesByParent = files.SelectMany(f => f.FilesInFullPath) + .Distinct() + .GroupBy(f => f.Parent ?? top) + .ToDictionary(f => f.Key); + + async Task HandleFile(VirtualFile file, IStreamFactory sfn) + { + if (files.Contains(file)) await callback(file, sfn); + if (filesByParent.TryGetValue(file, out var children)) + { + var fileNames = children.ToDictionary(c => c.RelativeName); + await FileExtractor2.GatheringExtract(sfn, + r => fileNames.ContainsKey(r), + async (rel, csf) => + { + await HandleFile(fileNames[rel], sfn); + return 0; + }); + } + + } + await filesByParent[top].PMap(queue, async file => await HandleFile(file, new NativeFileStreamFactory(file.AbsoluteName))); + } + public async Task> Stage(IEnumerable files) { await _cleanupTask; @@ -227,6 +261,12 @@ namespace Wabbajack.VirtualFileSystem }; } + public async Task CopyTo(WorkQueue queue, IEnumerable<(VirtualFile src, AbsolutePath dest)> directives) + { + var plans = StagingPlan.StagingPlan.CreatePlan(directives).ToArray(); + await StagingPlan.StagingPlan.ExecutePlans(queue, plans); + } + public async Task> StageWith(IEnumerable files) { return new AsyncDisposableList(await Stage(files), files); diff --git a/Wabbajack.VirtualFileSystem/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor.cs index c78d95c4..850f6053 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.IO; using System.Linq; using System.Reactive.Linq; using System.Threading.Tasks; @@ -19,22 +20,21 @@ namespace Wabbajack.VirtualFileSystem { public class FileExtractor { - - private static SignatureChecker archiveSigs = new SignatureChecker(Definitions.FileType.TES3, + public static readonly SignatureChecker ArchiveSigs = new SignatureChecker(Definitions.FileType.TES3, Definitions.FileType.BSA, Definitions.FileType.BA2, Definitions.FileType.ZIP, Definitions.FileType.EXE, Definitions.FileType.RAR, Definitions.FileType._7Z); - + public static async Task ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable OnlyFiles = null, bool throwOnError = true) { OnlyFiles ??= new RelativePath[0]; try { - var sig = await archiveSigs.MatchesAsync(source); + var sig = await ArchiveSigs.MatchesAsync(source); if (source.Extension == Consts.OMOD) return await ExtractAllWithOMOD(source); @@ -71,7 +71,7 @@ namespace Wabbajack.VirtualFileSystem if (isArchive) { - return await ExtractAllWith7Zip(source, null); + return await ExtractAllWith7Zip(source, (IEnumerable) null); } var dest = await TempFolder.Create(); @@ -183,7 +183,7 @@ namespace Wabbajack.VirtualFileSystem /// public static async Task CanExtract(AbsolutePath v) { - var found = await archiveSigs.MatchesAsync(v); + var found = await ArchiveSigs.MatchesAsync(v); switch (found) { case null: @@ -220,5 +220,64 @@ namespace Wabbajack.VirtualFileSystem { return ext == _exeExtension || Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext); } + + /// + /// Extract the specific files to the specific locations + /// + /// + /// + /// + /// + public static async Task ExtractTo(WorkQueue queue, AbsolutePath source, Dictionary indexed) + { + try + { + var sig = await ArchiveSigs.MatchesAsync(source); + + /*if (source.Extension == Consts.OMOD) + return await ExtractAllWithOMOD(source);*/ + + switch (sig) + { + case Definitions.FileType.BSA: + case Definitions.FileType.TES3: + case Definitions.FileType.BA2: + await ExtractAllWithBSA(queue, source, indexed); + return; + case Definitions.FileType.EXE: + await ExtractAllExe(source, indexed); + return; + case Definitions.FileType._7Z: + case Definitions.FileType.ZIP: + case Definitions.FileType.RAR: + await ExtractAllWith7Zip(source, indexed); + return; + } + throw new Exception("Invalid archive format"); + } + catch (Exception ex) + { + Utils.Log(ex.ToString()); + Utils.ErrorThrow(ex, $"Error while extracting {source}"); + throw new Exception(); + } + } + + private static async Task ExtractAllWith7Zip(AbsolutePath source, Dictionary onlyFiles) + { + using var archive = await ArchiveFile.Open(source); + await archive.Extract(f => onlyFiles.TryGetValue(f, out var dest) ? dest : default); + } + + private static async Task ExtractAllExe(AbsolutePath source, Dictionary indexed) + { + throw new NotImplementedException(); + } + + private static async Task ExtractAllWithBSA(WorkQueue queue, AbsolutePath source, Dictionary indexed) + { + throw new NotImplementedException(); + + } } } diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs new file mode 100644 index 00000000..ba193e04 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Common.FileSignatures; +using Wabbajack.VirtualFileSystem.SevenZipExtractor; + +namespace Wabbajack.VirtualFileSystem +{ + public class GatheringExtractor : IArchiveExtractCallback + { + private ArchiveFile _archive; + private Predicate _shouldExtract; + private Func> _mapFn; + private Dictionary _results; + private Dictionary _indexes; + private Stream _stream; + private Definitions.FileType _sig; + + public GatheringExtractor(Stream stream, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) + { + + _shouldExtract = shouldExtract; + _mapFn = mapfn; + _results = new Dictionary(); + _stream = stream; + _sig = sig; + } + + public async Task> Extract() + { + var source = new TaskCompletionSource(); + + var th = new Thread(() => + { + try + { + _archive = ArchiveFile.Open(_stream, _sig).Result; + _indexes = _archive.Entries + .Where(f => !f.IsFolder) + .Select((entry, idx) => ((RelativePath)entry.FileName, (uint)idx)) + .Where(t => _shouldExtract(t.Item1)) + .ToDictionary(t => t.Item2, t => t.Item1); + + + _archive._archive.Extract(null, 0xFFFFFFFF, 0, this); + _archive.Dispose(); + source.SetResult(true); + } + catch (Exception ex) + { + source.SetException(ex); + } + + }) {Priority = ThreadPriority.BelowNormal, Name = "7Zip Extraction Worker Thread"}; + th.Start(); + + + + await source.Task; + return _results; + } + + public void SetTotal(ulong total) + { + + } + + public void SetCompleted(ref ulong completeValue) + { + + } + + public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode) + { + if (_indexes.ContainsKey(index)) + { + outStream = new GatheringExtractorStream(this, index); + return 0; + } + + outStream = null; + return 0; + } + + public void PrepareOperation(AskMode askExtractMode) + { + + } + + public void SetOperationResult(OperationResult resultEOperationResult) + { + + } + + private class GatheringExtractorStream : ISequentialOutStream, IOutStream + { + private GatheringExtractor _extractor; + private uint _index; + + public GatheringExtractorStream(GatheringExtractor extractor, uint index) + { + _extractor = extractor; + _index = index; + } + + public int Write(IntPtr data, uint size, IntPtr processedSize) + { + unsafe + { + var result = _extractor._mapFn(_extractor._indexes[_index], new UnmanagedStreamFactory((byte*)data, size)).AsTask().Result; + + _extractor._results[_extractor._indexes[_index]] = result; + + if (processedSize != IntPtr.Zero) + { + Marshal.WriteInt32(processedSize, (int) size); + } + + return 0; + } + } + + public void Seek(long offset, uint seekOrigin, IntPtr newPosition) + { + + } + + public int SetSize(long newSize) + { + return 0; + } + } + } +} diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/IStreamFactory.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/IStreamFactory.cs new file mode 100644 index 00000000..e942ba5a --- /dev/null +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/IStreamFactory.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem +{ + public interface IStreamFactory + { + Task GetStream(); + + DateTime LastModifiedUtc { get; } + + } + + public class UnmanagedStreamFactory : IStreamFactory + { + private readonly unsafe byte* _data; + private readonly long _size; + + public unsafe UnmanagedStreamFactory(byte* data, long size) + { + _data = data; + _size = size; + } + public async Task GetStream() + { + unsafe + { + return new UnmanagedMemoryStream(_data, _size); + } + } + + public DateTime LastModifiedUtc => DateTime.UtcNow; + } + + public class NativeFileStreamFactory : IStreamFactory + { + private AbsolutePath _file; + + public NativeFileStreamFactory(AbsolutePath file) + { + _file = file; + } + public async Task GetStream() + { + return await _file.OpenRead(); + } + + public DateTime LastModifiedUtc => _file.LastModifiedUtc; + } +} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs index 7de4369c..c16810c8 100644 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs @@ -12,7 +12,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor public class ArchiveFile : IDisposable { private SevenZipHandle _sevenZipHandle; - private IInArchive _archive; + internal IInArchive _archive; private InStreamWrapper _archiveStream; private IList _entries; @@ -42,11 +42,11 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor return self; } - public static async Task Open(Stream archiveStream, SevenZipFormat? format = null, string libraryFilePath = null) + public static async Task Open(Stream archiveStream, Definitions.FileType format) { var self = new ArchiveFile(); self.InitializeAndValidateLibrary(); - self._archive = self._sevenZipHandle.CreateInArchive(Formats.FormatGuidMapping[SevenZipFormat.SevenZip]); + self._archive = self._sevenZipHandle.CreateInArchive(Formats.FileTypeGuidMapping[format]); self._archiveStream = new InStreamWrapper(archiveStream); return self; } diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs index 91666547..cfc0d371 100644 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs @@ -136,14 +136,14 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor //string CryptoGetTextPassword(); } - internal enum AskMode : int + public enum AskMode : int { kExtract = 0, kTest, kSkip } - internal enum OperationResult : int + public enum OperationResult : int { kOK = 0, kUnSupportedMethod, @@ -203,11 +203,11 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor [ComImport] [Guid("23170F69-40C1-278A-0000-000300020000")] [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - internal interface ISequentialOutStream + public interface ISequentialOutStream { [PreserveSig] int Write( - [In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, + IntPtr data, uint size, IntPtr processedSize); // ref uint processedSize /* @@ -246,7 +246,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor { [PreserveSig] int Write( - [In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, + IntPtr data, uint size, IntPtr processedSize); // ref uint processedSize @@ -444,16 +444,18 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor return 0; } - public int Write(byte[] data, uint size, IntPtr processedSize) + public int Write(IntPtr data, uint size, IntPtr processedSize) { + throw new NotImplementedException(); + /* this.BaseStream.Write(data, 0, (int) size); if (processedSize != IntPtr.Zero) { Marshal.WriteInt32(processedSize, (int) size); } - +*/ return 0; } } -} \ No newline at end of file +} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/TypedExtractor.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/TypedExtractor.cs new file mode 100644 index 00000000..6904ff87 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/TypedExtractor.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem.SevenZipExtractor +{ + internal class TypedExtractor : IArchiveExtractCallback + { + private Dictionary _mappings; + private Action> _callback; + private Dictionary _indexToFile; + + public TypedExtractor(Dictionary mappings, Action> callback) + { + _mappings = mappings; + _callback = callback; + } + + public void Extract(ArchiveFile file) + { + _indexToFile = new Dictionary(); + + uint idx = 0; + foreach (var entry in file.Entries) + { + var rel = (RelativePath)entry.FileName; + if (_mappings.ContainsKey(rel)) ; + { + _indexToFile.Add(idx, rel); + } + idx += 1; + } + + file._archive.Extract(null, 0xFFFFFFFF, 0, this); + } + + public void SetTotal(ulong total) + { + throw new System.NotImplementedException(); + } + + public void SetCompleted(ref ulong completeValue) + { + throw new System.NotImplementedException(); + } + + public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode) + { + outStream = null; + throw new System.NotImplementedException(); + } + + public void PrepareOperation(AskMode askExtractMode) + { + throw new System.NotImplementedException(); + } + + public void SetOperationResult(OperationResult resultEOperationResult) + { + throw new System.NotImplementedException(); + } + } +} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs b/Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs new file mode 100644 index 00000000..bef272f3 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs @@ -0,0 +1,25 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem.StagingPlan +{ + public abstract class ASubStage : ISubStage + { + private IEnumerable _plans; + + public ASubStage(IEnumerable plans) + { + _plans = plans; + } + + public abstract ValueTask DisposeAsync(); + public abstract AbsolutePath Destination { get; } + public abstract IPath Source { get; } + + public async Task Execute(WorkQueue queue) + { + await StagingPlan.ExecutePlan(queue, async () => await Destination.OpenWrite(), _plans); + } + } +} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs b/Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs new file mode 100644 index 00000000..056be6bf --- /dev/null +++ b/Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs @@ -0,0 +1,23 @@ +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem.StagingPlan +{ + public class CopyTo : IStagingSrc + { + public CopyTo(IPath src, AbsolutePath destination) + { + Destination = destination; + Source = src; + } + + public AbsolutePath Destination { get; } + + public async ValueTask DisposeAsync() + { + + } + + public IPath Source { get; } + } +} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs b/Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs new file mode 100644 index 00000000..8255e229 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs @@ -0,0 +1,26 @@ +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem.StagingPlan +{ + public class DuplicateTo : IStagingPlan + { + private readonly IStagingPlan _src; + + public DuplicateTo(IStagingPlan src, AbsolutePath destination) + { + _src = src; + Destination = destination; + } + public async ValueTask DisposeAsync() + { + } + + public async Task Execute() + { + await _src.Destination.CopyToAsync(Destination); + } + + public AbsolutePath Destination { get; } + } +} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs b/Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs new file mode 100644 index 00000000..a6e5cba9 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Common.FileSignatures; +using Wabbajack.VirtualFileSystem.SevenZipExtractor; + +namespace Wabbajack.VirtualFileSystem.StagingPlan +{ + public interface IStagingPlan : IAsyncDisposable + { + AbsolutePath Destination { get; } + } + + public static class StagingPlan + { + + public static IEnumerable CreatePlan(IEnumerable<(VirtualFile src, AbsolutePath dest)> directives) + { + var top = new VirtualFile(); + var original = directives.ToHashSet(); + var files = directives.GroupBy(f => f.src).ToDictionary(f => f.Key); + + var childrenForParent = files.Keys.GroupBy(f => f.Parent ?? top).ToDictionary(f => f.Key); + + var allFilesForParent = files.Select(f => f.Key) + .SelectMany(f => f.FilesInFullPath) + .Distinct() + .GroupBy(f => f.Parent ?? top) + .ToDictionary(f => f.Key, f => f.ToArray()); + + var baseDirectives = new List(); + + // Recursive internal function to get the plans for a given parent, we'll then start at the + // null parent (the top) and recurse our way down into all the children files. + IEnumerable GetPlans(VirtualFile parent) + { + foreach (var forParent in allFilesForParent[parent]) + { + // Do we need files inside this file? + if (childrenForParent.TryGetValue(forParent, out var children)) + { + if (files.TryGetValue(forParent, out var copies)) + { + ASubStage subStage; + if (parent == top) + { + subStage = new NativeArchive(forParent.AbsoluteName, GetPlans(forParent)); + } + else + { + subStage = new SubStage(forParent.RelativeName, copies.First().dest, GetPlans(forParent)); + } + yield return subStage; + foreach (var copy in copies) + { + yield return new DuplicateTo(subStage, copy.dest); + } + } + else + { + if (parent == top) + { + yield return new NativeArchive(forParent.AbsoluteName, GetPlans(forParent)); + } + else + { + yield return new TempSubStage(forParent.RelativeName, GetPlans(forParent)); + } + } + } + else + { + // If not then we need to copy this file around + var copies = files[forParent]; + var firstCopy = new CopyTo(copies.Key.Name, copies.First().dest); + yield return firstCopy; + + foreach (var duplicate in copies.Skip(1)) + { + yield return new DuplicateTo(firstCopy, duplicate.dest); + } + } + } + } + + return GetPlans(top); + } + public static async ValueTask ExecutePlan(WorkQueue queue, Func> src, IEnumerable plans) + { + // Extract these files + + await using var stream = await src(); + var sig = await FileExtractor.ArchiveSigs.MatchesAsync(stream); + stream.Position = 0; + + switch (sig) + { + case Definitions.FileType.ZIP: + await ExtractWith7Zip((Definitions.FileType)sig, stream, plans); + + + break; + + default: + throw new Exception($"Invalid archive for extraction"); + } + + + // Copy around the duplicates + foreach (var file in plans.OfType()) + { + await file.Execute(); + } + + // Execute the sub-stages + foreach (var subStage in plans.OfType()) + { + await subStage.Execute(queue); + } + + // Dispose of all plans + foreach (var file in plans) + { + await file.DisposeAsync(); + } + + } + + private static async ValueTask ExtractWith7Zip(Definitions.FileType sig, Stream stream, IEnumerable plans) + { + using var archive = await ArchiveFile.Open(stream, sig); + + void HandleFile(RelativePath path, IStagingPlan src, Func sf) + { + + } + + var extractor = new TypedExtractor(plans.OfType().ToDictionary(s => s.Source.FileName, s => (IStagingPlan)s), + HandleFile); + extractor.Extract(archive); + + } + + + public static async Task ExecutePlans(WorkQueue queue, IEnumerable plans) + { + foreach (var file in plans.OfType()) + { + await file.Execute(queue); + } + } + } +} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs b/Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs new file mode 100644 index 00000000..6e22b250 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs @@ -0,0 +1,9 @@ +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem.StagingPlan +{ + public interface IStagingSrc : IStagingPlan + { + public IPath Source { get; } + } +} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs b/Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs new file mode 100644 index 00000000..6120194b --- /dev/null +++ b/Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs @@ -0,0 +1,10 @@ +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem.StagingPlan +{ + public interface ISubStage : IStagingSrc + { + Task Execute(WorkQueue queue); + } +} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs b/Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs new file mode 100644 index 00000000..2fcaf81b --- /dev/null +++ b/Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs @@ -0,0 +1,22 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem.StagingPlan +{ + public class NativeArchive : ASubStage + { + public NativeArchive(AbsolutePath src, IEnumerable plans) : base(plans) + { + Source = src; + Destination = src; + } + + public override async ValueTask DisposeAsync() + { + } + + public override AbsolutePath Destination { get; } + public override IPath Source { get; } + } +} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs b/Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs new file mode 100644 index 00000000..33d2dfd7 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem.StagingPlan +{ + public class SubStage : ASubStage + { + public SubStage(RelativePath src, AbsolutePath destination, IEnumerable plans) : base(plans) + { + Source = src; + Destination = destination; + } + + public override async ValueTask DisposeAsync() + { + } + public override AbsolutePath Destination { get; } + public override IPath Source { get; } + } +} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs b/Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs new file mode 100644 index 00000000..32055793 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs @@ -0,0 +1,25 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem.StagingPlan +{ + public class TempSubStage : ASubStage + { + private readonly TempFile _temp; + + public TempSubStage(RelativePath src, IEnumerable plans) : base(plans) + { + Source = src; + _temp = new TempFile(); + } + + public override async ValueTask DisposeAsync() + { + await _temp.DisposeAsync(); + } + + public override AbsolutePath Destination => _temp.Path; + public override IPath Source { get; } + } +} diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 8d962349..9a24eb3e 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -190,34 +190,26 @@ namespace Wabbajack.VirtualFileSystem } - public static async Task Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile, + public static async Task Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile, IPath relPath, int depth = 0) { - var hash = await extractedFile.HashAsync(); - - if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension)) - { - // Disabled because it isn't enabled on the server - IndexedVirtualFile result = null; //await TryGetContentsFromServer(hash); - - if (result != null) - { - Utils.Log($"Downloaded VFS data for {relPath.FileName}"); - - - return ConvertFromIndexedFile(context, result, relPath, parent, extractedFile); - } - } + await using var stream = await extractedFile.GetStream(); + var hash = await stream.xxHashAsync(); + stream.Position = 0; + var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream); + + /* TODO if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself)) return vself; + */ var self = new VirtualFile { Context = context, Name = relPath, Parent = parent, - Size = extractedFile.Size, + Size = stream.Length, LastModified = extractedFile.LastModifiedUtc.AsUnixTime(), LastAnalyzed = DateTime.Now.AsUnixTime(), Hash = hash @@ -226,20 +218,19 @@ namespace Wabbajack.VirtualFileSystem self.FillFullPath(depth); if (context.UseExtendedHashes) - self.ExtendedHashes = await ExtendedHashes.FromFile(extractedFile); + self.ExtendedHashes = await ExtendedHashes.FromStream(stream); - if (!await extractedFile.CanExtract()) return self; + // Can't extract, so return + if (!sig.HasValue) return self; try { - await using var extracted = await extractedFile.ExtractAll(context.Queue, throwOnError:false); - - var list = await extracted - .PMap(context.Queue, - file => Analyze(context, self, file.Value, file.Key, depth + 1)); - - self.Children = list.ToImmutableList(); + var list = await FileExtractor2.GatheringExtract(extractedFile, + _ => true, + async (path, sfactory) => await Analyze(context, self, sfactory, path, depth + 1)); + + self.Children = list.Values.ToImmutableList(); } catch (Exception ex) { @@ -405,10 +396,10 @@ namespace Wabbajack.VirtualFileSystem public string MD5 { get; set; } public string CRC { get; set; } - public static async ValueTask FromFile(IExtractedFile file) + public static async ValueTask FromStream(Stream stream) { var hashes = new ExtendedHashes(); - await using var stream = await file.OpenRead(); + stream.Position = 0; hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex(); stream.Position = 0; hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex(); diff --git a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj index 1a13f2af..d592ed0f 100644 --- a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj +++ b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj @@ -4,6 +4,7 @@ netstandard2.1 x64 win10-x64 + true Wabbajack.VirtualFileSystem.xml From 6602f1a89556f9f3539cc7e7536205847d268d7e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 4 Sep 2020 15:00:37 -0600 Subject: [PATCH 03/13] WIP, massive rework of file extraction --- .../FileExtractor2/FileExtractor.cs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs new file mode 100644 index 00000000..37d05eda --- /dev/null +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Common.FileSignatures; +using Wabbajack.VirtualFileSystem.SevenZipExtractor; + +namespace Wabbajack.VirtualFileSystem +{ + public static class FileExtractor2 + { + public static readonly SignatureChecker ArchiveSigs = new SignatureChecker(Definitions.FileType.TES3, + Definitions.FileType.BSA, + Definitions.FileType.BA2, + Definitions.FileType.ZIP, + Definitions.FileType.EXE, + Definitions.FileType.RAR, + Definitions.FileType._7Z); + + + public static async Task> GatheringExtract(IStreamFactory sFn, + Predicate shouldExtract, Func> mapfn) + { + await using var archive = await sFn.GetStream(); + var sig = await ArchiveSigs.MatchesAsync(archive); + archive.Position = 0; + + switch (sig) + { + case Definitions.FileType.ZIP: + return await GatheringExtractWith7Zip(archive, (Definitions.FileType)sig, shouldExtract, mapfn); + + default: + throw new Exception("Invalid file format"); + } + } + + private static async Task> GatheringExtractWith7Zip(Stream stream, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) + { + return await new GatheringExtractor(stream, sig, shouldExtract, mapfn).Extract(); + } + } +} From a847d698513c88728513538539476f3f647d6fa6 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 5 Sep 2020 08:01:32 -0600 Subject: [PATCH 04/13] Patching and extraction works, fewer failing tests --- Compression.BSA/BA2Reader.cs | 26 ++- Compression.BSA/BSA/Reader/BSAReader.cs | 14 +- Compression.BSA/BSA/Reader/FileRecord.cs | 10 +- Compression.BSA/BSADispatch.cs | 26 ++- Compression.BSA/Interfaces/IFile.cs | 1 + Compression.BSA/MemoryStreamFactory.cs | 24 +++ Compression.BSA/StreamView.cs | 82 +++++++++ Compression.BSA/TES3Reader.cs | 21 ++- Wabbajack.Common/IStreamFactory.cs | 34 ++++ Wabbajack.Common/Patches.cs | 4 +- Wabbajack.Lib/AInstaller.cs | 47 ++++-- Wabbajack.Lib/MO2Compiler.cs | 76 +++++---- Wabbajack.Lib/MO2Installer.cs | 4 +- Wabbajack.VirtualFileSystem/Context.cs | 8 +- Wabbajack.VirtualFileSystem/FileExtractor.cs | 1 + .../FileExtractor2/FileExtractor.cs | 27 ++- .../{IStreamFactory.cs => StreamFactories.cs} | 24 +-- .../StagingPlan/ASubStage.cs | 25 --- .../StagingPlan/CopyTo.cs | 23 --- .../StagingPlan/DuplicateTo.cs | 26 --- .../StagingPlan/IStagingPlan.cs | 156 ------------------ .../StagingPlan/IStagingSrc.cs | 9 - .../StagingPlan/ISubStage.cs | 10 -- .../StagingPlan/NativeArchive.cs | 22 --- .../StagingPlan/SubStage.cs | 21 --- .../StagingPlan/TempSubStage.cs | 25 --- Wabbajack.VirtualFileSystem/VirtualFile.cs | 8 +- 27 files changed, 320 insertions(+), 434 deletions(-) create mode 100644 Compression.BSA/MemoryStreamFactory.cs create mode 100644 Compression.BSA/StreamView.cs create mode 100644 Wabbajack.Common/IStreamFactory.cs rename Wabbajack.VirtualFileSystem/FileExtractor2/{IStreamFactory.cs => StreamFactories.cs} (51%) delete mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs delete mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs delete mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs delete mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs delete mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs delete mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs delete mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs delete mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs delete mode 100644 Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs diff --git a/Compression.BSA/BA2Reader.cs b/Compression.BSA/BA2Reader.cs index d6c6137b..e3abe392 100644 --- a/Compression.BSA/BA2Reader.cs +++ b/Compression.BSA/BA2Reader.cs @@ -27,7 +27,6 @@ namespace Compression.BSA public class BA2Reader : IBSAReader { - internal AbsolutePath _filename; private Stream _stream; internal BinaryReader _rdr; internal uint _version; @@ -35,15 +34,16 @@ namespace Compression.BSA internal EntryType _type; internal uint _numFiles; internal ulong _nameTableOffset; + public IStreamFactory _streamFactory; public bool UseATIFourCC { get; set; } = false; public bool HasNameTable => _nameTableOffset > 0; - public static async Task Load(AbsolutePath filename) + public static async Task Load(IStreamFactory streamFactory) { - var rdr = new BA2Reader(await filename.OpenShared()) {_filename = filename}; + var rdr = new BA2Reader(await streamFactory.GetStream()) {_streamFactory = streamFactory}; await rdr.LoadHeaders(); return rdr; } @@ -206,7 +206,7 @@ namespace Compression.BSA WriteHeader(bw); - await using var fs = await _bsa._filename.OpenRead(); + await using var fs = await _bsa._streamFactory.GetStream(); using var br = new BinaryReader(fs); foreach (var chunk in _chunks) { @@ -344,6 +344,14 @@ namespace Compression.BSA break; } } + + public async ValueTask GetStreamFactory() + { + var ms = new MemoryStream(); + await CopyDataTo(ms); + ms.Position = 0; + return new MemoryStreamFactory(ms); + } } [JsonName("BA2DX10Entry")] @@ -483,7 +491,7 @@ namespace Compression.BSA public async ValueTask CopyDataTo(Stream output) { - await using var fs = await _bsa._filename.OpenRead(); + await using var fs = await _bsa._streamFactory.GetStream(); fs.Seek((long) _offset, SeekOrigin.Begin); uint len = Compressed ? _size : _realSize; @@ -503,6 +511,14 @@ namespace Compression.BSA await output.WriteAsync(uncompressed, 0, uncompressed.Length); } } + + public async ValueTask GetStreamFactory() + { + var ms = new MemoryStream(); + await CopyDataTo(ms); + ms.Position = 0; + return new MemoryStreamFactory(ms); + } } [JsonName("BA2FileEntryState")] diff --git a/Compression.BSA/BSA/Reader/BSAReader.cs b/Compression.BSA/BSA/Reader/BSAReader.cs index e67d1b1d..cd85723d 100644 --- a/Compression.BSA/BSA/Reader/BSAReader.cs +++ b/Compression.BSA/BSA/Reader/BSAReader.cs @@ -16,7 +16,6 @@ namespace Compression.BSA public const int HeaderLength = 0x24; internal uint _fileCount; - internal AbsolutePath _fileName; internal uint _folderCount; internal uint _folderRecordOffset; private Lazy _folders = null!; @@ -24,6 +23,7 @@ namespace Compression.BSA internal string _magic = string.Empty; internal uint _totalFileNameLength; internal uint _totalFolderNameLength; + public IStreamFactory _streamFactory = new NativeFileStreamFactory(default); public VersionType HeaderType { get; private set; } @@ -56,7 +56,7 @@ namespace Compression.BSA public void Dump(Action print) { - print($"File Name: {_fileName}"); + print($"File Name: {_streamFactory.Name}"); print($"File Count: {_fileCount}"); print($"Magic: {_magic}"); @@ -67,11 +67,11 @@ namespace Compression.BSA } } - public static async ValueTask LoadAsync(AbsolutePath filename) + public static async ValueTask LoadAsync(IStreamFactory factory) { - using var stream = await filename.OpenRead().ConfigureAwait(false); + await using var stream = await factory.GetStream().ConfigureAwait(false); using var br = new BinaryReader(stream); - var bsa = new BSAReader { _fileName = filename }; + var bsa = new BSAReader { _streamFactory = factory }; bsa.LoadHeaders(br); return bsa; } @@ -79,7 +79,7 @@ namespace Compression.BSA public static BSAReader Load(AbsolutePath filename) { - var bsa = new BSAReader { _fileName = filename }; + var bsa = new BSAReader { _streamFactory = new NativeFileStreamFactory(filename)}; using var rdr = bsa.GetStream(); bsa.LoadHeaders(rdr); return bsa; @@ -87,7 +87,7 @@ namespace Compression.BSA internal BinaryReader GetStream() { - return new BinaryReader(File.Open(_fileName.ToString(), FileMode.Open, FileAccess.Read, FileShare.Read)); + return new BinaryReader(_streamFactory.GetStream().Result); } private void LoadHeaders(BinaryReader rdr) diff --git a/Compression.BSA/BSA/Reader/FileRecord.cs b/Compression.BSA/BSA/Reader/FileRecord.cs index fcfc2dab..c7a0fa54 100644 --- a/Compression.BSA/BSA/Reader/FileRecord.cs +++ b/Compression.BSA/BSA/Reader/FileRecord.cs @@ -74,7 +74,7 @@ namespace Compression.BSA public async ValueTask CopyDataTo(Stream output) { - await using var in_file = await BSA._fileName.OpenRead().ConfigureAwait(false); + await using var in_file = await BSA._streamFactory.GetStream().ConfigureAwait(false); using var rdr = new BinaryReader(in_file); rdr.BaseStream.Position = Offset; @@ -165,5 +165,13 @@ namespace Compression.BSA print($"Raw Size: {RawSize}"); print($"Index: {_index}"); } + + public async ValueTask GetStreamFactory() + { + var ms = new MemoryStream(); + await CopyDataTo(ms); + ms.Position = 0; + return new MemoryStreamFactory(ms); + } } } diff --git a/Compression.BSA/BSADispatch.cs b/Compression.BSA/BSADispatch.cs index c3dcc60c..8213a055 100644 --- a/Compression.BSA/BSADispatch.cs +++ b/Compression.BSA/BSADispatch.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Text; using System.Threading.Tasks; @@ -13,9 +15,9 @@ namespace Compression.BSA { return await BSASignatures.MatchesAsync(filename) switch { - Definitions.FileType.TES3 => await TES3Reader.Load(filename), - Definitions.FileType.BSA => await BSAReader.LoadAsync(filename), - Definitions.FileType.BA2 => await BA2Reader.Load(filename), + Definitions.FileType.TES3 => await TES3Reader.Load(new NativeFileStreamFactory(filename)), + Definitions.FileType.BSA => await BSAReader.LoadAsync(new NativeFileStreamFactory(filename)), + Definitions.FileType.BA2 => await BA2Reader.Load(new NativeFileStreamFactory(filename)), _ => throw new InvalidDataException("Filename is not a .bsa or .ba2") }; } @@ -25,5 +27,21 @@ namespace Compression.BSA { return await BSASignatures.MatchesAsync(filename) != null; } + + public static async ValueTask OpenRead(IStreamFactory sFn, Definitions.FileType sig) + { + switch(sig) + { + case Definitions.FileType.TES3: + return await TES3Reader.Load(sFn); + case Definitions.FileType.BSA: + return await BSAReader.LoadAsync(sFn); + case Definitions.FileType.BA2: + return await BA2Reader.Load(sFn); + default: + throw new Exception($"Bad archive format for {sFn.Name}"); + + } + } } } diff --git a/Compression.BSA/Interfaces/IFile.cs b/Compression.BSA/Interfaces/IFile.cs index fe8054b5..c9aca0f5 100644 --- a/Compression.BSA/Interfaces/IFile.cs +++ b/Compression.BSA/Interfaces/IFile.cs @@ -32,5 +32,6 @@ namespace Compression.BSA ValueTask CopyDataTo(Stream output); void Dump(Action print); + ValueTask GetStreamFactory(); } } diff --git a/Compression.BSA/MemoryStreamFactory.cs b/Compression.BSA/MemoryStreamFactory.cs new file mode 100644 index 00000000..c428ed9e --- /dev/null +++ b/Compression.BSA/MemoryStreamFactory.cs @@ -0,0 +1,24 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Compression.BSA +{ + public class MemoryStreamFactory : IStreamFactory + { + private readonly MemoryStream _data; + + public MemoryStreamFactory(MemoryStream data) + { + _data = data; + } + public async ValueTask GetStream() + { + return new MemoryStream(_data.GetBuffer(), 0, (int)_data.Length); + } + + public DateTime LastModifiedUtc => DateTime.UtcNow; + public IPath Name => (RelativePath)"BSA Memory Stream"; + } +} diff --git a/Compression.BSA/StreamView.cs b/Compression.BSA/StreamView.cs new file mode 100644 index 00000000..24742231 --- /dev/null +++ b/Compression.BSA/StreamView.cs @@ -0,0 +1,82 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Compression.BSA +{ + public class StreamView : Stream + { + private Stream _base; + private long _startPos; + private long _length; + + public StreamView(Stream baseStream, long startPos, long length) + { + _base = baseStream; + _startPos = startPos; + _length = length; + } + + public override void Flush() + { + throw new System.NotImplementedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + var realCount = Math.Min(count, Length - Position); + return _base.Read(buffer, offset, (int)realCount); + } + + public override long Seek(long offset, SeekOrigin origin) + { + switch (origin) + { + case SeekOrigin.Begin: + Position = offset; + return Position; + case SeekOrigin.End: + Position = _length - offset; + return Position; + case SeekOrigin.Current: + Position += offset; + return Position; + default: + throw new ArgumentOutOfRangeException(nameof(origin), origin, null); + } + } + + public override void SetLength(long value) + { + throw new System.NotImplementedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new System.NotImplementedException(); + } + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + public override long Length => _length; + + public override long Position + { + get + { + return _base.Position - _startPos; + } + set + { + _base.Position = _startPos + value; + } + } + + public override async ValueTask DisposeAsync() + { + await _base.DisposeAsync(); + } + } +} diff --git a/Compression.BSA/TES3Reader.cs b/Compression.BSA/TES3Reader.cs index 0ebe34f6..f9fd7afb 100644 --- a/Compression.BSA/TES3Reader.cs +++ b/Compression.BSA/TES3Reader.cs @@ -17,15 +17,15 @@ namespace Compression.BSA private uint _fileCount; private TES3FileEntry[] _files; internal long _dataOffset; - internal AbsolutePath _filename; + public IStreamFactory _streamFactory; - public static async ValueTask Load(AbsolutePath filename) + public static async ValueTask Load(IStreamFactory factory) { - await using var fs = await filename.OpenRead(); + await using var fs = await factory.GetStream(); using var br = new BinaryReader(fs); var rdr = new TES3Reader { - _filename = filename, + _streamFactory = factory, _versionNumber = br.ReadUInt32(), _hashTableOffset = br.ReadUInt32(), _fileCount = br.ReadUInt32() @@ -125,16 +125,27 @@ namespace Compression.BSA public async ValueTask CopyDataTo(Stream output) { - await using var fs = await Archive._filename.OpenRead(); + await using var fs = await Archive._streamFactory.GetStream(); fs.Position = Archive._dataOffset + Offset; await fs.CopyToLimitAsync(output, (int)Size); } + public async ValueTask GetStreamFactory() + { + var ms = new MemoryStream(); + await CopyDataTo(ms); + ms.Position = 0; + return new MemoryStreamFactory(ms); + } + + + public void Dump(Action print) { throw new NotImplementedException(); } + public uint Offset { get; set; } public uint NameOffset { get; set; } public uint Hash1 { get; set; } diff --git a/Wabbajack.Common/IStreamFactory.cs b/Wabbajack.Common/IStreamFactory.cs new file mode 100644 index 00000000..783c425e --- /dev/null +++ b/Wabbajack.Common/IStreamFactory.cs @@ -0,0 +1,34 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.Common +{ + public interface IStreamFactory + { + ValueTask GetStream(); + + DateTime LastModifiedUtc { get; } + + IPath Name { get; } + + } + public class NativeFileStreamFactory : IStreamFactory + { + private AbsolutePath _file; + + public NativeFileStreamFactory(AbsolutePath file) + { + _file = file; + } + public async ValueTask GetStream() + { + return await _file.OpenRead(); + } + + public DateTime LastModifiedUtc => _file.LastModifiedUtc; + public IPath Name => _file; + } + +} diff --git a/Wabbajack.Common/Patches.cs b/Wabbajack.Common/Patches.cs index 6e9f800f..2791e0dd 100644 --- a/Wabbajack.Common/Patches.cs +++ b/Wabbajack.Common/Patches.cs @@ -50,7 +50,7 @@ namespace Wabbajack.Common await patch.CopyToAsync(output); } - public static async Task CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash, + public static async Task CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, Stream? patchOutStream = null) { var key = PatchKey(srcHash, destHash); @@ -128,7 +128,7 @@ namespace Wabbajack.Common public static Task CreatePatchCached(byte[] a, byte[] b, Stream output) => PatchCache.CreatePatchCached(a, b, output); - public static Task CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash, Stream? patchOutStream = null) => + public static Task CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, Stream? patchOutStream = null) => PatchCache.CreatePatchCached(srcStream, srcHash, destStream, destHash, patchOutStream); public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch) => diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index df948df3..ed41a416 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -9,6 +9,7 @@ using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; using Wabbajack.VirtualFileSystem; +using Wabbajack.VirtualFileSystem.SevenZipExtractor; using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; @@ -35,6 +36,8 @@ namespace Wabbajack.Lib public bool UseCompression { get; set; } + public TempFolder? ExtractedModlistFolder { get; set; } = null; + public AInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters? parameters, int steps, Game game) : base(steps) @@ -45,12 +48,23 @@ namespace Wabbajack.Lib DownloadFolder = downloadFolder; SystemParameters = parameters; Game = game.MetaData(); + } + private ExtractedFiles? ExtractedModListFiles { get; set; } = null; public async Task ExtractModlist() { - ExtractedModListFiles = await FileExtractor.ExtractAll(Queue, ModListArchive); + ExtractedModlistFolder = await TempFolder.Create(); + await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(ModListArchive), _ => true, + async (path, sfn) => + { + await using var s = await sfn.GetStream(); + var fp = ExtractedModlistFolder.Dir.Combine(path); + fp.Parent.CreateDirectory(); + await fp.WriteAllAsync(s); + return 0; + }); } @@ -73,8 +87,7 @@ namespace Wabbajack.Lib public async Task LoadBytesFromPath(RelativePath path) { - await using var e = await ExtractedModListFiles![path].OpenRead(); - return await e.ReadAllAsync(); + return await ExtractedModlistFolder!.Dir.Combine(path).ReadAllBytesAsync(); } public static ModList LoadFromFile(AbsolutePath path) @@ -113,23 +126,23 @@ namespace Wabbajack.Lib public async Task InstallArchives() { - await VFS.CopyTo(Queue, ModList.Directives - .OfType() - .Select(a => (VFS.Index.FileForArchiveHashPath(a.ArchiveHashPath), a.To.RelativeTo(OutputFolder)))); - /* - Info("Installing Archives"); - Info("Grouping Install Files"); var grouped = ModList.Directives .OfType() - .GroupBy(e => e.ArchiveHashPath.BaseHash) - .ToDictionary(k => k.Key); - var archives = ModList.Archives - .Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) }) - .Where(a => a.AbsolutePath != null) - .ToList(); + .Select(a => new {VF = VFS.Index.FileForArchiveHashPath(a.ArchiveHashPath), Directive = a}) + .GroupBy(a => a.VF) + .ToDictionary(a => a.Key); - Info("Installing Archives"); - await archives.PMap(Queue, UpdateTracker,a => InstallArchive(Queue, a.Archive, a.AbsolutePath, grouped[a.Archive.Hash]));*/ + if (grouped.Count == 0) return; + + await VFS.Extract(Queue, grouped.Keys.ToHashSet(), async (vf, sf) => + { + await using var s = await sf.GetStream(); + foreach (var directive in grouped[vf]) + { + s.Position = 0; + await directive.Directive.To.RelativeTo(OutputFolder).WriteAllAsync(s, false); + } + }); } private async Task InstallArchive(WorkQueue queue, Archive archive, AbsolutePath absolutePath, IGrouping grouping) diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 6a3651bc..9c980fc6 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -6,10 +6,12 @@ using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using AngleSharp.Common; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.Validation; +using Wabbajack.VirtualFileSystem; using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib @@ -495,17 +497,35 @@ namespace Wabbajack.Lib .ToArray(); if (toBuild.Length == 0) return; + + // Extract all the source files + var indexed = toBuild.GroupBy(f => (VFS.Index.FileForArchiveHashPath(f.ArchiveHashPath))) + .ToDictionary(f => f.Key); + await VFS.Extract(Queue, indexed.Keys.ToHashSet(), + async (vf, sf) => + { + // For each, extract the destination + var matches = indexed[vf]; + using var iqueue = new WorkQueue(1); + foreach (var match in matches) + { + var destFile = FindDestFile(match.To); + // Build the patch + await VFS.Extract(iqueue, new[] {destFile}.ToHashSet(), + async (destvf, destsfn) => + { + Info($"Patching {match.To}"); + Status($"Patching {match.To}"); + await using var srcStream = await sf.GetStream(); + await using var destStream = await destsfn.GetStream(); + var patchSize = await Utils.CreatePatchCached(srcStream, vf.Hash, destStream, destvf.Hash); + Info($"Patch size {patchSize} for {match.To}"); + }); + } + + }); - var groups = toBuild - .Where(p => p.PatchID == default) - .GroupBy(p => p.ArchiveHashPath.BaseHash) - .ToList(); - - Info($"Patching building patches from {groups.Count} archives"); - var absolutePaths = AllFiles.ToDictionary(e => e.Path, e => e.AbsolutePath); - await groups.PMap(Queue, group => BuildArchivePatches(group.Key, group, absolutePaths)); - - + // Load in the patches await InstallDirectives.OfType() .Where(p => p.PatchID == default) .PMap(Queue, async pfa => @@ -514,6 +534,7 @@ namespace Wabbajack.Lib .Select(c => (Utils.TryGetPatch(c.Hash, pfa.Hash, out var data), data, c)) .ToArray(); + // Pick the best patch if (patches.All(p => p.Item1)) { var (_, bytes, file) = IncludePatches.PickPatch(this, patches); @@ -529,42 +550,19 @@ namespace Wabbajack.Lib Error($"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}"); } - private async Task BuildArchivePatches(Hash archiveSha, IEnumerable group, - Dictionary absolutePaths) + private VirtualFile FindDestFile(RelativePath to) { - await using var files = await VFS.StageWith(@group.Select(g => VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath))); - var byPath = files.GroupBy(f => string.Join("|", f.FilesInFullPath.Skip(1).Select(i => i.Name))) - .ToDictionary(f => f.Key, f => f.First()); - // Now Create the patches - await @group.PMap(Queue, async entry => - { - Info($"Patching {entry.To}"); - Status($"Patching {entry.To}"); - var srcFile = byPath[string.Join("|", entry.ArchiveHashPath.Paths)]; - await using var srcStream = await srcFile.OpenRead(); - await using var destStream = await LoadDataForTo(entry.To, absolutePaths); - var patchSize = await Utils.CreatePatchCached(srcStream, srcFile.Hash, destStream, entry.Hash); - Info($"Patch size {patchSize} for {entry.To}"); - }); - } - - private async Task LoadDataForTo(RelativePath to, Dictionary absolutePaths) - { - if (absolutePaths.TryGetValue(to, out var absolute)) - return await absolute.OpenRead(); + var abs = to.RelativeTo(MO2Folder); + if (abs.Exists) + return VFS.Index.ByRootPath[abs]; if (to.StartsWith(Consts.BSACreationDir)) { var bsaId = (RelativePath)((string)to).Split('\\')[1]; var bsa = InstallDirectives.OfType().First(b => b.TempID == bsaId); - - var a = await BSADispatch.OpenRead(MO2Folder.Combine(bsa.To)); var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray()); - var file = a.Files.First(e => e.Path == find); - var returnStream = new TempStream(); - await file.CopyDataTo(returnStream); - returnStream.Position = 0; - return returnStream; + + return VFS.Index.ByRootPath[MO2Folder.Combine(bsa.To)].Children.First(c => c.RelativeName == find); } throw new ArgumentException($"Couldn't load data for {to}"); diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index 92877a92..8d1f3e6a 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -180,6 +180,7 @@ namespace Wabbajack.Lib SetScreenSizeInPrefs(); UpdateTracker.NextStep("Installation complete! You may exit the program."); + await ExtractedModlistFolder!.DisposeAsync(); await Metrics.Send(Metrics.FinishInstall, ModList.Name); return true; @@ -243,7 +244,8 @@ namespace Wabbajack.Lib Status($"Writing included .meta file {directive.To}"); var outPath = DownloadFolder.Combine(directive.To); if (outPath.IsFile) await outPath.DeleteAsync(); - await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID)); + var bytes = await LoadBytesFromPath(directive.SourceDataID); + await outPath.WriteAllBytesAsync(bytes); }); } diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index f980d9bf..9b6ba5cb 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -221,7 +221,7 @@ namespace Wabbajack.VirtualFileSystem r => fileNames.ContainsKey(r), async (rel, csf) => { - await HandleFile(fileNames[rel], sfn); + await HandleFile(fileNames[rel], csf); return 0; }); } @@ -261,12 +261,6 @@ namespace Wabbajack.VirtualFileSystem }; } - public async Task CopyTo(WorkQueue queue, IEnumerable<(VirtualFile src, AbsolutePath dest)> directives) - { - var plans = StagingPlan.StagingPlan.CreatePlan(directives).ToArray(); - await StagingPlan.StagingPlan.ExecutePlans(queue, plans); - } - public async Task> StageWith(IEnumerable files) { return new AsyncDisposableList(await Stage(files), files); diff --git a/Wabbajack.VirtualFileSystem/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor.cs index 850f6053..f736dbea 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor.cs @@ -30,6 +30,7 @@ namespace Wabbajack.VirtualFileSystem public static async Task ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable OnlyFiles = null, bool throwOnError = true) { + throw new NotImplementedException(); OnlyFiles ??= new RelativePath[0]; try diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs index 37d05eda..eaf965bf 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; +using Compression.BSA; using Wabbajack.Common; using Wabbajack.Common.FileSignatures; using Wabbajack.VirtualFileSystem.SevenZipExtractor; @@ -14,7 +15,7 @@ namespace Wabbajack.VirtualFileSystem Definitions.FileType.BSA, Definitions.FileType.BA2, Definitions.FileType.ZIP, - Definitions.FileType.EXE, + //Definitions.FileType.EXE, Definitions.FileType.RAR, Definitions.FileType._7Z); @@ -31,11 +32,33 @@ namespace Wabbajack.VirtualFileSystem case Definitions.FileType.ZIP: return await GatheringExtractWith7Zip(archive, (Definitions.FileType)sig, shouldExtract, mapfn); + case Definitions.FileType.TES3: + case Definitions.FileType.BSA: + case Definitions.FileType.BA2: + return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn); + + default: - throw new Exception("Invalid file format"); + throw new Exception($"Invalid file format {sFn.Name}"); } } + private static async Task> GatheringExtractWithBSA(IStreamFactory sFn, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) + { + var archive = await BSADispatch.OpenRead(sFn, sig); + var results = new Dictionary(); + foreach (var entry in archive.Files) + { + if (!shouldExtract(entry.Path)) + continue; + + var result = await mapfn(entry.Path, await entry.GetStreamFactory()); + results.Add(entry.Path, result); + } + + return results; + } + private static async Task> GatheringExtractWith7Zip(Stream stream, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) { return await new GatheringExtractor(stream, sig, shouldExtract, mapfn).Extract(); diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/IStreamFactory.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/StreamFactories.cs similarity index 51% rename from Wabbajack.VirtualFileSystem/FileExtractor2/IStreamFactory.cs rename to Wabbajack.VirtualFileSystem/FileExtractor2/StreamFactories.cs index e942ba5a..83294259 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/IStreamFactory.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/StreamFactories.cs @@ -5,13 +5,6 @@ using Wabbajack.Common; namespace Wabbajack.VirtualFileSystem { - public interface IStreamFactory - { - Task GetStream(); - - DateTime LastModifiedUtc { get; } - - } public class UnmanagedStreamFactory : IStreamFactory { @@ -23,7 +16,7 @@ namespace Wabbajack.VirtualFileSystem _data = data; _size = size; } - public async Task GetStream() + public async ValueTask GetStream() { unsafe { @@ -32,21 +25,8 @@ namespace Wabbajack.VirtualFileSystem } public DateTime LastModifiedUtc => DateTime.UtcNow; + public IPath Name => (RelativePath)"Unmanaged Memory Stream"; } - public class NativeFileStreamFactory : IStreamFactory - { - private AbsolutePath _file; - public NativeFileStreamFactory(AbsolutePath file) - { - _file = file; - } - public async Task GetStream() - { - return await _file.OpenRead(); - } - - public DateTime LastModifiedUtc => _file.LastModifiedUtc; - } } diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs b/Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs deleted file mode 100644 index bef272f3..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public abstract class ASubStage : ISubStage - { - private IEnumerable _plans; - - public ASubStage(IEnumerable plans) - { - _plans = plans; - } - - public abstract ValueTask DisposeAsync(); - public abstract AbsolutePath Destination { get; } - public abstract IPath Source { get; } - - public async Task Execute(WorkQueue queue) - { - await StagingPlan.ExecutePlan(queue, async () => await Destination.OpenWrite(), _plans); - } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs b/Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs deleted file mode 100644 index 056be6bf..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public class CopyTo : IStagingSrc - { - public CopyTo(IPath src, AbsolutePath destination) - { - Destination = destination; - Source = src; - } - - public AbsolutePath Destination { get; } - - public async ValueTask DisposeAsync() - { - - } - - public IPath Source { get; } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs b/Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs deleted file mode 100644 index 8255e229..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public class DuplicateTo : IStagingPlan - { - private readonly IStagingPlan _src; - - public DuplicateTo(IStagingPlan src, AbsolutePath destination) - { - _src = src; - Destination = destination; - } - public async ValueTask DisposeAsync() - { - } - - public async Task Execute() - { - await _src.Destination.CopyToAsync(Destination); - } - - public AbsolutePath Destination { get; } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs b/Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs deleted file mode 100644 index a6e5cba9..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs +++ /dev/null @@ -1,156 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Wabbajack.Common; -using Wabbajack.Common.FileSignatures; -using Wabbajack.VirtualFileSystem.SevenZipExtractor; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public interface IStagingPlan : IAsyncDisposable - { - AbsolutePath Destination { get; } - } - - public static class StagingPlan - { - - public static IEnumerable CreatePlan(IEnumerable<(VirtualFile src, AbsolutePath dest)> directives) - { - var top = new VirtualFile(); - var original = directives.ToHashSet(); - var files = directives.GroupBy(f => f.src).ToDictionary(f => f.Key); - - var childrenForParent = files.Keys.GroupBy(f => f.Parent ?? top).ToDictionary(f => f.Key); - - var allFilesForParent = files.Select(f => f.Key) - .SelectMany(f => f.FilesInFullPath) - .Distinct() - .GroupBy(f => f.Parent ?? top) - .ToDictionary(f => f.Key, f => f.ToArray()); - - var baseDirectives = new List(); - - // Recursive internal function to get the plans for a given parent, we'll then start at the - // null parent (the top) and recurse our way down into all the children files. - IEnumerable GetPlans(VirtualFile parent) - { - foreach (var forParent in allFilesForParent[parent]) - { - // Do we need files inside this file? - if (childrenForParent.TryGetValue(forParent, out var children)) - { - if (files.TryGetValue(forParent, out var copies)) - { - ASubStage subStage; - if (parent == top) - { - subStage = new NativeArchive(forParent.AbsoluteName, GetPlans(forParent)); - } - else - { - subStage = new SubStage(forParent.RelativeName, copies.First().dest, GetPlans(forParent)); - } - yield return subStage; - foreach (var copy in copies) - { - yield return new DuplicateTo(subStage, copy.dest); - } - } - else - { - if (parent == top) - { - yield return new NativeArchive(forParent.AbsoluteName, GetPlans(forParent)); - } - else - { - yield return new TempSubStage(forParent.RelativeName, GetPlans(forParent)); - } - } - } - else - { - // If not then we need to copy this file around - var copies = files[forParent]; - var firstCopy = new CopyTo(copies.Key.Name, copies.First().dest); - yield return firstCopy; - - foreach (var duplicate in copies.Skip(1)) - { - yield return new DuplicateTo(firstCopy, duplicate.dest); - } - } - } - } - - return GetPlans(top); - } - public static async ValueTask ExecutePlan(WorkQueue queue, Func> src, IEnumerable plans) - { - // Extract these files - - await using var stream = await src(); - var sig = await FileExtractor.ArchiveSigs.MatchesAsync(stream); - stream.Position = 0; - - switch (sig) - { - case Definitions.FileType.ZIP: - await ExtractWith7Zip((Definitions.FileType)sig, stream, plans); - - - break; - - default: - throw new Exception($"Invalid archive for extraction"); - } - - - // Copy around the duplicates - foreach (var file in plans.OfType()) - { - await file.Execute(); - } - - // Execute the sub-stages - foreach (var subStage in plans.OfType()) - { - await subStage.Execute(queue); - } - - // Dispose of all plans - foreach (var file in plans) - { - await file.DisposeAsync(); - } - - } - - private static async ValueTask ExtractWith7Zip(Definitions.FileType sig, Stream stream, IEnumerable plans) - { - using var archive = await ArchiveFile.Open(stream, sig); - - void HandleFile(RelativePath path, IStagingPlan src, Func sf) - { - - } - - var extractor = new TypedExtractor(plans.OfType().ToDictionary(s => s.Source.FileName, s => (IStagingPlan)s), - HandleFile); - extractor.Extract(archive); - - } - - - public static async Task ExecutePlans(WorkQueue queue, IEnumerable plans) - { - foreach (var file in plans.OfType()) - { - await file.Execute(queue); - } - } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs b/Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs deleted file mode 100644 index 6e22b250..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs +++ /dev/null @@ -1,9 +0,0 @@ -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public interface IStagingSrc : IStagingPlan - { - public IPath Source { get; } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs b/Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs deleted file mode 100644 index 6120194b..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public interface ISubStage : IStagingSrc - { - Task Execute(WorkQueue queue); - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs b/Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs deleted file mode 100644 index 2fcaf81b..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public class NativeArchive : ASubStage - { - public NativeArchive(AbsolutePath src, IEnumerable plans) : base(plans) - { - Source = src; - Destination = src; - } - - public override async ValueTask DisposeAsync() - { - } - - public override AbsolutePath Destination { get; } - public override IPath Source { get; } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs b/Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs deleted file mode 100644 index 33d2dfd7..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public class SubStage : ASubStage - { - public SubStage(RelativePath src, AbsolutePath destination, IEnumerable plans) : base(plans) - { - Source = src; - Destination = destination; - } - - public override async ValueTask DisposeAsync() - { - } - public override AbsolutePath Destination { get; } - public override IPath Source { get; } - } -} diff --git a/Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs b/Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs deleted file mode 100644 index 32055793..00000000 --- a/Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.StagingPlan -{ - public class TempSubStage : ASubStage - { - private readonly TempFile _temp; - - public TempSubStage(RelativePath src, IEnumerable plans) : base(plans) - { - Source = src; - _temp = new TempFile(); - } - - public override async ValueTask DisposeAsync() - { - await _temp.DisposeAsync(); - } - - public override AbsolutePath Destination => _temp.Path; - public override IPath Source { get; } - } -} diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 9a24eb3e..e6d30bac 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -141,7 +141,7 @@ namespace Wabbajack.VirtualFileSystem itm.ThisAndAllChildrenReduced(fn); } - private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IExtractedFile extractedFile) + private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IStreamFactory extractedFile) { var vself = new VirtualFile { @@ -161,7 +161,7 @@ namespace Wabbajack.VirtualFileSystem return vself; } - private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IExtractedFile extractedFile, Hash hash, out VirtualFile found) + private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile, Hash hash, out VirtualFile found) { var result = _vfsCache.Get(hash.ToArray()); if (result == null) @@ -199,11 +199,9 @@ namespace Wabbajack.VirtualFileSystem var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream); - /* TODO if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself)) return vself; - */ - + var self = new VirtualFile { Context = context, From 1ebb0b6492024ba5ac779921f06b6bde76df928d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 5 Sep 2020 13:36:44 -0600 Subject: [PATCH 05/13] All sanity tests work! --- Wabbajack.Lib/ACompiler.cs | 31 +++++++++++++++++-- Wabbajack.Lib/AInstaller.cs | 28 +++++++++++++++-- .../CompilationSteps/DeconstructBSAs.cs | 2 +- Wabbajack.Lib/CompilationSteps/IncludeAll.cs | 3 +- Wabbajack.Lib/Data.cs | 3 ++ Wabbajack.Lib/MO2Compiler.cs | 6 ++-- 6 files changed, 64 insertions(+), 9 deletions(-) diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 6667f101..4345c17b 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -101,12 +101,18 @@ namespace Wabbajack.Lib return id; } - internal async Task IncludeFile(AbsolutePath data) + internal async Task IncludeFile(Stream data) { var id = IncludeId(); - await data.CopyToAsync(ModListOutputFolder.Combine(id)); + await ModListOutputFolder.Combine(id).WriteAllAsync(data); return id; } + + internal async Task IncludeFile(AbsolutePath data) + { + await using var stream = await data.OpenRead(); + return await IncludeFile(stream); + } internal async Task<(RelativePath, AbsolutePath)> IncludeString(string str) @@ -302,6 +308,27 @@ namespace Wabbajack.Lib } } + protected async Task InlineFiles() + { + var grouped = ModList.Directives.OfType() + .Where(f => f.SourceDataID == default) + .GroupBy(f => f.SourceDataFile) + .ToDictionary(f => f.Key); + + await VFS.Extract(Queue, grouped.Keys.ToHashSet(), async (vf, sfn) => + { + await using var stream = await sfn.GetStream(); + var id = await IncludeFile(stream); + foreach (var file in grouped[vf]) + { + file.SourceDataID = id; + file.SourceDataFile = null; + } + + }); + } + + public bool CheckForNoMatchExit(ICollection noMatches) { if (noMatches.Count > 0) diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index ed41a416..be65cf8e 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -87,7 +87,11 @@ namespace Wabbajack.Lib public async Task LoadBytesFromPath(RelativePath path) { - return await ExtractedModlistFolder!.Dir.Combine(path).ReadAllBytesAsync(); + var fullPath = ExtractedModlistFolder!.Dir.Combine(path); + if (!fullPath.IsFile) + throw new Exception($"Cannot load inlined data {path} file does not exist"); + + return await fullPath.ReadAllBytesAsync(); } public static ModList LoadFromFile(AbsolutePath path) @@ -140,7 +144,27 @@ namespace Wabbajack.Lib foreach (var directive in grouped[vf]) { s.Position = 0; - await directive.Directive.To.RelativeTo(OutputFolder).WriteAllAsync(s, false); + + switch (directive.Directive) + { + case PatchedFromArchive pfa: + { + var patchData = await LoadBytesFromPath(pfa.PatchID); + await using var os = await directive.Directive.To.RelativeTo(OutputFolder).Create(); + Utils.ApplyPatch(s, () => new MemoryStream(patchData), os); + } + break; + + + + case FromArchive _: + await directive.Directive.To.RelativeTo(OutputFolder).WriteAllAsync(s, false); + break; + default: + throw new Exception($"No handler for {directive}"); + + + } } }); } diff --git a/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs b/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs index ee6600ae..e873c4cf 100644 --- a/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs +++ b/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs @@ -76,7 +76,7 @@ namespace Wabbajack.Lib.CompilationSteps Func? _cleanup = null; if (defaultInclude) { - _cleanup = await source.File.Context.Stage(source.File.Children); + //_cleanup = await source.File.Context.Stage(source.File.Children); } var matches = await sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e, Consts.BSACreationDir.Combine((RelativePath)id, (RelativePath)e.Name)))); diff --git a/Wabbajack.Lib/CompilationSteps/IncludeAll.cs b/Wabbajack.Lib/CompilationSteps/IncludeAll.cs index 3b4c8b5b..8138181d 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeAll.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeAll.cs @@ -14,8 +14,7 @@ namespace Wabbajack.Lib.CompilationSteps public override async ValueTask Run(RawSourceFile source) { var inline = source.EvolveTo(); - await using var file = await source.File.StagedFile.OpenRead(); - inline.SourceDataID = await _compiler.IncludeFile(await file.ReadAllAsync()); + inline.SourceDataFile = source.File; return inline; } } diff --git a/Wabbajack.Lib/Data.cs b/Wabbajack.Lib/Data.cs index c5552621..8da342e3 100644 --- a/Wabbajack.Lib/Data.cs +++ b/Wabbajack.Lib/Data.cs @@ -161,6 +161,9 @@ namespace Wabbajack.Lib /// Data that will be written as-is to the destination location; /// public RelativePath SourceDataID { get; set; } + + [JsonIgnore] + public VirtualFile? SourceDataFile { get; set; } } [JsonName("ArchiveMeta")] diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 9c980fc6..7b935812 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -57,7 +57,7 @@ namespace Wabbajack.Lib public HashSet SelectedProfiles { get; set; } = new HashSet(); public MO2Compiler(AbsolutePath mo2Folder, string mo2Profile, AbsolutePath outputFile) - : base(steps: 20) + : base(steps: 21) { MO2Folder = mo2Folder; MO2Profile = mo2Profile; @@ -352,6 +352,9 @@ namespace Wabbajack.Lib Version = ModlistVersion ?? new Version(1,0,0,0), IsNSFW = ModlistIsNSFW }; + + UpdateTracker.NextStep("Including required files"); + await InlineFiles(); UpdateTracker.NextStep("Running Validation"); @@ -370,7 +373,6 @@ namespace Wabbajack.Lib return true; } - public Dictionary> GameHashes { get; set; } = new Dictionary>(); public Dictionary GamesWithHashes { get; set; } = new Dictionary(); From 3bdab577e1f5c4f49739089a6a04d0fb73d6d52d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 5 Sep 2020 21:19:05 -0600 Subject: [PATCH 06/13] EndToEnd Tests pass --- Wabbajack.Common/StatusFileStream.cs | 2 +- Wabbajack.Common/Wabbajack.Common.csproj | 1 + Wabbajack.Lib/ACompiler.cs | 1 + Wabbajack.Test/EndToEndTests.cs | 8 ++-- .../FileExtractor2/FileExtractor.cs | 46 ++++++++++++++++++- .../FileExtractor2/GatheringExtractor.cs | 41 ++++++++++++++--- .../SevenZipExtractor/Formats.cs | 4 +- Wabbajack.VirtualFileSystem/VirtualFile.cs | 4 +- .../Wabbajack.VirtualFileSystem.csproj | 1 + 9 files changed, 91 insertions(+), 17 deletions(-) diff --git a/Wabbajack.Common/StatusFileStream.cs b/Wabbajack.Common/StatusFileStream.cs index 5638f053..e7f7cac3 100644 --- a/Wabbajack.Common/StatusFileStream.cs +++ b/Wabbajack.Common/StatusFileStream.cs @@ -17,7 +17,7 @@ namespace Wabbajack.Common _inner = fs; _message = message; _lastUpdate = DateTime.UnixEpoch; - _span = TimeSpan.FromMilliseconds(500); + _span = TimeSpan.FromMilliseconds(100); } public override void Flush() diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index e88e48b2..1f3a93c7 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -18,6 +18,7 @@ 2.2.1.2 2.2.1.2 2.2.1.2 + true Wabbajack.Common.xml diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 4345c17b..d1166171 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -315,6 +315,7 @@ namespace Wabbajack.Lib .GroupBy(f => f.SourceDataFile) .ToDictionary(f => f.Key); + if (grouped.Count == 0) return; await VFS.Extract(Queue, grouped.Keys.ToHashSet(), async (vf, sfn) => { await using var stream = await sfn.GetStream(); diff --git a/Wabbajack.Test/EndToEndTests.cs b/Wabbajack.Test/EndToEndTests.cs index 8e60880e..2fa7d9c1 100644 --- a/Wabbajack.Test/EndToEndTests.cs +++ b/Wabbajack.Test/EndToEndTests.cs @@ -110,9 +110,8 @@ namespace Wabbajack.Test var destFile = utils.DownloadsFolder.Combine(filename); await src.CopyToAsync(destFile); - await using var dest = await FileExtractor.ExtractAll(Queue, src); var modFolder = modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName); - await dest.MoveAllTo(modFolder); + await FileExtractor2.ExtractAll(src, modFolder); return (destFile, modFolder); } @@ -147,9 +146,8 @@ namespace Wabbajack.Test await src.CopyToAsync(dest); var modFolder = utils.ModsFolder.Combine(modName); - await using var files = await FileExtractor.ExtractAll(Queue, src); - await files.MoveAllTo(modFolder); - + await FileExtractor2.ExtractAll(src, modFolder); + await dest.WithExtension(Consts.MetaFileExtension).WriteAllTextAsync(ini); return (dest, modFolder); } diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs index eaf965bf..ab66907f 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs @@ -3,6 +3,9 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; using Compression.BSA; +using ICSharpCode.SharpZipLib.Zip.Compression.Streams; +using SharpCompress.Archives.SevenZip; +using SharpCompress.Readers; using Wabbajack.Common; using Wabbajack.Common.FileSignatures; using Wabbajack.VirtualFileSystem.SevenZipExtractor; @@ -16,7 +19,8 @@ namespace Wabbajack.VirtualFileSystem Definitions.FileType.BA2, Definitions.FileType.ZIP, //Definitions.FileType.EXE, - Definitions.FileType.RAR, + Definitions.FileType.RAR_OLD, + Definitions.FileType.RAR_NEW, Definitions.FileType._7Z); @@ -29,6 +33,9 @@ namespace Wabbajack.VirtualFileSystem switch (sig) { + case Definitions.FileType.RAR_OLD: + case Definitions.FileType.RAR_NEW: + case Definitions.FileType._7Z: case Definitions.FileType.ZIP: return await GatheringExtractWith7Zip(archive, (Definitions.FileType)sig, shouldExtract, mapfn); @@ -62,6 +69,43 @@ namespace Wabbajack.VirtualFileSystem private static async Task> GatheringExtractWith7Zip(Stream stream, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) { return await new GatheringExtractor(stream, sig, shouldExtract, mapfn).Extract(); + /* + IReader reader; + if (sig == Definitions.FileType._7Z) + reader = SevenZipArchive.Open(stream).ExtractAllEntries(); + else + { + reader = ReaderFactory.Open(stream); + } + + var results = new Dictionary(); + while (reader.MoveToNextEntry()) + { + var path = (RelativePath)reader.Entry.Key; + if (!reader.Entry.IsDirectory && shouldExtract(path)) + { + var ms = new MemoryStream(); + reader.WriteEntryTo(ms); + ms.Position = 0; + var result = await mapfn(path, new MemoryStreamFactory(ms)); + results.Add(path, result); + } + } + + return results; + */ + } + + public static async Task ExtractAll(AbsolutePath src, AbsolutePath dest) + { + await GatheringExtract(new NativeFileStreamFactory(src), _ => true, async (path, factory) => + { + var abs = path.RelativeTo(dest); + abs.Parent.CreateDirectory(); + await using var stream = await factory.GetStream(); + await abs.WriteAllAsync(stream); + return 0; + }); } } } diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs index ba193e04..154ea053 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; +using Compression.BSA; using Wabbajack.Common; using Wabbajack.Common.FileSignatures; using Wabbajack.VirtualFileSystem.SevenZipExtractor; @@ -17,7 +18,7 @@ namespace Wabbajack.VirtualFileSystem private Predicate _shouldExtract; private Func> _mapFn; private Dictionary _results; - private Dictionary _indexes; + private Dictionary _indexes; private Stream _stream; private Definitions.FileType _sig; @@ -41,10 +42,11 @@ namespace Wabbajack.VirtualFileSystem { _archive = ArchiveFile.Open(_stream, _sig).Result; _indexes = _archive.Entries - .Where(f => !f.IsFolder) - .Select((entry, idx) => ((RelativePath)entry.FileName, (uint)idx)) + .Select((entry, idx) => (entry, (uint)idx)) + .Where(f => !f.entry.IsFolder) + .Select(t => ((RelativePath)t.entry.FileName, t.Item2, t.entry.Size)) .Where(t => _shouldExtract(t.Item1)) - .ToDictionary(t => t.Item2, t => t.Item1); + .ToDictionary(t => t.Item2, t => (t.Item1, t.Size)); _archive._archive.Extract(null, 0xFFFFFFFF, 0, this); @@ -101,27 +103,52 @@ namespace Wabbajack.VirtualFileSystem { private GatheringExtractor _extractor; private uint _index; + private bool _written; + private ulong _totalSize; + private MemoryStream _tmpStream; public GatheringExtractorStream(GatheringExtractor extractor, uint index) { _extractor = extractor; _index = index; + _written = false; + _totalSize = extractor._indexes[index].Item2; + _tmpStream = new MemoryStream(); } public int Write(IntPtr data, uint size, IntPtr processedSize) { unsafe { - var result = _extractor._mapFn(_extractor._indexes[_index], new UnmanagedStreamFactory((byte*)data, size)).AsTask().Result; + var ums = new UnmanagedMemoryStream((byte*)data, size); + ums.CopyTo(_tmpStream); + if ((ulong)_tmpStream.Length >= _totalSize) + { + _tmpStream.Position = 0; + var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryStreamFactory(_tmpStream)).AsTask().Result; - _extractor._results[_extractor._indexes[_index]] = result; - + _extractor._results[_extractor._indexes[_index].Item1] = result; + } + if (processedSize != IntPtr.Zero) { Marshal.WriteInt32(processedSize, (int) size); } + } + + return 0; + + if (_written) throw new Exception("TODO"); + unsafe + { + + + + + _written = true; return 0; + } } diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs index a44d180a..64c36920 100644 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs @@ -91,8 +91,8 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor { {Definitions.FileType._7Z, new Guid("23170f69-40c1-278a-1000-000110070000")}, {Definitions.FileType.BZ2, new Guid("23170f69-40c1-278a-1000-000110020000")}, - {Definitions.FileType.RAR_NEW, new Guid("23170f69-40c1-278a-1000-000110030000")}, - {Definitions.FileType.RAR_OLD, new Guid("23170f69-40c1-278a-1000-000110CC0000")}, + {Definitions.FileType.RAR_OLD, new Guid("23170f69-40c1-278a-1000-000110030000")}, + {Definitions.FileType.RAR_NEW, new Guid("23170f69-40c1-278a-1000-000110CC0000")}, {Definitions.FileType.ZIP, new Guid("23170f69-40c1-278a-1000-000110010000")}, }; diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index e6d30bac..f874821f 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Net.Http; using System.Threading.Tasks; +using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using K4os.Hash.Crc; using RocksDbSharp; using Wabbajack.Common; @@ -198,8 +199,9 @@ namespace Wabbajack.VirtualFileSystem stream.Position = 0; var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream); + stream.Position = 0; - if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself)) + if (sig.HasValue && TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself)) return vself; var self = new VirtualFile diff --git a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj index d592ed0f..3674045e 100644 --- a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj +++ b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj @@ -17,6 +17,7 @@ + From 9de30ea8b7df4fd92fd3064ffa7ff9837a52ed40 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 5 Sep 2020 21:28:31 -0600 Subject: [PATCH 07/13] Removed cruft of old extractor --- Compression.BSA.Test/BSATests.cs | 3 +- Wabbajack.Lib/AInstaller.cs | 119 +------- Wabbajack.VirtualFileSystem/Context.cs | 37 --- .../ExtractedBSAFile.cs | 51 ---- .../ExtractedDiskFile.cs | 53 ---- Wabbajack.VirtualFileSystem/ExtractedFiles.cs | 64 ---- Wabbajack.VirtualFileSystem/FileExtractor.cs | 284 ----------------- Wabbajack.VirtualFileSystem/IExtractedFile.cs | 24 -- Wabbajack.VirtualFileSystem/RootDiskFile.cs | 20 -- .../SevenZipExtractor/ArchiveFile.cs | 87 +----- .../SevenZipExtractor/ArchiveFileCallback.cs | 57 ---- .../ArchiveStreamCallback.cs | 45 --- .../ArchiveStreamsCallback.cs | 58 ---- .../SevenZipExtractor/Entry.cs | 30 -- .../SevenZipExtractor/Formats.cs | 104 +------ .../SevenZipExtractor/SevenZipException.cs | 24 -- .../SevenZipExtractor/SevenZipFormat.cs | 285 ------------------ .../SevenZipExtractor/TypedExtractor.cs | 64 ---- Wabbajack.VirtualFileSystem/VirtualFile.cs | 22 -- 19 files changed, 17 insertions(+), 1414 deletions(-) delete mode 100644 Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs delete mode 100644 Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs delete mode 100644 Wabbajack.VirtualFileSystem/ExtractedFiles.cs delete mode 100644 Wabbajack.VirtualFileSystem/FileExtractor.cs delete mode 100644 Wabbajack.VirtualFileSystem/IExtractedFile.cs delete mode 100644 Wabbajack.VirtualFileSystem/RootDiskFile.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFileCallback.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamCallback.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamsCallback.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipException.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipFormat.cs delete mode 100644 Wabbajack.VirtualFileSystem/SevenZipExtractor/TypedExtractor.cs diff --git a/Compression.BSA.Test/BSATests.cs b/Compression.BSA.Test/BSATests.cs index c47ed51d..73a16434 100644 --- a/Compression.BSA.Test/BSATests.cs +++ b/Compression.BSA.Test/BSATests.cs @@ -83,8 +83,7 @@ namespace Compression.BSA.Test var folder = _bsaFolder.Combine(game.ToString(), modid.ToString()); await folder.DeleteDirectory(); folder.CreateDirectory(); - await using var files = await FileExtractor.ExtractAll(Queue, filename); - await files.MoveAllTo(folder); + await FileExtractor2.ExtractAll(filename, folder); foreach (var bsa in folder.EnumerateFiles().Where(f => Consts.SupportedBSAs.Contains(f.Extension))) { diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index be65cf8e..c76db728 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -51,8 +51,6 @@ namespace Wabbajack.Lib } - - private ExtractedFiles? ExtractedModListFiles { get; set; } = null; public async Task ExtractModlist() { ExtractedModlistFolder = await TempFolder.Create(); @@ -143,20 +141,27 @@ namespace Wabbajack.Lib await using var s = await sf.GetStream(); foreach (var directive in grouped[vf]) { + var file = directive.Directive; s.Position = 0; - switch (directive.Directive) + switch (file) { case PatchedFromArchive pfa: { var patchData = await LoadBytesFromPath(pfa.PatchID); - await using var os = await directive.Directive.To.RelativeTo(OutputFolder).Create(); + var toFile = file.To.RelativeTo(OutputFolder); + await using var os = await toFile.Create(); Utils.ApplyPatch(s, () => new MemoryStream(patchData), os); + + if (await VirusScanner.ShouldScan(toFile) && + await ClientAPI.GetVirusScanResult(toFile) == VirusScanner.Result.Malware) + { + await toFile.DeleteAsync(); + Utils.ErrorThrow(new Exception($"Virus scan of patched executable reported possible malware: {toFile.ToString()} ({(long)await toFile.FileHashCachedAsync()})")); + } } break; - - case FromArchive _: await directive.Directive.To.RelativeTo(OutputFolder).WriteAllAsync(s, false); break; @@ -165,102 +170,7 @@ namespace Wabbajack.Lib } - } - }); - } - - private async Task InstallArchive(WorkQueue queue, Archive archive, AbsolutePath absolutePath, IGrouping grouping) - { - Status($"Extracting {archive.Name}"); - - List vFiles = grouping.Select(g => - { - var file = VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath); - g.FromFile = file; - return g; - }).ToList(); - - var onFinish = await VFS.Stage(vFiles.Select(f => f.FromFile).Distinct()); - - - Status($"Copying files for {archive.Name}"); - - async ValueTask CopyFile(AbsolutePath from, AbsolutePath to) - { - if (to.Exists) - { - if (to.IsReadOnly) - to.IsReadOnly = false; - await to.DeleteAsync(); - } - - if (from.Exists) - { - if (from.IsReadOnly) - from.IsReadOnly = false; - } - - await @from.CopyToAsync(to); - // If we don't do this, the file will use the last-modified date of the file when it was compressed - // into an archive, which isn't really what we want in the case of files installed archives - to.LastModified = DateTime.Now; - } - - foreach (var (idx, group) in vFiles.GroupBy(f => f.FromFile).Select((grp, i) => (i, grp))) - { - Utils.Status("Installing files", Percent.FactoryPutInRange(idx, vFiles.Count)); - if (group.Key == null) - { - throw new ArgumentNullException("FromFile was null"); - } - var firstDest = OutputFolder.Combine(group.First().To); - - if (group.Key.IsNative) - { - await group.Key.AbsoluteName.HardLinkIfOversize(firstDest); - } - else - { - await group.Key.StagedFile.MoveTo(firstDest); - } - - foreach (var copy in group.Skip(1)) - { - await CopyFile(firstDest, OutputFolder.Combine(copy.To)); - } - - foreach (var toPatch in group.OfType()) - { - await using var patchStream = new MemoryStream(); - Status($"Patching {toPatch.To.FileName}"); - // Read in the patch data - Status($"Verifying unpatched file {toPatch.To.FileName}"); - var toFile = OutputFolder.Combine(toPatch.To); - - byte[] patchData = await LoadBytesFromPath(toPatch.PatchID); - - var oldData = new MemoryStream(await toFile.ReadAllBytesAsync()); - - // Remove the file we're about to patch - await toFile.DeleteAsync(); - - // Patch it - await using (var outStream = await toFile.Create()) - { - Utils.ApplyPatch(oldData, () => new MemoryStream(patchData), outStream); - } - - if (await VirusScanner.ShouldScan(toFile) && - await ClientAPI.GetVirusScanResult(toFile) == VirusScanner.Result.Malware) - { - await toFile.DeleteAsync(); - Utils.ErrorThrow(new Exception($"Virus scan of patched executable reported possible malware: {toFile.ToString()} ({(long)await toFile.FileHashCachedAsync()})")); - } - } - - foreach (var file in group) - { if (file is PatchedFromArchive) { await file.To.RelativeTo(OutputFolder).FileHashAsync(); @@ -276,12 +186,7 @@ namespace Wabbajack.Lib await file.To.RelativeTo(OutputFolder).Compact(FileCompaction.Algorithm.XPRESS16K); } } - - - } - - Status("Unstaging files"); - await onFinish(); + }); } public async Task DownloadArchives() diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index 9b6ba5cb..e1609de5 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -230,43 +230,6 @@ namespace Wabbajack.VirtualFileSystem await filesByParent[top].PMap(queue, async file => await HandleFile(file, new NativeFileStreamFactory(file.AbsoluteName))); } - public async Task> Stage(IEnumerable files) - { - await _cleanupTask; - - var grouped = files.SelectMany(f => f.FilesInFullPath) - .Distinct() - .Where(f => f.Parent != null) - .GroupBy(f => f.Parent) - .OrderBy(f => f.Key?.NestingFactor ?? 0) - .ToList(); - - var paths = new List(); - - foreach (var group in grouped) - { - var only = group.Select(f => f.RelativeName); - var extracted = await group.Key.StagedFile.ExtractAll(Queue, only, true); - paths.Add(extracted); - foreach (var file in group) - file.StagedFile = extracted[file.RelativeName]; - } - - return async () => - { - foreach (var p in paths) - { - await p.DisposeAsync(); - } - }; - } - - public async Task> StageWith(IEnumerable files) - { - return new AsyncDisposableList(await Stage(files), files); - } - - #region KnownFiles private List _knownFiles = new List(); diff --git a/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs b/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs deleted file mode 100644 index 8dd37a06..00000000 --- a/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading.Tasks; -using Compression.BSA; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem -{ - public class ExtractedBSAFile : IExtractedFile - { - private readonly IFile _file; - public ExtractedBSAFile(IFile file) - { - _file = file; - } - - public RelativePath Path => _file.Path; - - public async Task HashAsync() - { - await using var stream = await OpenRead(); - return await stream.xxHashAsync(); - } - public DateTime LastModifiedUtc => DateTime.UtcNow; - public long Size => _file.Size; - public async ValueTask OpenRead() - { - var ms = new MemoryStream(); - await _file.CopyDataTo(ms); - ms.Position = 0; - return ms; - } - - public async Task CanExtract() - { - return false; - } - - public Task ExtractAll(WorkQueue queue, IEnumerable OnlyFiles, bool throwOnError) - { - throw new Exception("BSAs can't contain archives"); - } - - public async Task MoveTo(AbsolutePath path) - { - await using var fs = await path.Create(); - await _file.CopyDataTo(fs); - } - } -} diff --git a/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs b/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs deleted file mode 100644 index aa440980..00000000 --- a/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs +++ /dev/null @@ -1,53 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem -{ - public class ExtractedDiskFile : IExtractedFile - { - protected AbsolutePath _path; - - public ExtractedDiskFile(AbsolutePath path) - { - if (path == default) - throw new InvalidDataException("Path cannot be empty"); - _path = path; - } - - public virtual async Task HashAsync() - { - return await _path.FileHashAsync(); - } - public DateTime LastModifiedUtc => _path.LastModifiedUtc; - public long Size => _path.Size; - public async ValueTask OpenRead() - { - return await _path.OpenRead(); - } - - public async Task CanExtract() - { - return await FileExtractor.CanExtract(_path); - } - - public Task ExtractAll(WorkQueue queue, IEnumerable onlyFiles, bool throwOnError) - { - return FileExtractor.ExtractAll(queue, _path, onlyFiles, throwOnError); - } - - public async Task MoveTo(AbsolutePath path) - { - if (FileExtractor.MightBeArchive(_path.Extension)) - { - path.Parent.CreateDirectory(); - await _path.CopyToAsync(path); - return; - } - await _path.MoveToAsync(path, true); - _path = path; - } - } -} diff --git a/Wabbajack.VirtualFileSystem/ExtractedFiles.cs b/Wabbajack.VirtualFileSystem/ExtractedFiles.cs deleted file mode 100644 index 81d68c41..00000000 --- a/Wabbajack.VirtualFileSystem/ExtractedFiles.cs +++ /dev/null @@ -1,64 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem -{ - public class ExtractedFiles : IAsyncDisposable, IEnumerable> - { - private Dictionary _files; - private IAsyncDisposable _disposable; - private AbsolutePath _tempFolder; - - public ExtractedFiles(Dictionary files, IAsyncDisposable disposeOther = null) - { - _files = files; - _disposable = disposeOther; - } - - public ExtractedFiles(TempFolder tempPath) - { - _files = tempPath.Dir.EnumerateFiles().ToDictionary(f => f.RelativeTo(tempPath.Dir), - f => (IExtractedFile)new ExtractedDiskFile(f)); - _disposable = tempPath; - } - - public async ValueTask DisposeAsync() - { - if (_disposable != null) - { - await _disposable.DisposeAsync(); - _disposable = null; - } - } - - public bool ContainsKey(RelativePath key) - { - return _files.ContainsKey(key); - } - - public int Count => _files.Count; - - public IExtractedFile this[RelativePath key] => _files[key]; - public IEnumerator> GetEnumerator() - { - return _files.GetEnumerator(); - } - - public async Task MoveAllTo(AbsolutePath folder) - { - foreach (var (key, value) in this) - { - await value.MoveTo(key.RelativeTo(folder)); - } - } - - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - } -} diff --git a/Wabbajack.VirtualFileSystem/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor.cs deleted file mode 100644 index f736dbea..00000000 --- a/Wabbajack.VirtualFileSystem/FileExtractor.cs +++ /dev/null @@ -1,284 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Reactive.Linq; -using System.Threading.Tasks; -using Alphaleonis.Win32.Filesystem; -using Compression.BSA; -using OMODFramework; -using Wabbajack.Common.StatusFeed; -using Wabbajack.Common.StatusFeed.Errors; -using Wabbajack.Common; -using Wabbajack.Common.FileSignatures; -using Wabbajack.VirtualFileSystem.SevenZipExtractor; -using Utils = Wabbajack.Common.Utils; - - -namespace Wabbajack.VirtualFileSystem -{ - public class FileExtractor - { - public static readonly SignatureChecker ArchiveSigs = new SignatureChecker(Definitions.FileType.TES3, - Definitions.FileType.BSA, - Definitions.FileType.BA2, - Definitions.FileType.ZIP, - Definitions.FileType.EXE, - Definitions.FileType.RAR, - Definitions.FileType._7Z); - - public static async Task ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable OnlyFiles = null, bool throwOnError = true) - { - throw new NotImplementedException(); - OnlyFiles ??= new RelativePath[0]; - - try - { - var sig = await ArchiveSigs.MatchesAsync(source); - - if (source.Extension == Consts.OMOD) - return await ExtractAllWithOMOD(source); - - switch (sig) - { - case Definitions.FileType.BSA: - case Definitions.FileType.TES3: - case Definitions.FileType.BA2: - return await ExtractAllWithBSA(queue, source); - case Definitions.FileType.EXE: - return await ExtractAllExe(source); - case Definitions.FileType._7Z: - case Definitions.FileType.ZIP: - case Definitions.FileType.RAR: - return await ExtractAllWith7Zip(source, OnlyFiles); - } - throw new Exception("Invalid archive format"); - } - catch (Exception ex) - { - if (!throwOnError) - return new ExtractedFiles(await TempFolder.Create()); - - Utils.Log(ex.ToString()); - Utils.ErrorThrow(ex, $"Error while extracting {source}"); - throw new Exception(); - } - } - - private static async Task ExtractAllExe(AbsolutePath source) - { - var isArchive = await TestWith7z(source); - - if (isArchive) - { - return await ExtractAllWith7Zip(source, (IEnumerable) null); - } - - var dest = await TempFolder.Create(); - Utils.Log($"Extracting {(string)source.FileName}"); - - var process = new ProcessHelper - { - Path = @"Extractors\innounp.exe".RelativeTo(AbsolutePath.EntryPoint), - Arguments = new object[] {"-x", "-y", "-b", $"-d\"{dest.Dir}\"", source} - }; - - - var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output) - .ForEachAsync(p => - { - var (_, line) = p; - if (line == null) - return; - - if (line.Length <= 4 || line[3] != '%') - return; - - int.TryParse(line.Substring(0, 3), out var percentInt); - Utils.Status($"Extracting {source.FileName} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d)); - }); - await process.Start(); - return new ExtractedFiles(dest); - } - - private class OMODProgress : ICodeProgress - { - private long _total; - - public void SetProgress(long inSize, long outSize) - { - Utils.Status("Extracting OMOD", Percent.FactoryPutInRange(inSize, _total)); - } - - public void Init(long totalSize, bool compressing) - { - _total = totalSize; - } - - public void Dispose() - { - // - } - } - - private static async Task ExtractAllWithOMOD(AbsolutePath source) - { - var dest = await TempFolder.Create(); - Utils.Log($"Extracting {(string)source.FileName}"); - - Framework.Settings.TempPath = (string)dest.Dir; - Framework.Settings.CodeProgress = new OMODProgress(); - - var omod = new OMOD((string)source); - omod.GetDataFiles(); - omod.GetPlugins(); - - return new ExtractedFiles(dest); - } - - - private static async Task ExtractAllWithBSA(WorkQueue queue, AbsolutePath source) - { - try - { - var arch = await BSADispatch.OpenRead(source); - var files = arch.Files.ToDictionary(f => f.Path, f => (IExtractedFile)new ExtractedBSAFile(f)); - return new ExtractedFiles(files); - } - catch (Exception ex) - { - Utils.ErrorThrow(ex, $"While Extracting {source}"); - throw new Exception(); - } - } - - private static async Task ExtractAllWith7Zip(AbsolutePath source, IEnumerable onlyFiles) - { - var dest = await TempFolder.Create(); - Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}", $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe")); - - var files = onlyFiles.ToHashSet(); - - using var archive = await ArchiveFile.Open(source); - if (files.Count > 0) - { - await archive.Extract(path => - { - Utils.Log($"Extract file {path} {files.Contains(path)} {dest.Dir.Combine(path)}"); - return files.Contains(path) ? dest.Dir.Combine(path) : default; - }); - } - else - { - await archive.Extract(path => dest.Dir.Combine(path)); - } - - return new ExtractedFiles(dest); - } - - /// - /// Returns true if the given extension type can be extracted - /// - /// - /// - public static async Task CanExtract(AbsolutePath v) - { - var found = await ArchiveSigs.MatchesAsync(v); - switch (found) - { - case null: - return false; - case Definitions.FileType.EXE: - { - var process = new ProcessHelper - { - Path = @"Extractors\innounp.exe".RelativeTo(AbsolutePath.EntryPoint), - Arguments = new object[] {"-t", v}, - }; - - return await process.Start() == 0; - } - default: - return true; - } - } - - public static async Task TestWith7z(AbsolutePath file) - { - var process = new ProcessHelper() - { - Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint), - Arguments = new object[] {"t", file}, - }; - - return await process.Start() == 0; - } - - private static Extension _exeExtension = new Extension(".exe"); - - public static bool MightBeArchive(Extension ext) - { - return ext == _exeExtension || Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext); - } - - /// - /// Extract the specific files to the specific locations - /// - /// - /// - /// - /// - public static async Task ExtractTo(WorkQueue queue, AbsolutePath source, Dictionary indexed) - { - try - { - var sig = await ArchiveSigs.MatchesAsync(source); - - /*if (source.Extension == Consts.OMOD) - return await ExtractAllWithOMOD(source);*/ - - switch (sig) - { - case Definitions.FileType.BSA: - case Definitions.FileType.TES3: - case Definitions.FileType.BA2: - await ExtractAllWithBSA(queue, source, indexed); - return; - case Definitions.FileType.EXE: - await ExtractAllExe(source, indexed); - return; - case Definitions.FileType._7Z: - case Definitions.FileType.ZIP: - case Definitions.FileType.RAR: - await ExtractAllWith7Zip(source, indexed); - return; - } - throw new Exception("Invalid archive format"); - } - catch (Exception ex) - { - Utils.Log(ex.ToString()); - Utils.ErrorThrow(ex, $"Error while extracting {source}"); - throw new Exception(); - } - } - - private static async Task ExtractAllWith7Zip(AbsolutePath source, Dictionary onlyFiles) - { - using var archive = await ArchiveFile.Open(source); - await archive.Extract(f => onlyFiles.TryGetValue(f, out var dest) ? dest : default); - } - - private static async Task ExtractAllExe(AbsolutePath source, Dictionary indexed) - { - throw new NotImplementedException(); - } - - private static async Task ExtractAllWithBSA(WorkQueue queue, AbsolutePath source, Dictionary indexed) - { - throw new NotImplementedException(); - - } - } -} diff --git a/Wabbajack.VirtualFileSystem/IExtractedFile.cs b/Wabbajack.VirtualFileSystem/IExtractedFile.cs deleted file mode 100644 index 27fa9678..00000000 --- a/Wabbajack.VirtualFileSystem/IExtractedFile.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem -{ - public interface IExtractedFile - { - public Task HashAsync(); - public DateTime LastModifiedUtc { get; } - public long Size { get; } - - public ValueTask OpenRead(); - - public Task CanExtract(); - - public Task ExtractAll(WorkQueue queue, IEnumerable Only = null, bool throwOnError = false); - - public Task MoveTo(AbsolutePath path); - - } -} diff --git a/Wabbajack.VirtualFileSystem/RootDiskFile.cs b/Wabbajack.VirtualFileSystem/RootDiskFile.cs deleted file mode 100644 index 6040cc48..00000000 --- a/Wabbajack.VirtualFileSystem/RootDiskFile.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading.Tasks; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem -{ - public class RootDiskFile : ExtractedDiskFile - { - public RootDiskFile(AbsolutePath path) : base(path) - { - } - - public override async Task HashAsync() - { - return await _path.FileHashCachedAsync(); - } - } -} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs index c16810c8..f2710986 100644 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs @@ -18,30 +18,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor private static readonly AbsolutePath LibraryFilePath = @"Extractors\7z.dll".RelativeTo(AbsolutePath.EntryPoint); private static SignatureChecker _checker = new SignatureChecker(Formats.FileTypeGuidMapping.Keys.ToArray()); - - public static async Task Open(AbsolutePath archiveFilePath) - { - var self = new ArchiveFile(); - - self.InitializeAndValidateLibrary(); - - if (!archiveFilePath.IsFile) - { - throw new SevenZipException("Archive file not found"); - } - - var format = await _checker.MatchesAsync(archiveFilePath); - - if (format == null) - { - throw new SevenZipException($"Unknown format for {archiveFilePath}"); - } - - self._archive = self._sevenZipHandle.CreateInArchive(Formats.FileTypeGuidMapping[format.Value]); - self._archiveStream = new InStreamWrapper(await archiveFilePath.OpenRead()); - return self; - } - + public static async Task Open(Stream archiveStream, Definitions.FileType format) { var self = new ArchiveFile(); @@ -51,64 +28,6 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor return self; } - public async Task Extract(AbsolutePath outputFolder, bool overwrite = false) - { - await this.Extract(entry => - { - var fileName = outputFolder.Combine(entry.FileName); - - if (!fileName.Exists || overwrite) - { - return fileName; - } - - return default; - }); - } - - public async Task Extract(Func getOutputPath) - { - IList fileStreams = new List(); - - try - { - foreach (Entry entry in Entries) - { - - AbsolutePath outputPath = entry.IsFolder ? default : getOutputPath((RelativePath)entry.FileName); - - if (outputPath == default) // getOutputPath = null means SKIP - { - fileStreams.Add(null); - continue; - } - - if (entry.IsFolder) - { - outputPath.CreateDirectory(); - fileStreams.Add(null); - continue; - } - - var directoryName = outputPath.Parent; - directoryName.CreateDirectory(); - - fileStreams.Add(await outputPath.Create()); - } - - this._archive.Extract(null, 0xFFFFFFFF, 0, new ArchiveStreamsCallback(fileStreams)); - } - finally - { - foreach (Stream stream in fileStreams) - { - if (stream == null) continue; - var tsk = stream?.DisposeAsync(); - await tsk.Value; - } - } - } - public IList Entries { get @@ -123,7 +42,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor if (open != 0) { - throw new SevenZipException("Unable to open archive"); + throw new Exception("Unable to open archive"); } uint itemsCount = this._archive.GetNumberOfItems(); @@ -221,7 +140,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor } catch (Exception e) { - throw new SevenZipException("Unable to initialize SevenZipHandle", e); + throw new Exception("Unable to initialize SevenZipHandle", e); } } diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFileCallback.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFileCallback.cs deleted file mode 100644 index 18ae341e..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFileCallback.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System; -using System.IO; - -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - internal class ArchiveFileCallback : IArchiveExtractCallback - { - private readonly string fileName; - private readonly uint fileNumber; - private OutStreamWrapper fileStream; // to be removed - - public ArchiveFileCallback(uint fileNumber, string fileName) - { - this.fileNumber = fileNumber; - this.fileName = fileName; - } - - public void SetTotal(ulong total) - { - } - - public void SetCompleted(ref ulong completeValue) - { - } - - public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode) - { - if ((index != this.fileNumber) || (askExtractMode != AskMode.kExtract)) - { - outStream = null; - return 0; - } - - string fileDir = Path.GetDirectoryName(this.fileName); - - if (!string.IsNullOrEmpty(fileDir)) - { - Directory.CreateDirectory(fileDir); - } - - this.fileStream = new OutStreamWrapper(File.Create(this.fileName)); - - outStream = this.fileStream; - - return 0; - } - - public void PrepareOperation(AskMode askExtractMode) - { - } - - public void SetOperationResult(OperationResult resultEOperationResult) - { - this.fileStream.Dispose(); - } - } -} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamCallback.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamCallback.cs deleted file mode 100644 index ecba71e3..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamCallback.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System.IO; - -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - internal class ArchiveStreamCallback : IArchiveExtractCallback - { - private readonly uint fileNumber; - private readonly Stream stream; - - public ArchiveStreamCallback(uint fileNumber, Stream stream) - { - this.fileNumber = fileNumber; - this.stream = stream; - } - - public void SetTotal(ulong total) - { - } - - public void SetCompleted(ref ulong completeValue) - { - } - - public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode) - { - if ((index != this.fileNumber) || (askExtractMode != AskMode.kExtract)) - { - outStream = null; - return 0; - } - - outStream = new OutStreamWrapper(this.stream); - - return 0; - } - - public void PrepareOperation(AskMode askExtractMode) - { - } - - public void SetOperationResult(OperationResult resultEOperationResult) - { - } - } -} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamsCallback.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamsCallback.cs deleted file mode 100644 index 1cf9feed..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveStreamsCallback.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Collections.Generic; -using System.IO; - -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - internal class ArchiveStreamsCallback : IArchiveExtractCallback - { - private readonly IList streams; - - public ArchiveStreamsCallback(IList streams) - { - this.streams = streams; - } - - public void SetTotal(ulong total) - { - } - - public void SetCompleted(ref ulong completeValue) - { - } - - public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode) - { - if (askExtractMode != AskMode.kExtract) - { - outStream = null; - return 0; - } - - if (this.streams == null) - { - outStream = null; - return 0; - } - - Stream stream = this.streams[(int) index]; - - if (stream == null) - { - outStream = null; - return 0; - } - - outStream = new OutStreamWrapper(stream); - - return 0; - } - - public void PrepareOperation(AskMode askExtractMode) - { - } - - public void SetOperationResult(OperationResult resultEOperationResult) - { - } - } -} diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs index ff6b8e20..15a960c0 100644 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Entry.cs @@ -85,35 +85,5 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor /// True if there are parts of this file in next split archive parts /// public bool IsSplitAfter { get; set; } - - public void Extract(string fileName, bool preserveTimestamp = true) - { - if (this.IsFolder) - { - Directory.CreateDirectory(fileName); - return; - } - - string directoryName = Path.GetDirectoryName(fileName); - - if (!string.IsNullOrWhiteSpace(directoryName)) - { - Directory.CreateDirectory(directoryName); - } - - using (FileStream fileStream = File.Create(fileName)) - { - this.Extract(fileStream); - } - - if (preserveTimestamp) - { - File.SetLastWriteTime(fileName, this.LastWriteTime); - } - } - public void Extract(Stream stream) - { - this.archive.Extract(new[] { this.index }, 1, 0, new ArchiveStreamCallback(this.index, stream)); - } } } diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs index 64c36920..bc82e4a8 100644 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs @@ -6,87 +6,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor { public class Formats { - internal static readonly Dictionary ExtensionFormatMapping = new Dictionary - { - {"7z", SevenZipFormat.SevenZip}, - {"gz", SevenZipFormat.GZip}, - {"tar", SevenZipFormat.Tar}, - {"rar", SevenZipFormat.Rar}, - {"zip", SevenZipFormat.Zip}, - {"lzma", SevenZipFormat.Lzma}, - {"lzh", SevenZipFormat.Lzh}, - {"arj", SevenZipFormat.Arj}, - {"bz2", SevenZipFormat.BZip2}, - {"cab", SevenZipFormat.Cab}, - {"chm", SevenZipFormat.Chm}, - {"deb", SevenZipFormat.Deb}, - {"iso", SevenZipFormat.Iso}, - {"rpm", SevenZipFormat.Rpm}, - {"wim", SevenZipFormat.Wim}, - {"udf", SevenZipFormat.Udf}, - {"mub", SevenZipFormat.Mub}, - {"xar", SevenZipFormat.Xar}, - {"hfs", SevenZipFormat.Hfs}, - {"dmg", SevenZipFormat.Dmg}, - {"z", SevenZipFormat.Lzw}, - {"xz", SevenZipFormat.XZ}, - {"flv", SevenZipFormat.Flv}, - {"swf", SevenZipFormat.Swf}, - {"exe", SevenZipFormat.PE}, - {"dll", SevenZipFormat.PE}, - {"vhd", SevenZipFormat.Vhd} - }; - - internal static Dictionary FormatGuidMapping = new Dictionary - { - {SevenZipFormat.SevenZip, new Guid("23170f69-40c1-278a-1000-000110070000")}, - {SevenZipFormat.Arj, new Guid("23170f69-40c1-278a-1000-000110040000")}, - {SevenZipFormat.BZip2, new Guid("23170f69-40c1-278a-1000-000110020000")}, - {SevenZipFormat.Cab, new Guid("23170f69-40c1-278a-1000-000110080000")}, - {SevenZipFormat.Chm, new Guid("23170f69-40c1-278a-1000-000110e90000")}, - {SevenZipFormat.Compound, new Guid("23170f69-40c1-278a-1000-000110e50000")}, - {SevenZipFormat.Cpio, new Guid("23170f69-40c1-278a-1000-000110ed0000")}, - {SevenZipFormat.Deb, new Guid("23170f69-40c1-278a-1000-000110ec0000")}, - {SevenZipFormat.GZip, new Guid("23170f69-40c1-278a-1000-000110ef0000")}, - {SevenZipFormat.Iso, new Guid("23170f69-40c1-278a-1000-000110e70000")}, - {SevenZipFormat.Lzh, new Guid("23170f69-40c1-278a-1000-000110060000")}, - {SevenZipFormat.Lzma, new Guid("23170f69-40c1-278a-1000-0001100a0000")}, - {SevenZipFormat.Nsis, new Guid("23170f69-40c1-278a-1000-000110090000")}, - {SevenZipFormat.Rar, new Guid("23170f69-40c1-278a-1000-000110030000")}, - {SevenZipFormat.Rar5, new Guid("23170f69-40c1-278a-1000-000110CC0000")}, - {SevenZipFormat.Rpm, new Guid("23170f69-40c1-278a-1000-000110eb0000")}, - {SevenZipFormat.Split, new Guid("23170f69-40c1-278a-1000-000110ea0000")}, - {SevenZipFormat.Tar, new Guid("23170f69-40c1-278a-1000-000110ee0000")}, - {SevenZipFormat.Wim, new Guid("23170f69-40c1-278a-1000-000110e60000")}, - {SevenZipFormat.Lzw, new Guid("23170f69-40c1-278a-1000-000110050000")}, - {SevenZipFormat.Zip, new Guid("23170f69-40c1-278a-1000-000110010000")}, - {SevenZipFormat.Udf, new Guid("23170f69-40c1-278a-1000-000110E00000")}, - {SevenZipFormat.Xar, new Guid("23170f69-40c1-278a-1000-000110E10000")}, - {SevenZipFormat.Mub, new Guid("23170f69-40c1-278a-1000-000110E20000")}, - {SevenZipFormat.Hfs, new Guid("23170f69-40c1-278a-1000-000110E30000")}, - {SevenZipFormat.Dmg, new Guid("23170f69-40c1-278a-1000-000110E40000")}, - {SevenZipFormat.XZ, new Guid("23170f69-40c1-278a-1000-0001100C0000")}, - {SevenZipFormat.Mslz, new Guid("23170f69-40c1-278a-1000-000110D50000")}, - {SevenZipFormat.PE, new Guid("23170f69-40c1-278a-1000-000110DD0000")}, - {SevenZipFormat.Elf, new Guid("23170f69-40c1-278a-1000-000110DE0000")}, - {SevenZipFormat.Swf, new Guid("23170f69-40c1-278a-1000-000110D70000")}, - {SevenZipFormat.Vhd, new Guid("23170f69-40c1-278a-1000-000110DC0000")}, - {SevenZipFormat.Flv, new Guid("23170f69-40c1-278a-1000-000110D60000")}, - {SevenZipFormat.SquashFS, new Guid("23170f69-40c1-278a-1000-000110D20000")}, - {SevenZipFormat.Lzma86, new Guid("23170f69-40c1-278a-1000-0001100B0000")}, - {SevenZipFormat.Ppmd, new Guid("23170f69-40c1-278a-1000-0001100D0000")}, - {SevenZipFormat.TE, new Guid("23170f69-40c1-278a-1000-000110CF0000")}, - {SevenZipFormat.UEFIc, new Guid("23170f69-40c1-278a-1000-000110D00000")}, - {SevenZipFormat.UEFIs, new Guid("23170f69-40c1-278a-1000-000110D10000")}, - {SevenZipFormat.CramFS, new Guid("23170f69-40c1-278a-1000-000110D30000")}, - {SevenZipFormat.APM, new Guid("23170f69-40c1-278a-1000-000110D40000")}, - {SevenZipFormat.Swfc, new Guid("23170f69-40c1-278a-1000-000110D80000")}, - {SevenZipFormat.Ntfs, new Guid("23170f69-40c1-278a-1000-000110D90000")}, - {SevenZipFormat.Fat, new Guid("23170f69-40c1-278a-1000-000110DA0000")}, - {SevenZipFormat.Mbr, new Guid("23170f69-40c1-278a-1000-000110DB0000")}, - {SevenZipFormat.MachO, new Guid("23170f69-40c1-278a-1000-000110DF0000")} - }; - + internal static Dictionary FileTypeGuidMapping = new Dictionary { {Definitions.FileType._7Z, new Guid("23170f69-40c1-278a-1000-000110070000")}, @@ -96,27 +16,5 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor {Definitions.FileType.ZIP, new Guid("23170f69-40c1-278a-1000-000110010000")}, }; - internal static Dictionary FileSignatures = new Dictionary - { - {SevenZipFormat.Rar5, new byte[] {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00}}, - {SevenZipFormat.Rar, new byte[] { 0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00 }}, - {SevenZipFormat.Vhd, new byte[] { 0x63, 0x6F, 0x6E, 0x65, 0x63, 0x74, 0x69, 0x78 }}, - {SevenZipFormat.Deb, new byte[] { 0x21, 0x3C, 0x61, 0x72, 0x63, 0x68, 0x3E }}, - {SevenZipFormat.Dmg, new byte[] { 0x78, 0x01, 0x73, 0x0D, 0x62, 0x62, 0x60 }}, - {SevenZipFormat.SevenZip, new byte[] { 0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C }}, - {SevenZipFormat.Tar, new byte[] { 0x75, 0x73, 0x74, 0x61, 0x72 }}, - {SevenZipFormat.Iso, new byte[] { 0x43, 0x44, 0x30, 0x30, 0x31 }}, - {SevenZipFormat.Cab, new byte[] { 0x4D, 0x53, 0x43, 0x46 }}, - {SevenZipFormat.Rpm, new byte[] { 0xed, 0xab, 0xee, 0xdb }}, - {SevenZipFormat.Xar, new byte[] { 0x78, 0x61, 0x72, 0x21 }}, - {SevenZipFormat.Chm, new byte[] { 0x49, 0x54, 0x53, 0x46 }}, - {SevenZipFormat.BZip2, new byte[] { 0x42, 0x5A, 0x68 }}, - {SevenZipFormat.Flv, new byte[] { 0x46, 0x4C, 0x56 }}, - {SevenZipFormat.Swf, new byte[] { 0x46, 0x57, 0x53 }}, - {SevenZipFormat.GZip, new byte[] { 0x1f, 0x0b }}, - {SevenZipFormat.Zip, new byte[] { 0x50, 0x4b }}, - {SevenZipFormat.Arj, new byte[] { 0x60, 0xEA }}, - {SevenZipFormat.Lzh, new byte[] { 0x2D, 0x6C, 0x68 }} - }; } } diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipException.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipException.cs deleted file mode 100644 index 9fd9665c..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipException.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using System.Runtime.Serialization; - -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - public class SevenZipException : Exception - { - public SevenZipException() - { - } - - public SevenZipException(string message) : base(message) - { - } - - public SevenZipException(string message, Exception innerException) : base(message, innerException) - { - } - - protected SevenZipException(SerializationInfo info, StreamingContext context) : base(info, context) - { - } - } -} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipFormat.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipFormat.cs deleted file mode 100644 index 6ca1ed4b..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipFormat.cs +++ /dev/null @@ -1,285 +0,0 @@ -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - /// - /// - /// - public enum SevenZipFormat - { - // Default invalid format value - Undefined = 0, - - /// - /// Open 7-zip archive format. - /// - /// Wikipedia information - SevenZip, - - /// - /// Proprietary Arj archive format. - /// - /// Wikipedia information - Arj, - - /// - /// Open Bzip2 archive format. - /// - /// Wikipedia information - BZip2, - - /// - /// Microsoft cabinet archive format. - /// - /// Wikipedia information - Cab, - - /// - /// Microsoft Compiled HTML Help file format. - /// - /// Wikipedia information - Chm, - - /// - /// Microsoft Compound file format. - /// - /// Wikipedia information - Compound, - - /// - /// Open Cpio archive format. - /// - /// Wikipedia information - Cpio, - - /// - /// Open Debian software package format. - /// - /// Wikipedia information - Deb, - - /// - /// Open Gzip archive format. - /// - /// Wikipedia information - GZip, - - /// - /// Open ISO disk image format. - /// - /// Wikipedia information - Iso, - - /// - /// Open Lzh archive format. - /// - /// Wikipedia information - Lzh, - - /// - /// Open core 7-zip Lzma raw archive format. - /// - /// Wikipedia information - Lzma, - - /// - /// Nullsoft installation package format. - /// - /// Wikipedia information - Nsis, - - /// - /// RarLab Rar archive format. - /// - /// Wikipedia information - Rar, - - /// - /// RarLab Rar archive format, version 5. - /// - /// Wikipedia information - Rar5, - - /// - /// Open Rpm software package format. - /// - /// Wikipedia information - Rpm, - - /// - /// Open split file format. - /// - /// Wikipedia information - Split, - - /// - /// Open Tar archive format. - /// - /// Wikipedia information - Tar, - - /// - /// Microsoft Windows Imaging disk image format. - /// - /// Wikipedia information - Wim, - - /// - /// Open LZW archive format; implemented in "compress" program; also known as "Z" archive format. - /// - /// Wikipedia information - Lzw, - - /// - /// Open Zip archive format. - /// - /// Wikipedia information - Zip, - - /// - /// Open Udf disk image format. - /// - Udf, - - /// - /// Xar open source archive format. - /// - /// Wikipedia information - Xar, - - /// - /// Mub - /// - Mub, - - /// - /// Macintosh Disk Image on CD. - /// - /// Wikipedia information - Hfs, - - /// - /// Apple Mac OS X Disk Copy Disk Image format. - /// - Dmg, - - /// - /// Open Xz archive format. - /// - /// Wikipedia information - XZ, - - /// - /// MSLZ archive format. - /// - Mslz, - - /// - /// Flash video format. - /// - /// Wikipedia information - Flv, - - /// - /// Shockwave Flash format. - /// - /// Wikipedia information - Swf, - - /// - /// Windows PE executable format. - /// - /// Wikipedia information - PE, - - /// - /// Linux executable Elf format. - /// - /// Wikipedia information - Elf, - - /// - /// Windows Installer Database. - /// - /// Wikipedia information - Msi, - - /// - /// Microsoft virtual hard disk file format. - /// - /// Wikipedia information - Vhd, - - /// - /// SquashFS file system format. - /// - /// Wikipedia information - SquashFS, - - /// - /// Lzma86 file format. - /// - Lzma86, - - /// - /// Prediction by Partial Matching by Dmitry algorithm. - /// - /// Wikipedia information - Ppmd, - - /// - /// TE format. - /// - TE, - - /// - /// UEFIc format. - /// - /// Wikipedia information - UEFIc, - - /// - /// UEFIs format. - /// - /// Wikipedia information - UEFIs, - - /// - /// Compressed ROM file system format. - /// - /// Wikipedia information - CramFS, - - /// - /// APM format. - /// - APM, - - /// - /// Swfc format. - /// - Swfc, - - /// - /// NTFS file system format. - /// - /// Wikipedia information - Ntfs, - - /// - /// FAT file system format. - /// - /// Wikipedia information - Fat, - - /// - /// MBR format. - /// - /// Wikipedia information - Mbr, - - /// - /// Mach-O file format. - /// - /// Wikipedia information - MachO - } -} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/TypedExtractor.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/TypedExtractor.cs deleted file mode 100644 index 6904ff87..00000000 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/TypedExtractor.cs +++ /dev/null @@ -1,64 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using Wabbajack.Common; - -namespace Wabbajack.VirtualFileSystem.SevenZipExtractor -{ - internal class TypedExtractor : IArchiveExtractCallback - { - private Dictionary _mappings; - private Action> _callback; - private Dictionary _indexToFile; - - public TypedExtractor(Dictionary mappings, Action> callback) - { - _mappings = mappings; - _callback = callback; - } - - public void Extract(ArchiveFile file) - { - _indexToFile = new Dictionary(); - - uint idx = 0; - foreach (var entry in file.Entries) - { - var rel = (RelativePath)entry.FileName; - if (_mappings.ContainsKey(rel)) ; - { - _indexToFile.Add(idx, rel); - } - idx += 1; - } - - file._archive.Extract(null, 0xFFFFFFFF, 0, this); - } - - public void SetTotal(ulong total) - { - throw new System.NotImplementedException(); - } - - public void SetCompleted(ref ulong completeValue) - { - throw new System.NotImplementedException(); - } - - public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode) - { - outStream = null; - throw new System.NotImplementedException(); - } - - public void PrepareOperation(AskMode askExtractMode) - { - throw new System.NotImplementedException(); - } - - public void SetOperationResult(OperationResult resultEOperationResult) - { - throw new System.NotImplementedException(); - } - } -} diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index f874821f..9719ccbb 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -49,23 +49,6 @@ namespace Wabbajack.VirtualFileSystem public Context Context { get; set; } - private IExtractedFile _stagedFile = null; - public IExtractedFile StagedFile - { - get - { - if (IsNative) return new ExtractedDiskFile(AbsoluteName); - if (_stagedFile == null) - throw new InvalidDataException("File is unstaged"); - return _stagedFile; - } - set - { - _stagedFile = value; - } - - } - /// /// Returns the nesting factor for this file. Native files will have a nesting of 1, the factor /// goes up for each nesting of a file in an archive. @@ -382,11 +365,6 @@ namespace Wabbajack.VirtualFileSystem var path = new HashRelativePath(FilesInFullPath.First().Hash, paths); return path; } - - public async ValueTask OpenRead() - { - return await StagedFile.OpenRead(); - } } public class ExtendedHashes From 5d7bceb6dc1bb6b27549aa97bc8dc2d10e49595b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 7 Sep 2020 20:22:23 -0600 Subject: [PATCH 08/13] Spool large files to disk during extraction to save memory --- Compression.BSA/MemoryStreamFactory.cs | 19 +++ Wabbajack.Lib/AInstaller.cs | 8 +- .../FileExtractor2/FileExtractor.cs | 4 + .../FileExtractor2/GatheringExtractor.cs | 119 +++++++++++++----- .../SevenZipExtractor/SevenZipInterface.cs | 6 +- Wabbajack.VirtualFileSystem/VirtualFile.cs | 10 +- 6 files changed, 127 insertions(+), 39 deletions(-) diff --git a/Compression.BSA/MemoryStreamFactory.cs b/Compression.BSA/MemoryStreamFactory.cs index c428ed9e..09821140 100644 --- a/Compression.BSA/MemoryStreamFactory.cs +++ b/Compression.BSA/MemoryStreamFactory.cs @@ -21,4 +21,23 @@ namespace Compression.BSA public DateTime LastModifiedUtc => DateTime.UtcNow; public IPath Name => (RelativePath)"BSA Memory Stream"; } + + public class MemoryBufferFactory : IStreamFactory + { + private readonly byte[] _data; + private int _size; + + public MemoryBufferFactory(byte[] data, int size) + { + _data = data; + _size = size; + } + public async ValueTask GetStream() + { + return new MemoryStream(_data, 0, _size); + } + + public DateTime LastModifiedUtc => DateTime.UtcNow; + public IPath Name => (RelativePath)"BSA Memory Stream"; + } } diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index c76db728..ad291fed 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -150,9 +150,11 @@ namespace Wabbajack.Lib { var patchData = await LoadBytesFromPath(pfa.PatchID); var toFile = file.To.RelativeTo(OutputFolder); - await using var os = await toFile.Create(); - Utils.ApplyPatch(s, () => new MemoryStream(patchData), os); - + { + await using var os = await toFile.Create(); + Utils.ApplyPatch(s, () => new MemoryStream(patchData), os); + } + if (await VirusScanner.ShouldScan(toFile) && await ClientAPI.GetVirusScanResult(toFile) == VirusScanner.Result.Malware) { diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs index ab66907f..27261a26 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs @@ -27,6 +27,10 @@ namespace Wabbajack.VirtualFileSystem public static async Task> GatheringExtract(IStreamFactory sFn, Predicate shouldExtract, Func> mapfn) { + if (sFn is NativeFileStreamFactory) + { + Utils.Log($"Extracting {sFn.Name}"); + } await using var archive = await sFn.GetStream(); var sig = await ArchiveSigs.MatchesAsync(archive); archive.Position = 0; diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs index 154ea053..c19c9158 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs @@ -21,6 +21,7 @@ namespace Wabbajack.VirtualFileSystem private Dictionary _indexes; private Stream _stream; private Definitions.FileType _sig; + private Exception _killException; public GatheringExtractor(Stream stream, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) { @@ -51,7 +52,14 @@ namespace Wabbajack.VirtualFileSystem _archive._archive.Extract(null, 0xFFFFFFFF, 0, this); _archive.Dispose(); - source.SetResult(true); + if (_killException != null) + { + source.SetException(_killException); + } + else + { + source.SetResult(true); + } } catch (Exception ex) { @@ -105,51 +113,97 @@ namespace Wabbajack.VirtualFileSystem private uint _index; private bool _written; private ulong _totalSize; - private MemoryStream _tmpStream; + private Stream _tmpStream; + private TempFile _tmpFile; + private IStreamFactory _factory; + private bool _diskCached; public GatheringExtractorStream(GatheringExtractor extractor, uint index) { _extractor = extractor; _index = index; - _written = false; _totalSize = extractor._indexes[index].Item2; - _tmpStream = new MemoryStream(); + _diskCached = _totalSize >= 500_000_000; } - public int Write(IntPtr data, uint size, IntPtr processedSize) + public int Write(byte[] data, uint size, IntPtr processedSize) { - unsafe + try { - var ums = new UnmanagedMemoryStream((byte*)data, size); - ums.CopyTo(_tmpStream); - if ((ulong)_tmpStream.Length >= _totalSize) - { - _tmpStream.Position = 0; - var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryStreamFactory(_tmpStream)).AsTask().Result; + if (size == _totalSize) + WriteSingleCall(data, size); + else if (_diskCached) + WriteDiskCached(data, size); + else + WriteMemoryCached(data, size); - _extractor._results[_extractor._indexes[_index].Item1] = result; - } - if (processedSize != IntPtr.Zero) { - Marshal.WriteInt32(processedSize, (int) size); + Marshal.WriteInt32(processedSize, (int)size); } - } - - return 0; - - if (_written) throw new Exception("TODO"); - unsafe - { - - - - - _written = true; - - return 0; + return 0; } + catch (Exception ex) + { + Utils.Log($"Error during extraction {ex}"); + _extractor.Kill(ex); + + return 1; + } + } + + private void WriteSingleCall(byte[] data, in uint size) + { + var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryBufferFactory(data, (int)size)).Result; + AddResult(result); + Cleanup(); + } + + private void Cleanup() + { + _tmpStream?.Dispose(); + _tmpFile?.DisposeAsync().AsTask().Wait(); + } + + private void AddResult(T result) + { + _extractor._results.Add(_extractor._indexes[_index].Item1, result); + } + + private void WriteMemoryCached(byte[] data, in uint size) + { + if (_tmpStream == null) + _tmpStream = new MemoryStream(); + _tmpStream.Write(data, 0, (int)size); + + if (_tmpStream.Length != (long)_totalSize) return; + + _tmpStream.Flush(); + _tmpStream.Position = 0; + var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryStreamFactory((MemoryStream)_tmpStream)).Result; + AddResult(result); + Cleanup(); + } + + private void WriteDiskCached(byte[] data, in uint size) + { + if (_tmpFile == null) + { + _tmpFile = new TempFile(); + _tmpStream = _tmpFile.Path.Create().Result; + } + + _tmpStream.Write(data, 0, (int)size); + + if (_tmpStream.Length != (long)_totalSize) return; + + _tmpStream.Flush(); + _tmpStream.Close(); + + var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new NativeFileStreamFactory(_tmpFile.Path)).Result; + AddResult(result); + Cleanup(); } public void Seek(long offset, uint seekOrigin, IntPtr newPosition) @@ -162,5 +216,10 @@ namespace Wabbajack.VirtualFileSystem return 0; } } + + private void Kill(Exception ex) + { + _killException = ex; + } } } diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs index cfc0d371..f9b6811c 100644 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs @@ -207,7 +207,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor { [PreserveSig] int Write( - IntPtr data, + [In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)]byte[] data, uint size, IntPtr processedSize); // ref uint processedSize /* @@ -246,7 +246,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor { [PreserveSig] int Write( - IntPtr data, + [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 1)] byte[] data, uint size, IntPtr processedSize); // ref uint processedSize @@ -444,7 +444,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor return 0; } - public int Write(IntPtr data, uint size, IntPtr processedSize) + public int Write(byte[] data, uint size, IntPtr processedSize) { throw new NotImplementedException(); /* diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 9719ccbb..63bdf4b2 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -209,12 +209,16 @@ namespace Wabbajack.VirtualFileSystem try { - var list = await FileExtractor2.GatheringExtract(extractedFile, - _ => true, + var list = await FileExtractor2.GatheringExtract(extractedFile, + _ => true, async (path, sfactory) => await Analyze(context, self, sfactory, path, depth + 1)); - + self.Children = list.Values.ToImmutableList(); } + catch (EndOfStreamException ex) + { + return self; + } catch (Exception ex) { Utils.Log($"Error while examining the contents of {relPath.FileName}"); From 548a0553d241d45ea326ce8b0026f115d63162d3 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 8 Sep 2020 16:15:33 -0600 Subject: [PATCH 09/13] Re-add OMOD support --- Compression.BSA/BA2Reader.cs | 4 +- Compression.BSA/BSA/Reader/FileRecord.cs | 2 +- Compression.BSA/MemoryStreamFactory.cs | 10 +- Compression.BSA/TES3Reader.cs | 2 +- Wabbajack.Common/IStreamFactory.cs | 9 +- Wabbajack.Common/Paths/FileCompaction.cs | 10 ++ Wabbajack.Lib/MO2Installer.cs | 13 ++- .../FileExtractorTests.cs | 43 +++++++++ .../Wabbajack.VirtualFileSystem.Test.csproj | 1 + .../FileExtractor2/FileExtractor.cs | 94 +++++++++++++------ .../FileExtractor2/GatheringExtractor.cs | 11 ++- .../Wabbajack.VirtualFileSystem.csproj | 3 + 12 files changed, 161 insertions(+), 41 deletions(-) diff --git a/Compression.BSA/BA2Reader.cs b/Compression.BSA/BA2Reader.cs index e3abe392..19b08e3c 100644 --- a/Compression.BSA/BA2Reader.cs +++ b/Compression.BSA/BA2Reader.cs @@ -350,7 +350,7 @@ namespace Compression.BSA var ms = new MemoryStream(); await CopyDataTo(ms); ms.Position = 0; - return new MemoryStreamFactory(ms); + return new MemoryStreamFactory(ms, Path); } } @@ -517,7 +517,7 @@ namespace Compression.BSA var ms = new MemoryStream(); await CopyDataTo(ms); ms.Position = 0; - return new MemoryStreamFactory(ms); + return new MemoryStreamFactory(ms, Path); } } diff --git a/Compression.BSA/BSA/Reader/FileRecord.cs b/Compression.BSA/BSA/Reader/FileRecord.cs index c7a0fa54..c8f7d495 100644 --- a/Compression.BSA/BSA/Reader/FileRecord.cs +++ b/Compression.BSA/BSA/Reader/FileRecord.cs @@ -171,7 +171,7 @@ namespace Compression.BSA var ms = new MemoryStream(); await CopyDataTo(ms); ms.Position = 0; - return new MemoryStreamFactory(ms); + return new MemoryStreamFactory(ms, Path); } } } diff --git a/Compression.BSA/MemoryStreamFactory.cs b/Compression.BSA/MemoryStreamFactory.cs index 09821140..ce08513c 100644 --- a/Compression.BSA/MemoryStreamFactory.cs +++ b/Compression.BSA/MemoryStreamFactory.cs @@ -9,9 +9,10 @@ namespace Compression.BSA { private readonly MemoryStream _data; - public MemoryStreamFactory(MemoryStream data) + public MemoryStreamFactory(MemoryStream data, IPath path) { _data = data; + Name = path; } public async ValueTask GetStream() { @@ -19,7 +20,7 @@ namespace Compression.BSA } public DateTime LastModifiedUtc => DateTime.UtcNow; - public IPath Name => (RelativePath)"BSA Memory Stream"; + public IPath Name { get; } } public class MemoryBufferFactory : IStreamFactory @@ -27,10 +28,11 @@ namespace Compression.BSA private readonly byte[] _data; private int _size; - public MemoryBufferFactory(byte[] data, int size) + public MemoryBufferFactory(byte[] data, int size, IPath path) { _data = data; _size = size; + Name = path; } public async ValueTask GetStream() { @@ -38,6 +40,6 @@ namespace Compression.BSA } public DateTime LastModifiedUtc => DateTime.UtcNow; - public IPath Name => (RelativePath)"BSA Memory Stream"; + public IPath Name { get; } } } diff --git a/Compression.BSA/TES3Reader.cs b/Compression.BSA/TES3Reader.cs index f9fd7afb..2598057f 100644 --- a/Compression.BSA/TES3Reader.cs +++ b/Compression.BSA/TES3Reader.cs @@ -135,7 +135,7 @@ namespace Compression.BSA var ms = new MemoryStream(); await CopyDataTo(ms); ms.Position = 0; - return new MemoryStreamFactory(ms); + return new MemoryStreamFactory(ms, Path); } diff --git a/Wabbajack.Common/IStreamFactory.cs b/Wabbajack.Common/IStreamFactory.cs index 783c425e..4fc2d541 100644 --- a/Wabbajack.Common/IStreamFactory.cs +++ b/Wabbajack.Common/IStreamFactory.cs @@ -18,9 +18,16 @@ namespace Wabbajack.Common { private AbsolutePath _file; + public NativeFileStreamFactory(AbsolutePath file, IPath path) + { + _file = file; + Name = path; + } + public NativeFileStreamFactory(AbsolutePath file) { _file = file; + Name = file; } public async ValueTask GetStream() { @@ -28,7 +35,7 @@ namespace Wabbajack.Common } public DateTime LastModifiedUtc => _file.LastModifiedUtc; - public IPath Name => _file; + public IPath Name { get; } } } diff --git a/Wabbajack.Common/Paths/FileCompaction.cs b/Wabbajack.Common/Paths/FileCompaction.cs index a1a6e64a..cd5bbaf3 100644 --- a/Wabbajack.Common/Paths/FileCompaction.cs +++ b/Wabbajack.Common/Paths/FileCompaction.cs @@ -57,5 +57,15 @@ namespace Wabbajack.Common return await proc.Start() == 0; } } + + public static async Task CompactFolder(this AbsolutePath folder, WorkQueue queue, Algorithm algorithm) + { + await folder.EnumerateFiles(true) + .PMap(queue, async path => + { + Utils.Status($"Compacting {path.FileName}"); + await path.Compact(algorithm); + }); + } } } diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index 8d1f3e6a..ae004242 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -40,7 +40,7 @@ namespace Wabbajack.Lib outputFolder: outputFolder, downloadFolder: downloadFolder, parameters: parameters, - steps: 21, + steps: 22, game: modList.GameType) { var gameExe = Consts.GameFolderFilesDir.Combine(modList.GameType.MetaData().MainExecutable!); @@ -178,6 +178,9 @@ namespace Wabbajack.Lib UpdateTracker.NextStep("Updating System-specific ini settings"); SetScreenSizeInPrefs(); + + UpdateTracker.NextStep("Compacting files"); + await CompactFiles(); UpdateTracker.NextStep("Installation complete! You may exit the program."); await ExtractedModlistFolder!.DisposeAsync(); @@ -186,6 +189,14 @@ namespace Wabbajack.Lib return true; } + private async Task CompactFiles() + { + if (this.UseCompression) + { + await OutputFolder.CompactFolder(Queue, FileCompaction.Algorithm.XPRESS16K); + } + } + private void CreateOutputMods() { OutputFolder.Combine("profiles") diff --git a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs index f166b7ae..1be835f1 100644 --- a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs @@ -1,7 +1,10 @@ using System; using System.IO.Compression; +using System.Linq; using System.Threading.Tasks; using Wabbajack.Common; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.NexusApi; using Xunit; namespace Wabbajack.VirtualFileSystem.Test @@ -33,8 +36,26 @@ namespace Wabbajack.VirtualFileSystem.Test { Assert.Equal(await temp.Dir.Combine(path).FileHashAsync(), hash); } + } + + private static Extension OMODExtension = new Extension(".omod"); + private static Extension CRCExtension = new Extension(".crc"); + [Fact] + public async Task CanGatherDataFromOMODFiles() + { + var src = await DownloadMod(Game.Oblivion, 18498); + await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(src), + p => p.Extension == OMODExtension, async (path, sfn) => + { + await FileExtractor2.GatheringExtract(sfn, _ => true, async (ipath, isfn) => { + // We shouldn't have any .crc files because this file should be recognized as a OMOD and extracted correctly + Assert.NotEqual(CRCExtension, ipath.Extension); + return 0; + }); + return 0; + }); } @@ -59,5 +80,27 @@ namespace Wabbajack.VirtualFileSystem.Test await folder.DeleteDirectory(); } + + private static AbsolutePath _stagingFolder = ((RelativePath)"NexusDownloads").RelativeToEntryPoint(); + private static async Task DownloadMod(Game game, int mod) + { + using var client = await NexusApiClient.Get(); + var results = await client.GetModFiles(game, mod); + var file = results.files.FirstOrDefault(f => f.is_primary) ?? + results.files.OrderByDescending(f => f.uploaded_timestamp).First(); + var src = _stagingFolder.Combine(file.file_name); + + if (src.Exists) return src; + + var state = new NexusDownloader.State + { + ModID = mod, + Game = game, + FileID = file.file_id + }; + await state.Download(src); + return src; + } + } } diff --git a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj index 5910556b..c12e83ff 100644 --- a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj +++ b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj @@ -15,6 +15,7 @@ + diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs index 27261a26..a7c1a7c5 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs @@ -4,11 +4,13 @@ using System.IO; using System.Threading.Tasks; using Compression.BSA; using ICSharpCode.SharpZipLib.Zip.Compression.Streams; +using OMODFramework; using SharpCompress.Archives.SevenZip; using SharpCompress.Readers; using Wabbajack.Common; using Wabbajack.Common.FileSignatures; using Wabbajack.VirtualFileSystem.SevenZipExtractor; +using Utils = Wabbajack.Common.Utils; namespace Wabbajack.VirtualFileSystem { @@ -22,6 +24,8 @@ namespace Wabbajack.VirtualFileSystem Definitions.FileType.RAR_OLD, Definitions.FileType.RAR_NEW, Definitions.FileType._7Z); + + private static Extension OMODExtension = new Extension(".omod"); public static async Task> GatheringExtract(IStreamFactory sFn, @@ -39,10 +43,21 @@ namespace Wabbajack.VirtualFileSystem { case Definitions.FileType.RAR_OLD: case Definitions.FileType.RAR_NEW: - case Definitions.FileType._7Z: + case Definitions.FileType._7Z: case Definitions.FileType.ZIP: - return await GatheringExtractWith7Zip(archive, (Definitions.FileType)sig, shouldExtract, mapfn); - + { + if (sFn.Name.FileName.Extension == OMODExtension) + { + return await GatheringExtractWithOMOD(archive, shouldExtract, mapfn); + + } + else + { + return await GatheringExtractWith7Zip(archive, (Definitions.FileType)sig, shouldExtract, + mapfn); + } + } + case Definitions.FileType.TES3: case Definitions.FileType.BSA: case Definitions.FileType.BA2: @@ -54,6 +69,54 @@ namespace Wabbajack.VirtualFileSystem } } + private static async Task> GatheringExtractWithOMOD(Stream archive, Predicate shouldExtract, Func> mapfn) + { + var tmpFile = new TempFile(); + await tmpFile.Path.WriteAllAsync(archive); + var dest = await TempFolder.Create(); + Utils.Log($"Extracting {(string)tmpFile.Path}"); + + Framework.Settings.TempPath = (string)dest.Dir; + Framework.Settings.CodeProgress = new OMODProgress(); + + var omod = new OMOD((string)tmpFile.Path); + omod.GetDataFiles(); + omod.GetPlugins(); + + var results = new Dictionary(); + foreach (var file in dest.Dir.EnumerateFiles()) + { + var path = file.RelativeTo(dest.Dir); + if (!shouldExtract(path)) continue; + + var result = await mapfn(path, new NativeFileStreamFactory(file, path)); + results.Add(path, result); + } + + return results; + } + + private class OMODProgress : ICodeProgress + { + private long _total; + + public void SetProgress(long inSize, long outSize) + { + Utils.Status("Extracting OMOD", Percent.FactoryPutInRange(inSize, _total)); + } + + public void Init(long totalSize, bool compressing) + { + _total = totalSize; + } + + public void Dispose() + { + // + } + } + + private static async Task> GatheringExtractWithBSA(IStreamFactory sFn, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) { var archive = await BSADispatch.OpenRead(sFn, sig); @@ -73,31 +136,6 @@ namespace Wabbajack.VirtualFileSystem private static async Task> GatheringExtractWith7Zip(Stream stream, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) { return await new GatheringExtractor(stream, sig, shouldExtract, mapfn).Extract(); - /* - IReader reader; - if (sig == Definitions.FileType._7Z) - reader = SevenZipArchive.Open(stream).ExtractAllEntries(); - else - { - reader = ReaderFactory.Open(stream); - } - - var results = new Dictionary(); - while (reader.MoveToNextEntry()) - { - var path = (RelativePath)reader.Entry.Key; - if (!reader.Entry.IsDirectory && shouldExtract(path)) - { - var ms = new MemoryStream(); - reader.WriteEntryTo(ms); - ms.Position = 0; - var result = await mapfn(path, new MemoryStreamFactory(ms)); - results.Add(path, result); - } - } - - return results; - */ } public static async Task ExtractAll(AbsolutePath src, AbsolutePath dest) diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs index c19c9158..c86eb922 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs @@ -126,6 +126,11 @@ namespace Wabbajack.VirtualFileSystem _diskCached = _totalSize >= 500_000_000; } + private IPath GetPath() + { + return _extractor._indexes[_index].Item1; + } + public int Write(byte[] data, uint size, IntPtr processedSize) { try @@ -155,7 +160,7 @@ namespace Wabbajack.VirtualFileSystem private void WriteSingleCall(byte[] data, in uint size) { - var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryBufferFactory(data, (int)size)).Result; + var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryBufferFactory(data, (int)size, GetPath())).Result; AddResult(result); Cleanup(); } @@ -181,7 +186,7 @@ namespace Wabbajack.VirtualFileSystem _tmpStream.Flush(); _tmpStream.Position = 0; - var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryStreamFactory((MemoryStream)_tmpStream)).Result; + var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryStreamFactory((MemoryStream)_tmpStream, GetPath())).Result; AddResult(result); Cleanup(); } @@ -201,7 +206,7 @@ namespace Wabbajack.VirtualFileSystem _tmpStream.Flush(); _tmpStream.Close(); - var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new NativeFileStreamFactory(_tmpFile.Path)).Result; + var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new NativeFileStreamFactory(_tmpFile.Path, GetPath())).Result; AddResult(result); Cleanup(); } diff --git a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj index 3674045e..c440c185 100644 --- a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj +++ b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj @@ -31,4 +31,7 @@ Always + + + \ No newline at end of file From 8a452fd17247b5c0f23c5c072e62048c54e4db57 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 8 Sep 2020 17:25:17 -0600 Subject: [PATCH 10/13] Disable broken BNET test --- Wabbajack.Test/DownloaderTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 72ea36c6..53f4143a 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -485,6 +485,7 @@ namespace Wabbajack.Test Consts.TestMode = true; } + /* Disabled, will be removed in the future [Fact] public async Task BethesdaNetDownload() { @@ -510,7 +511,7 @@ namespace Wabbajack.Test using var archive = new ZipArchive(fs); var entries = archive.Entries.Select(e => e.FullName).ToList(); Assert.Equal(entries, new List {@"Data\TestCK.esp", @"Data\TestCK.ini"}); - } + }*/ /* [Fact] From d79277af7485ec13e7c7a5eceb2b39be71a5bf56 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 8 Sep 2020 20:34:53 -0600 Subject: [PATCH 11/13] Fix file extractor tests --- .../FileExtractorTests.cs | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs index 1be835f1..eb261159 100644 --- a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs @@ -6,11 +6,30 @@ using Wabbajack.Common; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.NexusApi; using Xunit; +using Xunit.Abstractions; namespace Wabbajack.VirtualFileSystem.Test { - public class FileExtractorTests + public class FileExtractorTests : IAsyncLifetime { + private ITestOutputHelper _helper; + private IDisposable _unsub; + + public FileExtractorTests(ITestOutputHelper helper) + { + _helper = helper; + _unsub = Utils.LogMessages.Subscribe(f => _helper.WriteLine(f.ShortDescription)); + } + + public async Task InitializeAsync() + { + } + + public async Task DisposeAsync() + { + _unsub.Dispose(); + } + [Fact] public async Task CanGatherDataFromZipFiles() { @@ -82,6 +101,7 @@ namespace Wabbajack.VirtualFileSystem.Test private static AbsolutePath _stagingFolder = ((RelativePath)"NexusDownloads").RelativeToEntryPoint(); + private static async Task DownloadMod(Game game, int mod) { using var client = await NexusApiClient.Get(); From 4c6f34cae53ff17e06680751b31c99de5e31004f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 8 Sep 2020 22:06:15 -0600 Subject: [PATCH 12/13] Supress log errors in testing --- .../FileExtractorTests.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs index eb261159..607410ab 100644 --- a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs @@ -18,7 +18,17 @@ namespace Wabbajack.VirtualFileSystem.Test public FileExtractorTests(ITestOutputHelper helper) { _helper = helper; - _unsub = Utils.LogMessages.Subscribe(f => _helper.WriteLine(f.ShortDescription)); + _unsub = Utils.LogMessages.Subscribe(f => + { + try + { + _helper.WriteLine(f.ShortDescription); + } + catch (Exception _) + { + // ignored + } + }); } public async Task InitializeAsync() From 2332af1252d2f01dd90d85fb2690d15e4c042f64 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 9 Sep 2020 05:28:34 -0600 Subject: [PATCH 13/13] Supress log errors in testing --- .../VirtualFileSystemTests.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs index b06dee3c..a69cc2bb 100644 --- a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs @@ -19,12 +19,23 @@ namespace Wabbajack.VirtualFileSystem.Test private Context context; private readonly ITestOutputHelper _helper; + private IDisposable _unsub; private WorkQueue Queue { get; } = new WorkQueue(); public VFSTests(ITestOutputHelper helper) { _helper = helper; - Utils.LogMessages.Subscribe(f => _helper.WriteLine(f.ShortDescription)); + _unsub = Utils.LogMessages.Subscribe(f => + { + try + { + _helper.WriteLine(f.ShortDescription); + } + catch (Exception ex) + { + // ignored + } + }); context = new Context(Queue); } @@ -36,6 +47,7 @@ namespace Wabbajack.VirtualFileSystem.Test public async Task DisposeAsync() { + _unsub.Dispose(); await VFS_TEST_DIR.DeleteDirectory(); }