Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix TSO extraction issues

Browse Source
This commit is contained in:
Timothy Baldridge 2020-10-05 16:12:21 -06:00
parent 21e308687f
commit 8047bc3caa
7 changed files with 117 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.Lib/MO2Installer.cs
View File

@ -325,6 +325,8 @@ namespace Wabbajack.Lib
await a.Build(OutputFolder.Combine(bsa.To));
streams.Do(s => s.Dispose());
await sourceDir.DeleteDirectory();
if (UseCompression)
await OutputFolder.Combine(bsa.To).Compact(FileCompaction.Algorithm.XPRESS16K);
}

19
Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs
View File

@ -95,6 +95,25 @@ namespace Wabbajack.VirtualFileSystem.Test
}
}
[Fact]
public async Task MissingFileFromArchiveShouldBeFound()
{
FileExtractor2.FavorPerfOverRAM = true;
// From a bug in 2.3.0.1
var src = await DownloadMod(Game.SkyrimSpecialEdition, 21166, 136259);
var results = await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(src),
f => true,
async (p, s) =>
{
await using var stream = await s.GetStream();
return stream.Length;
});
Assert.NotEmpty(results);
}
[Fact]
public async Task CanExtractEmptyFiles()

BIN
Wabbajack.VirtualFileSystem/Extractors/7z.dll
View File

Binary file not shown.

BIN
Wabbajack.VirtualFileSystem/Extractors/7z.exe
View File

Binary file not shown.

6
Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs
View File

@ -59,7 +59,7 @@ namespace Wabbajack.VirtualFileSystem
}
else
{
return await GatheringExtractWith7Zip<T>(archive, (Definitions.FileType)sig, shouldExtract,
return await GatheringExtractWith7Zip<T>(sFn, (Definitions.FileType)sig, shouldExtract,
mapfn);
}
}
@ -139,9 +139,9 @@ namespace Wabbajack.VirtualFileSystem
return results;
}
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(IStreamFactory sf, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
{
return await new GatheringExtractor<T>(stream, sig, shouldExtract, mapfn).Extract();
return await new GatheringExtractor<T>(sf, sig, shouldExtract, mapfn).Extract();
}
public static async Task ExtractAll(AbsolutePath src, AbsolutePath dest)

99
Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs
View File

@ -2,12 +2,14 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reactive.Linq;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Compression.BSA;
using Wabbajack.Common;
using Wabbajack.Common.FileSignatures;
using Wabbajack.Common.StatusFeed.Errors;
using Wabbajack.VirtualFileSystem.SevenZipExtractor;
namespace Wabbajack.VirtualFileSystem
@ -18,17 +20,17 @@ namespace Wabbajack.VirtualFileSystem
private Predicate<RelativePath> _shouldExtract;
private Func<RelativePath, IStreamFactory, ValueTask<T>> _mapFn;
private Dictionary<RelativePath, T> _results;
private Stream _stream;
private Definitions.FileType _sig;
private Exception _killException;
private uint _itemsCount;
private IStreamFactory _streamFactory;
public GatheringExtractor(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory, ValueTask<T>> mapfn)
public GatheringExtractor(IStreamFactory sF, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory, ValueTask<T>> mapfn)
{
_shouldExtract = shouldExtract;
_mapFn = mapfn;
_results = new Dictionary<RelativePath, T>();
_stream = stream;
_streamFactory = sF;
_sig = sig;
}
@ -40,11 +42,18 @@ namespace Wabbajack.VirtualFileSystem
{
try
{
_archive = ArchiveFile.Open(_stream, _sig).Result;
ulong checkPos = 1024 * 32;
_archive._archive.Open(_archive._archiveStream, ref checkPos, new ArchiveCallback());
using var stream = _streamFactory.GetStream().Result;
_archive = ArchiveFile.Open(stream, _sig).Result;
ulong checkPos = (ulong)stream.Length;
var oresult = _archive._archive.Open(_archive._archiveStream, ref checkPos, new ArchiveCallback());
// Can't read this with the COM interface for some reason
if (oresult != 0)
{
var _ = ExtractSlow(source, _streamFactory);
return;
}
_itemsCount = _archive._archive.GetNumberOfItems();
_archive._archive.Extract(null, 0xFFFFFFFF, 0, this);
var result = _archive._archive.Extract(null, 0xFFFFFFFF, 0, this);
_archive.Dispose();
if (_killException != null)
{
@ -69,6 +78,82 @@ namespace Wabbajack.VirtualFileSystem
return _results;
}
private async Task ExtractSlow(TaskCompletionSource<bool> tcs, IStreamFactory streamFactory)
{
try
{
TempFile tempFile = null;
AbsolutePath source;
if (streamFactory is NativeFileStreamFactory nsf)
{
source = (AbsolutePath)nsf.Name;
}
else
{
await using var stream = await streamFactory.GetStream();
tempFile = new TempFile();
await tempFile.Path.WriteAllAsync(stream);
}
var dest = await TempFolder.Create();
Utils.Log(
$"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe");
var process = new ProcessHelper {Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),};
process.Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, "-mmt=off"};
var _ = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p =>
{
var (_, line) = p;
if (line == null)
return;
if (line.Length <= 4 || line[3] != '%') return;
int.TryParse(line.Substring(0, 3), out var percentInt);
Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}",
Percent.FactoryPutInRange(percentInt / 100d));
});
var exitCode = await process.Start();
if (exitCode != 0)
{
Utils.ErrorThrow(new _7zipReturnError(exitCode, source, dest.Dir, ""));
}
else
{
Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true);
}
if (tempFile != null)
{
await tempFile.DisposeAsync();
}
foreach (var file in dest.Dir.EnumerateFiles())
{
var relPath = file.RelativeTo(dest.Dir);
if (!_shouldExtract(relPath)) continue;
var result = await _mapFn(relPath, new NativeFileStreamFactory(file));
_results[relPath] = result;
await file.DeleteAsync();
}
tcs.SetResult(true);
}
catch (Exception ex)
{
tcs.SetException(ex);
}
}
public void SetTotal(ulong total)
{

33
Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs
View File

@ -29,6 +29,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
if (!FileExtractor2.FavorPerfOverRAM)
{
self.SetCompressionProperties(new Dictionary<string, string>() {{"mt", "off"}});
}
@ -130,38 +131,6 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
}
}
public IList<Entry> Entries
{
get
{
if (this._entries != null)
{
return this._entries;
}
ulong checkPos = 32 * 1024;
int open = this._archive.Open(this._archiveStream, ref checkPos, null);
if (open != 0)
{
throw new Exception("Unable to open archive");
}
uint itemsCount = this._archive.GetNumberOfItems();
this._entries = new List<Entry>();
for (uint fileIndex = 0; fileIndex < itemsCount; fileIndex++)
{
var entry = GetEntry(fileIndex);
this._entries.Add(entry);
}
return this._entries;
}
}
internal Entry GetEntry(uint fileIndex)
{
string fileName = this.GetProperty<string>(fileIndex, ItemPropId.kpidPath);