Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

EndToEnd Tests pass

Browse Source
This commit is contained in:
Timothy Baldridge 2020-09-05 21:19:05 -06:00
parent 1ebb0b6492
commit 3bdab577e1
9 changed files with 91 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.Common/StatusFileStream.cs
View File

@ -17,7 +17,7 @@ namespace Wabbajack.Common
_inner = fs;
_message = message;
_lastUpdate = DateTime.UnixEpoch;
_span = TimeSpan.FromMilliseconds(500);
_span = TimeSpan.FromMilliseconds(100);
}
public override void Flush()

1
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -18,6 +18,7 @@
<Version>2.2.1.2</Version>
<AssemblyVersion>2.2.1.2</AssemblyVersion>
<FileVersion>2.2.1.2</FileVersion>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<DocumentationFile>Wabbajack.Common.xml</DocumentationFile>

1
Wabbajack.Lib/ACompiler.cs
View File

@ -315,6 +315,7 @@ namespace Wabbajack.Lib
.GroupBy(f => f.SourceDataFile)
.ToDictionary(f => f.Key);
if (grouped.Count == 0) return;
await VFS.Extract(Queue, grouped.Keys.ToHashSet(), async (vf, sfn) =>
{
await using var stream = await sfn.GetStream();

8
Wabbajack.Test/EndToEndTests.cs
View File

@ -110,9 +110,8 @@ namespace Wabbajack.Test
var destFile = utils.DownloadsFolder.Combine(filename);
await src.CopyToAsync(destFile);
await using var dest = await FileExtractor.ExtractAll(Queue, src);
var modFolder = modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName);
await dest.MoveAllTo(modFolder);
await FileExtractor2.ExtractAll(src, modFolder);
return (destFile, modFolder);
}
@ -147,9 +146,8 @@ namespace Wabbajack.Test
await src.CopyToAsync(dest);
var modFolder = utils.ModsFolder.Combine(modName);
await using var files = await FileExtractor.ExtractAll(Queue, src);
await files.MoveAllTo(modFolder);
await FileExtractor2.ExtractAll(src, modFolder);
await dest.WithExtension(Consts.MetaFileExtension).WriteAllTextAsync(ini);
return (dest, modFolder);
}

46
Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs
View File

@ -3,6 +3,9 @@ using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Compression.BSA;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using SharpCompress.Archives.SevenZip;
using SharpCompress.Readers;
using Wabbajack.Common;
using Wabbajack.Common.FileSignatures;
using Wabbajack.VirtualFileSystem.SevenZipExtractor;
@ -16,7 +19,8 @@ namespace Wabbajack.VirtualFileSystem
Definitions.FileType.BA2,
Definitions.FileType.ZIP,
//Definitions.FileType.EXE,
Definitions.FileType.RAR,
Definitions.FileType.RAR_OLD,
Definitions.FileType.RAR_NEW,
Definitions.FileType._7Z);
@ -29,6 +33,9 @@ namespace Wabbajack.VirtualFileSystem
switch (sig)
{
case Definitions.FileType.RAR_OLD:
case Definitions.FileType.RAR_NEW:
case Definitions.FileType._7Z:
case Definitions.FileType.ZIP:
return await GatheringExtractWith7Zip<T>(archive, (Definitions.FileType)sig, shouldExtract, mapfn);
@ -62,6 +69,43 @@ namespace Wabbajack.VirtualFileSystem
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
{
return await new GatheringExtractor<T>(stream, sig, shouldExtract, mapfn).Extract();
/*
IReader reader;
if (sig == Definitions.FileType._7Z)
reader = SevenZipArchive.Open(stream).ExtractAllEntries();
else
{
reader = ReaderFactory.Open(stream);
}
var results = new Dictionary<RelativePath, T>();
while (reader.MoveToNextEntry())
{
var path = (RelativePath)reader.Entry.Key;
if (!reader.Entry.IsDirectory && shouldExtract(path))
{
var ms = new MemoryStream();
reader.WriteEntryTo(ms);
ms.Position = 0;
var result = await mapfn(path, new MemoryStreamFactory(ms));
results.Add(path, result);
}
}
return results;
*/
}
public static async Task ExtractAll(AbsolutePath src, AbsolutePath dest)
{
await GatheringExtract(new NativeFileStreamFactory(src), _ => true, async (path, factory) =>
{
var abs = path.RelativeTo(dest);
abs.Parent.CreateDirectory();
await using var stream = await factory.GetStream();
await abs.WriteAllAsync(stream);
return 0;
});
}
}
}

41
Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs
View File

@ -5,6 +5,7 @@ using System.Linq;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Compression.BSA;
using Wabbajack.Common;
using Wabbajack.Common.FileSignatures;
using Wabbajack.VirtualFileSystem.SevenZipExtractor;
@ -17,7 +18,7 @@ namespace Wabbajack.VirtualFileSystem
private Predicate<RelativePath> _shouldExtract;
private Func<RelativePath, IStreamFactory, ValueTask<T>> _mapFn;
private Dictionary<RelativePath, T> _results;
private Dictionary<uint, RelativePath> _indexes;
private Dictionary<uint, (RelativePath, ulong)> _indexes;
private Stream _stream;
private Definitions.FileType _sig;
@ -41,10 +42,11 @@ namespace Wabbajack.VirtualFileSystem
{
_archive = ArchiveFile.Open(_stream, _sig).Result;
_indexes = _archive.Entries
.Where(f => !f.IsFolder)
.Select((entry, idx) => ((RelativePath)entry.FileName, (uint)idx))
.Select((entry, idx) => (entry, (uint)idx))
.Where(f => !f.entry.IsFolder)
.Select(t => ((RelativePath)t.entry.FileName, t.Item2, t.entry.Size))
.Where(t => _shouldExtract(t.Item1))
.ToDictionary(t => t.Item2, t => t.Item1);
.ToDictionary(t => t.Item2, t => (t.Item1, t.Size));
_archive._archive.Extract(null, 0xFFFFFFFF, 0, this);
@ -101,27 +103,52 @@ namespace Wabbajack.VirtualFileSystem
{
private GatheringExtractor<T> _extractor;
private uint _index;
private bool _written;
private ulong _totalSize;
private MemoryStream _tmpStream;
public GatheringExtractorStream(GatheringExtractor<T> extractor, uint index)
{
_extractor = extractor;
_index = index;
_written = false;
_totalSize = extractor._indexes[index].Item2;
_tmpStream = new MemoryStream();
}
public int Write(IntPtr data, uint size, IntPtr processedSize)
{
unsafe
{
var result = _extractor._mapFn(_extractor._indexes[_index], new UnmanagedStreamFactory((byte*)data, size)).AsTask().Result;
var ums = new UnmanagedMemoryStream((byte*)data, size);
ums.CopyTo(_tmpStream);
if ((ulong)_tmpStream.Length >= _totalSize)
{
_tmpStream.Position = 0;
var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryStreamFactory(_tmpStream)).AsTask().Result;
_extractor._results[_extractor._indexes[_index]] = result;
_extractor._results[_extractor._indexes[_index].Item1] = result;
}
if (processedSize != IntPtr.Zero)
{
Marshal.WriteInt32(processedSize, (int) size);
}
}
return 0;
if (_written) throw new Exception("TODO");
unsafe
{
_written = true;
return 0;
}
}

4
Wabbajack.VirtualFileSystem/SevenZipExtractor/Formats.cs
View File

@ -91,8 +91,8 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
{
{Definitions.FileType._7Z, new Guid("23170f69-40c1-278a-1000-000110070000")},
{Definitions.FileType.BZ2, new Guid("23170f69-40c1-278a-1000-000110020000")},
{Definitions.FileType.RAR_NEW, new Guid("23170f69-40c1-278a-1000-000110030000")},
{Definitions.FileType.RAR_OLD, new Guid("23170f69-40c1-278a-1000-000110CC0000")},
{Definitions.FileType.RAR_OLD, new Guid("23170f69-40c1-278a-1000-000110030000")},
{Definitions.FileType.RAR_NEW, new Guid("23170f69-40c1-278a-1000-000110CC0000")},
{Definitions.FileType.ZIP, new Guid("23170f69-40c1-278a-1000-000110010000")},
};

4
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -5,6 +5,7 @@ using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using K4os.Hash.Crc;
using RocksDbSharp;
using Wabbajack.Common;
@ -198,8 +199,9 @@ namespace Wabbajack.VirtualFileSystem
stream.Position = 0;
var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);
stream.Position = 0;
if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
if (sig.HasValue && TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
return vself;
var self = new VirtualFile

1
Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj
View File

@ -17,6 +17,7 @@
<PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
<PackageReference Include="K4os.Hash.Crc" Version="1.1.4" />
<PackageReference Include="OMODFramework" Version="2.0.1" />
<PackageReference Include="SharpCompress" Version="0.26.0" />
<PackageReference Include="System.Collections.Immutable" Version="5.0.0-preview.6.20305.6" />
</ItemGroup>
<ItemGroup>