Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Patching and extraction works, fewer failing tests

Browse Source
This commit is contained in:
Timothy Baldridge 2020-09-05 08:01:32 -06:00
parent 6602f1a895
commit a847d69851
27 changed files with 320 additions and 434 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
Compression.BSA/BA2Reader.cs
View File

@ -27,7 +27,6 @@ namespace Compression.BSA
public class BA2Reader : IBSAReader
{
internal AbsolutePath _filename;
private Stream _stream;
internal BinaryReader _rdr;
internal uint _version;
@ -35,15 +34,16 @@ namespace Compression.BSA
internal EntryType _type;
internal uint _numFiles;
internal ulong _nameTableOffset;
public IStreamFactory _streamFactory;
public bool UseATIFourCC { get; set; } = false;
public bool HasNameTable => _nameTableOffset > 0;
public static async Task<BA2Reader> Load(AbsolutePath filename)
public static async Task<BA2Reader> Load(IStreamFactory streamFactory)
{
var rdr = new BA2Reader(await filename.OpenShared()) {_filename = filename};
var rdr = new BA2Reader(await streamFactory.GetStream()) {_streamFactory = streamFactory};
await rdr.LoadHeaders();
return rdr;
}
@ -206,7 +206,7 @@ namespace Compression.BSA
WriteHeader(bw);
await using var fs = await _bsa._filename.OpenRead();
await using var fs = await _bsa._streamFactory.GetStream();
using var br = new BinaryReader(fs);
foreach (var chunk in _chunks)
{
@ -344,6 +344,14 @@ namespace Compression.BSA
break;
}
}
public async ValueTask<IStreamFactory> GetStreamFactory()
{
var ms = new MemoryStream();
await CopyDataTo(ms);
ms.Position = 0;
return new MemoryStreamFactory(ms);
}
}
[JsonName("BA2DX10Entry")]
@ -483,7 +491,7 @@ namespace Compression.BSA
public async ValueTask CopyDataTo(Stream output)
{
await using var fs = await _bsa._filename.OpenRead();
await using var fs = await _bsa._streamFactory.GetStream();
fs.Seek((long) _offset, SeekOrigin.Begin);
uint len = Compressed ? _size : _realSize;
@ -503,6 +511,14 @@ namespace Compression.BSA
await output.WriteAsync(uncompressed, 0, uncompressed.Length);
}
}
public async ValueTask<IStreamFactory> GetStreamFactory()
{
var ms = new MemoryStream();
await CopyDataTo(ms);
ms.Position = 0;
return new MemoryStreamFactory(ms);
}
}
[JsonName("BA2FileEntryState")]

14
Compression.BSA/BSA/Reader/BSAReader.cs
View File

@ -16,7 +16,6 @@ namespace Compression.BSA
public const int HeaderLength = 0x24;
internal uint _fileCount;
internal AbsolutePath _fileName;
internal uint _folderCount;
internal uint _folderRecordOffset;
private Lazy<FolderRecord[]> _folders = null!;
@ -24,6 +23,7 @@ namespace Compression.BSA
internal string _magic = string.Empty;
internal uint _totalFileNameLength;
internal uint _totalFolderNameLength;
public IStreamFactory _streamFactory = new NativeFileStreamFactory(default);
public VersionType HeaderType { get; private set; }
@ -56,7 +56,7 @@ namespace Compression.BSA
public void Dump(Action<string> print)
{
print($"File Name: {_fileName}");
print($"File Name: {_streamFactory.Name}");
print($"File Count: {_fileCount}");
print($"Magic: {_magic}");
@ -67,11 +67,11 @@ namespace Compression.BSA
}
}
public static async ValueTask<BSAReader> LoadAsync(AbsolutePath filename)
public static async ValueTask<BSAReader> LoadAsync(IStreamFactory factory)
{
using var stream = await filename.OpenRead().ConfigureAwait(false);
await using var stream = await factory.GetStream().ConfigureAwait(false);
using var br = new BinaryReader(stream);
var bsa = new BSAReader { _fileName = filename };
var bsa = new BSAReader { _streamFactory = factory };
bsa.LoadHeaders(br);
return bsa;
}
@ -79,7 +79,7 @@ namespace Compression.BSA
public static BSAReader Load(AbsolutePath filename)
{
var bsa = new BSAReader { _fileName = filename };
var bsa = new BSAReader { _streamFactory = new NativeFileStreamFactory(filename)};
using var rdr = bsa.GetStream();
bsa.LoadHeaders(rdr);
return bsa;
@ -87,7 +87,7 @@ namespace Compression.BSA
internal BinaryReader GetStream()
{
return new BinaryReader(File.Open(_fileName.ToString(), FileMode.Open, FileAccess.Read, FileShare.Read));
return new BinaryReader(_streamFactory.GetStream().Result);
}
private void LoadHeaders(BinaryReader rdr)

10
Compression.BSA/BSA/Reader/FileRecord.cs
View File

@ -74,7 +74,7 @@ namespace Compression.BSA
public async ValueTask CopyDataTo(Stream output)
{
await using var in_file = await BSA._fileName.OpenRead().ConfigureAwait(false);
await using var in_file = await BSA._streamFactory.GetStream().ConfigureAwait(false);
using var rdr = new BinaryReader(in_file);
rdr.BaseStream.Position = Offset;
@ -165,5 +165,13 @@ namespace Compression.BSA
print($"Raw Size: {RawSize}");
print($"Index: {_index}");
}
public async ValueTask<IStreamFactory> GetStreamFactory()
{
var ms = new MemoryStream();
await CopyDataTo(ms);
ms.Position = 0;
return new MemoryStreamFactory(ms);
}
}
}

26
Compression.BSA/BSADispatch.cs
View File

@ -1,4 +1,6 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading.Tasks;
@ -13,9 +15,9 @@ namespace Compression.BSA
{
return await BSASignatures.MatchesAsync(filename) switch
{
Definitions.FileType.TES3 => await TES3Reader.Load(filename),
Definitions.FileType.BSA => await BSAReader.LoadAsync(filename),
Definitions.FileType.BA2 => await BA2Reader.Load(filename),
Definitions.FileType.TES3 => await TES3Reader.Load(new NativeFileStreamFactory(filename)),
Definitions.FileType.BSA => await BSAReader.LoadAsync(new NativeFileStreamFactory(filename)),
Definitions.FileType.BA2 => await BA2Reader.Load(new NativeFileStreamFactory(filename)),
_ => throw new InvalidDataException("Filename is not a .bsa or .ba2")
};
}
@ -25,5 +27,21 @@ namespace Compression.BSA
{
return await BSASignatures.MatchesAsync(filename) != null;
}
public static async ValueTask<IBSAReader> OpenRead(IStreamFactory sFn, Definitions.FileType sig)
{
switch(sig)
{
case Definitions.FileType.TES3:
return await TES3Reader.Load(sFn);
case Definitions.FileType.BSA:
return await BSAReader.LoadAsync(sFn);
case Definitions.FileType.BA2:
return await BA2Reader.Load(sFn);
default:
throw new Exception($"Bad archive format for {sFn.Name}");
}
}
}
}

1
Compression.BSA/Interfaces/IFile.cs
View File

@ -32,5 +32,6 @@ namespace Compression.BSA
ValueTask CopyDataTo(Stream output);
void Dump(Action<string> print);
ValueTask<IStreamFactory> GetStreamFactory();
}
}

24
Compression.BSA/MemoryStreamFactory.cs Normal file
View File

@ -0,0 +1,24 @@
using System;
using System.IO;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Compression.BSA
{
public class MemoryStreamFactory : IStreamFactory
{
private readonly MemoryStream _data;
public MemoryStreamFactory(MemoryStream data)
{
_data = data;
}
public async ValueTask<Stream> GetStream()
{
return new MemoryStream(_data.GetBuffer(), 0, (int)_data.Length);
}
public DateTime LastModifiedUtc => DateTime.UtcNow;
public IPath Name => (RelativePath)"BSA Memory Stream";
}
}

82
Compression.BSA/StreamView.cs Normal file
View File

@ -0,0 +1,82 @@
using System;
using System.IO;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Compression.BSA
{
public class StreamView : Stream
{
private Stream _base;
private long _startPos;
private long _length;
public StreamView(Stream baseStream, long startPos, long length)
{
_base = baseStream;
_startPos = startPos;
_length = length;
}
public override void Flush()
{
throw new System.NotImplementedException();
}
public override int Read(byte[] buffer, int offset, int count)
{
var realCount = Math.Min(count, Length - Position);
return _base.Read(buffer, offset, (int)realCount);
}
public override long Seek(long offset, SeekOrigin origin)
{
switch (origin)
{
case SeekOrigin.Begin:
Position = offset;
return Position;
case SeekOrigin.End:
Position = _length - offset;
return Position;
case SeekOrigin.Current:
Position += offset;
return Position;
default:
throw new ArgumentOutOfRangeException(nameof(origin), origin, null);
}
}
public override void SetLength(long value)
{
throw new System.NotImplementedException();
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new System.NotImplementedException();
}
public override bool CanRead => true;
public override bool CanSeek => true;
public override bool CanWrite => false;
public override long Length => _length;
public override long Position
{
get
{
return _base.Position - _startPos;
}
set
{
_base.Position = _startPos + value;
}
}
public override async ValueTask DisposeAsync()
{
await _base.DisposeAsync();
}
}
}

21
Compression.BSA/TES3Reader.cs
View File

@ -17,15 +17,15 @@ namespace Compression.BSA
private uint _fileCount;
private TES3FileEntry[] _files;
internal long _dataOffset;
internal AbsolutePath _filename;
public IStreamFactory _streamFactory;
public static async ValueTask<TES3Reader> Load(AbsolutePath filename)
public static async ValueTask<TES3Reader> Load(IStreamFactory factory)
{
await using var fs = await filename.OpenRead();
await using var fs = await factory.GetStream();
using var br = new BinaryReader(fs);
var rdr = new TES3Reader
{
_filename = filename,
_streamFactory = factory,
_versionNumber = br.ReadUInt32(),
_hashTableOffset = br.ReadUInt32(),
_fileCount = br.ReadUInt32()
@ -125,16 +125,27 @@ namespace Compression.BSA
public async ValueTask CopyDataTo(Stream output)
{
await using var fs = await Archive._filename.OpenRead();
await using var fs = await Archive._streamFactory.GetStream();
fs.Position = Archive._dataOffset + Offset;
await fs.CopyToLimitAsync(output, (int)Size);
}
public async ValueTask<IStreamFactory> GetStreamFactory()
{
var ms = new MemoryStream();
await CopyDataTo(ms);
ms.Position = 0;
return new MemoryStreamFactory(ms);
}
public void Dump(Action<string> print)
{
throw new NotImplementedException();
}
public uint Offset { get; set; }
public uint NameOffset { get; set; }
public uint Hash1 { get; set; }

34
Wabbajack.Common/IStreamFactory.cs Normal file
View File

@ -0,0 +1,34 @@
using System;
using System.IO;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.Common
{
public interface IStreamFactory
{
ValueTask<Stream> GetStream();
DateTime LastModifiedUtc { get; }
IPath Name { get; }
}
public class NativeFileStreamFactory : IStreamFactory
{
private AbsolutePath _file;
public NativeFileStreamFactory(AbsolutePath file)
{
_file = file;
}
public async ValueTask<Stream> GetStream()
{
return await _file.OpenRead();
}
public DateTime LastModifiedUtc => _file.LastModifiedUtc;
public IPath Name => _file;
}
}

4
Wabbajack.Common/Patches.cs
View File

@ -50,7 +50,7 @@ namespace Wabbajack.Common
await patch.CopyToAsync(output);
}
public static async Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash,
public static async Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash,
Stream? patchOutStream = null)
{
var key = PatchKey(srcHash, destHash);
@ -128,7 +128,7 @@ namespace Wabbajack.Common
public static Task CreatePatchCached(byte[] a, byte[] b, Stream output) =>
PatchCache.CreatePatchCached(a, b, output);
public static Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash, Stream? patchOutStream = null) =>
public static Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, Stream? patchOutStream = null) =>
PatchCache.CreatePatchCached(srcStream, srcHash, destStream, destHash, patchOutStream);
public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch) =>

47
Wabbajack.Lib/AInstaller.cs
View File

@ -9,6 +9,7 @@ using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
using Wabbajack.VirtualFileSystem.SevenZipExtractor;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
@ -35,6 +36,8 @@ namespace Wabbajack.Lib
public bool UseCompression { get; set; }
public TempFolder? ExtractedModlistFolder { get; set; } = null;
public AInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters? parameters, int steps, Game game)
: base(steps)
@ -45,12 +48,23 @@ namespace Wabbajack.Lib
DownloadFolder = downloadFolder;
SystemParameters = parameters;
Game = game.MetaData();
}
private ExtractedFiles? ExtractedModListFiles { get; set; } = null;
public async Task ExtractModlist()
{
ExtractedModListFiles = await FileExtractor.ExtractAll(Queue, ModListArchive);
ExtractedModlistFolder = await TempFolder.Create();
await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(ModListArchive), _ => true,
async (path, sfn) =>
{
await using var s = await sfn.GetStream();
var fp = ExtractedModlistFolder.Dir.Combine(path);
fp.Parent.CreateDirectory();
await fp.WriteAllAsync(s);
return 0;
});
}
@ -73,8 +87,7 @@ namespace Wabbajack.Lib
public async Task<byte[]> LoadBytesFromPath(RelativePath path)
{
await using var e = await ExtractedModListFiles![path].OpenRead();
return await e.ReadAllAsync();
return await ExtractedModlistFolder!.Dir.Combine(path).ReadAllBytesAsync();
}
public static ModList LoadFromFile(AbsolutePath path)
@ -113,23 +126,23 @@ namespace Wabbajack.Lib
public async Task InstallArchives()
{
await VFS.CopyTo(Queue, ModList.Directives
.OfType<FromArchive>()
.Select(a => (VFS.Index.FileForArchiveHashPath(a.ArchiveHashPath), a.To.RelativeTo(OutputFolder))));
/*
Info("Installing Archives");
Info("Grouping Install Files");
var grouped = ModList.Directives
.OfType<FromArchive>()
.GroupBy(e => e.ArchiveHashPath.BaseHash)
.ToDictionary(k => k.Key);
var archives = ModList.Archives
.Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) })
.Where(a => a.AbsolutePath != null)
.ToList();
.Select(a => new {VF = VFS.Index.FileForArchiveHashPath(a.ArchiveHashPath), Directive = a})
.GroupBy(a => a.VF)
.ToDictionary(a => a.Key);
Info("Installing Archives");
await archives.PMap(Queue, UpdateTracker,a => InstallArchive(Queue, a.Archive, a.AbsolutePath, grouped[a.Archive.Hash]));*/
if (grouped.Count == 0) return;
await VFS.Extract(Queue, grouped.Keys.ToHashSet(), async (vf, sf) =>
{
await using var s = await sf.GetStream();
foreach (var directive in grouped[vf])
{
s.Position = 0;
await directive.Directive.To.RelativeTo(OutputFolder).WriteAllAsync(s, false);
}
});
}
private async Task InstallArchive(WorkQueue queue, Archive archive, AbsolutePath absolutePath, IGrouping<Hash, FromArchive> grouping)

76
Wabbajack.Lib/MO2Compiler.cs
View File

@ -6,10 +6,12 @@ using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AngleSharp.Common;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.Validation;
using Wabbajack.VirtualFileSystem;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Lib
@ -495,17 +497,35 @@ namespace Wabbajack.Lib
.ToArray();
if (toBuild.Length == 0) return;
// Extract all the source files
var indexed = toBuild.GroupBy(f => (VFS.Index.FileForArchiveHashPath(f.ArchiveHashPath)))
.ToDictionary(f => f.Key);
await VFS.Extract(Queue, indexed.Keys.ToHashSet(),
async (vf, sf) =>
{
// For each, extract the destination
var matches = indexed[vf];
using var iqueue = new WorkQueue(1);
foreach (var match in matches)
{
var destFile = FindDestFile(match.To);
// Build the patch
await VFS.Extract(iqueue, new[] {destFile}.ToHashSet(),
async (destvf, destsfn) =>
{
Info($"Patching {match.To}");
Status($"Patching {match.To}");
await using var srcStream = await sf.GetStream();
await using var destStream = await destsfn.GetStream();
var patchSize = await Utils.CreatePatchCached(srcStream, vf.Hash, destStream, destvf.Hash);
Info($"Patch size {patchSize} for {match.To}");
});
}
});
var groups = toBuild
.Where(p => p.PatchID == default)
.GroupBy(p => p.ArchiveHashPath.BaseHash)
.ToList();
Info($"Patching building patches from {groups.Count} archives");
var absolutePaths = AllFiles.ToDictionary(e => e.Path, e => e.AbsolutePath);
await groups.PMap(Queue, group => BuildArchivePatches(group.Key, group, absolutePaths));
// Load in the patches
await InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => p.PatchID == default)
.PMap(Queue, async pfa =>
@ -514,6 +534,7 @@ namespace Wabbajack.Lib
.Select(c => (Utils.TryGetPatch(c.Hash, pfa.Hash, out var data), data, c))
.ToArray();
// Pick the best patch
if (patches.All(p => p.Item1))
{
var (_, bytes, file) = IncludePatches.PickPatch(this, patches);
@ -529,42 +550,19 @@ namespace Wabbajack.Lib
Error($"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}");
}
private async Task BuildArchivePatches(Hash archiveSha, IEnumerable<PatchedFromArchive> group,
Dictionary<RelativePath, AbsolutePath> absolutePaths)
private VirtualFile FindDestFile(RelativePath to)
{
await using var files = await VFS.StageWith(@group.Select(g => VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath)));
var byPath = files.GroupBy(f => string.Join("|", f.FilesInFullPath.Skip(1).Select(i => i.Name)))
.ToDictionary(f => f.Key, f => f.First());
// Now Create the patches
await @group.PMap(Queue, async entry =>
{
Info($"Patching {entry.To}");
Status($"Patching {entry.To}");
var srcFile = byPath[string.Join("|", entry.ArchiveHashPath.Paths)];
await using var srcStream = await srcFile.OpenRead();
await using var destStream = await LoadDataForTo(entry.To, absolutePaths);
var patchSize = await Utils.CreatePatchCached(srcStream, srcFile.Hash, destStream, entry.Hash);
Info($"Patch size {patchSize} for {entry.To}");
});
}
private async Task<FileStream> LoadDataForTo(RelativePath to, Dictionary<RelativePath, AbsolutePath> absolutePaths)
{
if (absolutePaths.TryGetValue(to, out var absolute))
return await absolute.OpenRead();
var abs = to.RelativeTo(MO2Folder);
if (abs.Exists)
return VFS.Index.ByRootPath[abs];
if (to.StartsWith(Consts.BSACreationDir))
{
var bsaId = (RelativePath)((string)to).Split('\\')[1];
var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsaId);
var a = await BSADispatch.OpenRead(MO2Folder.Combine(bsa.To));
var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray());
var file = a.Files.First(e => e.Path == find);
var returnStream = new TempStream();
await file.CopyDataTo(returnStream);
returnStream.Position = 0;
return returnStream;
return VFS.Index.ByRootPath[MO2Folder.Combine(bsa.To)].Children.First(c => c.RelativeName == find);
}
throw new ArgumentException($"Couldn't load data for {to}");

4
Wabbajack.Lib/MO2Installer.cs
View File

@ -180,6 +180,7 @@ namespace Wabbajack.Lib
SetScreenSizeInPrefs();
UpdateTracker.NextStep("Installation complete! You may exit the program.");
await ExtractedModlistFolder!.DisposeAsync();
await Metrics.Send(Metrics.FinishInstall, ModList.Name);
return true;
@ -243,7 +244,8 @@ namespace Wabbajack.Lib
Status($"Writing included .meta file {directive.To}");
var outPath = DownloadFolder.Combine(directive.To);
if (outPath.IsFile) await outPath.DeleteAsync();
await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID));
var bytes = await LoadBytesFromPath(directive.SourceDataID);
await outPath.WriteAllBytesAsync(bytes);
});
}

8
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -221,7 +221,7 @@ namespace Wabbajack.VirtualFileSystem
r => fileNames.ContainsKey(r),
async (rel, csf) =>
{
await HandleFile(fileNames[rel], sfn);
await HandleFile(fileNames[rel], csf);
return 0;
});
}
@ -261,12 +261,6 @@ namespace Wabbajack.VirtualFileSystem
};
}
public async Task CopyTo(WorkQueue queue, IEnumerable<(VirtualFile src, AbsolutePath dest)> directives)
{
var plans = StagingPlan.StagingPlan.CreatePlan(directives).ToArray();
await StagingPlan.StagingPlan.ExecutePlans(queue, plans);
}
public async Task<AsyncDisposableList<VirtualFile>> StageWith(IEnumerable<VirtualFile> files)
{
return new AsyncDisposableList<VirtualFile>(await Stage(files), files);

1
Wabbajack.VirtualFileSystem/FileExtractor.cs
View File

@ -30,6 +30,7 @@ namespace Wabbajack.VirtualFileSystem
public static async Task<ExtractedFiles> ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable<RelativePath> OnlyFiles = null, bool throwOnError = true)
{
throw new NotImplementedException();
OnlyFiles ??= new RelativePath[0];
try

27
Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs
View File

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Compression.BSA;
using Wabbajack.Common;
using Wabbajack.Common.FileSignatures;
using Wabbajack.VirtualFileSystem.SevenZipExtractor;
@ -14,7 +15,7 @@ namespace Wabbajack.VirtualFileSystem
Definitions.FileType.BSA,
Definitions.FileType.BA2,
Definitions.FileType.ZIP,
Definitions.FileType.EXE,
//Definitions.FileType.EXE,
Definitions.FileType.RAR,
Definitions.FileType._7Z);
@ -31,11 +32,33 @@ namespace Wabbajack.VirtualFileSystem
case Definitions.FileType.ZIP:
return await GatheringExtractWith7Zip<T>(archive, (Definitions.FileType)sig, shouldExtract, mapfn);
case Definitions.FileType.TES3:
case Definitions.FileType.BSA:
case Definitions.FileType.BA2:
return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
default:
throw new Exception("Invalid file format");
throw new Exception($"Invalid file format {sFn.Name}");
}
}
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithBSA<T>(IStreamFactory sFn, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
{
var archive = await BSADispatch.OpenRead(sFn, sig);
var results = new Dictionary<RelativePath, T>();
foreach (var entry in archive.Files)
{
if (!shouldExtract(entry.Path))
continue;
var result = await mapfn(entry.Path, await entry.GetStreamFactory());
results.Add(entry.Path, result);
}
return results;
}
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
{
return await new GatheringExtractor<T>(stream, sig, shouldExtract, mapfn).Extract();

24
Wabbajack.VirtualFileSystem/FileExtractor2/IStreamFactory.cs → Wabbajack.VirtualFileSystem/FileExtractor2/StreamFactories.cs
View File

@ -5,13 +5,6 @@ using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public interface IStreamFactory
{
Task<Stream> GetStream();
DateTime LastModifiedUtc { get; }
}
public class UnmanagedStreamFactory : IStreamFactory
{
@ -23,7 +16,7 @@ namespace Wabbajack.VirtualFileSystem
_data = data;
_size = size;
}
public async Task<Stream> GetStream()
public async ValueTask<Stream> GetStream()
{
unsafe
{
@ -32,21 +25,8 @@ namespace Wabbajack.VirtualFileSystem
}
public DateTime LastModifiedUtc => DateTime.UtcNow;
public IPath Name => (RelativePath)"Unmanaged Memory Stream";
}
public class NativeFileStreamFactory : IStreamFactory
{
private AbsolutePath _file;
public NativeFileStreamFactory(AbsolutePath file)
{
_file = file;
}
public async Task<Stream> GetStream()
{
return await _file.OpenRead();
}
public DateTime LastModifiedUtc => _file.LastModifiedUtc;
}
}

25
Wabbajack.VirtualFileSystem/StagingPlan/ASubStage.cs
View File

@ -1,25 +0,0 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public abstract class ASubStage : ISubStage
{
private IEnumerable<IStagingPlan> _plans;
public ASubStage(IEnumerable<IStagingPlan> plans)
{
_plans = plans;
}
public abstract ValueTask DisposeAsync();
public abstract AbsolutePath Destination { get; }
public abstract IPath Source { get; }
public async Task Execute(WorkQueue queue)
{
await StagingPlan.ExecutePlan(queue, async () => await Destination.OpenWrite(), _plans);
}
}
}

23
Wabbajack.VirtualFileSystem/StagingPlan/CopyTo.cs
View File

@ -1,23 +0,0 @@
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public class CopyTo : IStagingSrc
{
public CopyTo(IPath src, AbsolutePath destination)
{
Destination = destination;
Source = src;
}
public AbsolutePath Destination { get; }
public async ValueTask DisposeAsync()
{
}
public IPath Source { get; }
}
}

26
Wabbajack.VirtualFileSystem/StagingPlan/DuplicateTo.cs
View File

@ -1,26 +0,0 @@
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public class DuplicateTo : IStagingPlan
{
private readonly IStagingPlan _src;
public DuplicateTo(IStagingPlan src, AbsolutePath destination)
{
_src = src;
Destination = destination;
}
public async ValueTask DisposeAsync()
{
}
public async Task Execute()
{
await _src.Destination.CopyToAsync(Destination);
}
public AbsolutePath Destination { get; }
}
}

156
Wabbajack.VirtualFileSystem/StagingPlan/IStagingPlan.cs
View File

@ -1,156 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.FileSignatures;
using Wabbajack.VirtualFileSystem.SevenZipExtractor;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public interface IStagingPlan : IAsyncDisposable
{
AbsolutePath Destination { get; }
}
public static class StagingPlan
{
public static IEnumerable<IStagingPlan> CreatePlan(IEnumerable<(VirtualFile src, AbsolutePath dest)> directives)
{
var top = new VirtualFile();
var original = directives.ToHashSet();
var files = directives.GroupBy(f => f.src).ToDictionary(f => f.Key);
var childrenForParent = files.Keys.GroupBy(f => f.Parent ?? top).ToDictionary(f => f.Key);
var allFilesForParent = files.Select(f => f.Key)
.SelectMany(f => f.FilesInFullPath)
.Distinct()
.GroupBy(f => f.Parent ?? top)
.ToDictionary(f => f.Key, f => f.ToArray());
var baseDirectives = new List<IStagingPlan>();
// Recursive internal function to get the plans for a given parent, we'll then start at the
// null parent (the top) and recurse our way down into all the children files.
IEnumerable<IStagingPlan> GetPlans(VirtualFile parent)
{
foreach (var forParent in allFilesForParent[parent])
{
// Do we need files inside this file?
if (childrenForParent.TryGetValue(forParent, out var children))
{
if (files.TryGetValue(forParent, out var copies))
{
ASubStage subStage;
if (parent == top)
{
subStage = new NativeArchive(forParent.AbsoluteName, GetPlans(forParent));
}
else
{
subStage = new SubStage(forParent.RelativeName, copies.First().dest, GetPlans(forParent));
}
yield return subStage;
foreach (var copy in copies)
{
yield return new DuplicateTo(subStage, copy.dest);
}
}
else
{
if (parent == top)
{
yield return new NativeArchive(forParent.AbsoluteName, GetPlans(forParent));
}
else
{
yield return new TempSubStage(forParent.RelativeName, GetPlans(forParent));
}
}
}
else
{
// If not then we need to copy this file around
var copies = files[forParent];
var firstCopy = new CopyTo(copies.Key.Name, copies.First().dest);
yield return firstCopy;
foreach (var duplicate in copies.Skip(1))
{
yield return new DuplicateTo(firstCopy, duplicate.dest);
}
}
}
}
return GetPlans(top);
}
public static async ValueTask ExecutePlan(WorkQueue queue, Func<ValueTask<Stream>> src, IEnumerable<IStagingPlan> plans)
{
// Extract these files
await using var stream = await src();
var sig = await FileExtractor.ArchiveSigs.MatchesAsync(stream);
stream.Position = 0;
switch (sig)
{
case Definitions.FileType.ZIP:
await ExtractWith7Zip((Definitions.FileType)sig, stream, plans);
break;
default:
throw new Exception($"Invalid archive for extraction");
}
// Copy around the duplicates
foreach (var file in plans.OfType<DuplicateTo>())
{
await file.Execute();
}
// Execute the sub-stages
foreach (var subStage in plans.OfType<ISubStage>())
{
await subStage.Execute(queue);
}
// Dispose of all plans
foreach (var file in plans)
{
await file.DisposeAsync();
}
}
private static async ValueTask ExtractWith7Zip(Definitions.FileType sig, Stream stream, IEnumerable<IStagingPlan> plans)
{
using var archive = await ArchiveFile.Open(stream, sig);
void HandleFile(RelativePath path, IStagingPlan src, Func<Stream> sf)
{
}
var extractor = new TypedExtractor<IStagingPlan>(plans.OfType<IStagingSrc>().ToDictionary(s => s.Source.FileName, s => (IStagingPlan)s),
HandleFile);
extractor.Extract(archive);
}
public static async Task ExecutePlans(WorkQueue queue, IEnumerable<IStagingPlan> plans)
{
foreach (var file in plans.OfType<NativeArchive>())
{
await file.Execute(queue);
}
}
}
}

9
Wabbajack.VirtualFileSystem/StagingPlan/IStagingSrc.cs
View File

@ -1,9 +0,0 @@
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public interface IStagingSrc : IStagingPlan
{
public IPath Source { get; }
}
}

10
Wabbajack.VirtualFileSystem/StagingPlan/ISubStage.cs
View File

@ -1,10 +0,0 @@
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public interface ISubStage : IStagingSrc
{
Task Execute(WorkQueue queue);
}
}

22
Wabbajack.VirtualFileSystem/StagingPlan/NativeArchive.cs
View File

@ -1,22 +0,0 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public class NativeArchive : ASubStage
{
public NativeArchive(AbsolutePath src, IEnumerable<IStagingPlan> plans) : base(plans)
{
Source = src;
Destination = src;
}
public override async ValueTask DisposeAsync()
{
}
public override AbsolutePath Destination { get; }
public override IPath Source { get; }
}
}

21
Wabbajack.VirtualFileSystem/StagingPlan/SubStage.cs
View File

@ -1,21 +0,0 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public class SubStage : ASubStage
{
public SubStage(RelativePath src, AbsolutePath destination, IEnumerable<IStagingPlan> plans) : base(plans)
{
Source = src;
Destination = destination;
}
public override async ValueTask DisposeAsync()
{
}
public override AbsolutePath Destination { get; }
public override IPath Source { get; }
}
}

25
Wabbajack.VirtualFileSystem/StagingPlan/TempSubStage.cs
View File

@ -1,25 +0,0 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem.StagingPlan
{
public class TempSubStage : ASubStage
{
private readonly TempFile _temp;
public TempSubStage(RelativePath src, IEnumerable<IStagingPlan> plans) : base(plans)
{
Source = src;
_temp = new TempFile();
}
public override async ValueTask DisposeAsync()
{
await _temp.DisposeAsync();
}
public override AbsolutePath Destination => _temp.Path;
public override IPath Source { get; }
}
}

8
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -141,7 +141,7 @@ namespace Wabbajack.VirtualFileSystem
itm.ThisAndAllChildrenReduced(fn);
}
private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IExtractedFile extractedFile)
private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IStreamFactory extractedFile)
{
var vself = new VirtualFile
{
@ -161,7 +161,7 @@ namespace Wabbajack.VirtualFileSystem
return vself;
}
private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IExtractedFile extractedFile, Hash hash, out VirtualFile found)
private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile, Hash hash, out VirtualFile found)
{
var result = _vfsCache.Get(hash.ToArray());
if (result == null)
@ -199,11 +199,9 @@ namespace Wabbajack.VirtualFileSystem
var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);
/* TODO
if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
return vself;
*/
var self = new VirtualFile
{
Context = context,