Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

BSA archives are now lazily extracted.

Browse Source 
7Zip extracted archives now only extract the fewest files required.
Audited the uses of .Wait
Lazily init the VFS cleaning
This commit is contained in:
Timothy Baldridge 2020-04-16 21:52:19 -06:00
parent 3219d48b70
commit bb9ef89dee
42 changed files with 373 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Compression.BSA.Test/BSATests.cs
View File

@ -81,7 +81,8 @@ namespace Compression.BSA.Test
var folder = _bsaFolder.Combine(game.ToString(), modid.ToString());
await folder.DeleteDirectory();
folder.CreateDirectory();
await FileExtractor.ExtractAll(Queue, filename, folder);
await using var files = await FileExtractor.ExtractAll(Queue, filename);
await files.MoveAllTo(folder);
foreach (var bsa in folder.EnumerateFiles().Where(f => Consts.SupportedBSAs.Contains(f.Extension)))
{
@ -94,7 +95,7 @@ namespace Compression.BSA.Test
var tempFile = ((RelativePath)"tmp.bsa").RelativeToEntryPoint();
var size = bsa.Size;
using var a = BSADispatch.OpenRead(bsa);
await using var a = BSADispatch.OpenRead(bsa);
await a.Files.PMap(Queue, file =>
{
var absName = _tempDir.Combine(file.Path);
@ -111,7 +112,7 @@ namespace Compression.BSA.Test
TestContext.WriteLine($"Building {bsa}");
using (var w = ViaJson(a.State).MakeBuilder(size))
await using (var w = ViaJson(a.State).MakeBuilder(size))
{
var streams = await a.Files.PMap(Queue, file =>
{
@ -125,7 +126,7 @@ namespace Compression.BSA.Test
}
TestContext.WriteLine($"Verifying {bsa}");
using var b = BSADispatch.OpenRead(tempFile);
await using var b = BSADispatch.OpenRead(tempFile);
TestContext.WriteLine($"Performing A/B tests on {bsa}");
Assert.Equal(a.State.ToJson(), b.State.ToJson());

3
Compression.BSA/BA2Builder.cs
View File

@ -4,6 +4,7 @@ using System.IO;
using System.IO.MemoryMappedFiles;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using Wabbajack.Common;
@ -34,7 +35,7 @@ namespace Compression.BSA
_slab = new DiskSlabAllocator(size);
}
public void Dispose()
public async ValueTask DisposeAsync()
{
_slab.Dispose();
}

3
Compression.BSA/BA2Reader.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
@ -99,7 +100,7 @@ namespace Compression.BSA
}
public void Dispose()
public async ValueTask DisposeAsync()
{
_stream?.Dispose();
_rdr?.Dispose();

3
Compression.BSA/BSABuilder.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using K4os.Compression.LZ4;
using K4os.Compression.LZ4.Streams;
@ -75,7 +76,7 @@ namespace Compression.BSA
public bool HasNameBlobs => (_archiveFlags & 0x100) > 0;
public void Dispose()
public async ValueTask DisposeAsync()
{
_slab.Dispose();
}

5
Compression.BSA/BSAReader.cs
View File

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using K4os.Compression.LZ4.Streams;
using Wabbajack.Common;
@ -49,7 +50,7 @@ namespace Compression.BSA
Miscellaneous = 0x100
}
public class BSAReader : IDisposable, IBSAReader
public class BSAReader : IAsyncDisposable, IBSAReader
{
internal uint _archiveFlags;
internal uint _fileCount;
@ -113,7 +114,7 @@ namespace Compression.BSA
}
}
public void Dispose()
public async ValueTask DisposeAsync()
{
_stream.Close();
}

4
Compression.BSA/IBSAReader.cs
View File

@ -5,7 +5,7 @@ using Wabbajack.Common;
namespace Compression.BSA
{
public interface IBSAReader : IDisposable
public interface IBSAReader : IAsyncDisposable
{
/// <summary>
/// The files defined by the archive
@ -15,7 +15,7 @@ namespace Compression.BSA
ArchiveStateObject State { get; }
}
public interface IBSABuilder : IDisposable
public interface IBSABuilder : IAsyncDisposable
{
void AddFile(FileStateObject state, Stream src);
void Build(AbsolutePath filename);

3
Compression.BSA/TES3Builder.cs
View File

@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using File = Alphaleonis.Win32.Filesystem.File;
@ -71,7 +72,7 @@ namespace Compression.BSA
}
}
public void Dispose()
public async ValueTask DisposeAsync()
{
}
}

3
Compression.BSA/TES3Reader.cs
View File

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
@ -61,7 +62,7 @@ namespace Compression.BSA
_dataOffset = br.BaseStream.Position;
}
public void Dispose()
public async ValueTask DisposeAsync()
{
}

2
Wabbajack.BuildServer/Controllers/ListValidation.cs
View File

@ -117,7 +117,7 @@ namespace Wabbajack.BuildServer.Controllers
private static AsyncLock _findPatchLock = new AsyncLock();
private async Task<(Archive, ArchiveStatus)> TryToFix(SqlService.ValidationData data, Archive archive)
{
using var _ = await _findPatchLock.Wait();
using var _ = await _findPatchLock.WaitAsync();
var result = await _updater.GetAlternative(archive.Hash.ToHex());
return result switch

2
Wabbajack.BuildServer/Controllers/UploadedFiles.cs
View File

@ -63,7 +63,7 @@ namespace Wabbajack.BuildServer.Controllers
Utils.Log($"Writing {ms.Length} at position {Offset} in ingest file {Key}");
long position;
using (var _ = await _writeLocks[Key].Wait())
using (var _ = await _writeLocks[Key].WaitAsync())
{
await using var file = _settings.TempPath.Combine(Key).WriteShared();
file.Position = Offset;

4
Wabbajack.CLI/Verbs/DownloadUrl.cs
View File

@ -32,11 +32,11 @@ namespace Wabbajack.CLI.Verbs
.Debounce(TimeSpan.FromSeconds(1))
.Subscribe(s => Console.WriteLine($"Downloading {s.ProgressPercent}"));
new[] {state}
await new[] {state}
.PMap(queue, async s =>
{
await s.Download(new Archive(state: null!) {Name = Path.GetFileName(Output)}, (AbsolutePath)Output);
}).Wait();
});
File.WriteAllLines(Output + ".meta", state.GetMetaIni());
return 0;

8
Wabbajack.Common.Test/AsyncLockTests.cs
View File

@ -13,7 +13,7 @@ namespace Wabbajack.Common.Test
bool firstRun = false;
var first = Task.Run(async () =>
{
using (await asyncLock.Wait())
using (await asyncLock.WaitAsync())
{
await Task.Delay(500);
firstRun = true;
@ -22,7 +22,7 @@ namespace Wabbajack.Common.Test
var second = Task.Run(async () =>
{
await Task.Delay(200);
using (await asyncLock.Wait())
using (await asyncLock.WaitAsync())
{
Assert.True(firstRun);
}
@ -41,7 +41,7 @@ namespace Wabbajack.Common.Test
{
return Task.Run(async () =>
{
using (await asyncLock.Wait())
using (await asyncLock.WaitAsync())
{
await Task.Delay(500);
firstRun = true;
@ -55,7 +55,7 @@ namespace Wabbajack.Common.Test
Task.Run(async () =>
{
await Task.Delay(200);
using (await asyncLock.Wait())
using (await asyncLock.WaitAsync())
{
Assert.True(firstRun);
secondRun = true;

11
Wabbajack.Common/Hash.cs
View File

@ -145,6 +145,17 @@ namespace Wabbajack.Common
var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
return Hash.FromULong(BitConverter.ToUInt64(value.Hash));
}
public static Hash xxHash(this Stream stream)
{
var hash = new xxHashConfig();
hash.HashSizeInBits = 64;
hash.Seed = 0x42;
var config = new xxHashConfig {HashSizeInBits = 64};
using var f = new StatusFileStream(stream, $"Hashing memory stream");
var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
return Hash.FromULong(BitConverter.ToUInt64(value.Hash));
}
public static Hash FileHashCached(this AbsolutePath file, bool nullOnIoError = false)
{
if (TryGetHashCache(file, out var foundHash)) return foundHash;

4
Wabbajack.Common/OctoDiff.cs
View File

@ -28,7 +28,7 @@ namespace Wabbajack.Common
return sigStream;
}
private static void CreateSignature(FileStream oldData, FileStream sigStream)
private static void CreateSignature(Stream oldData, FileStream sigStream)
{
Utils.Status("Creating Patch Signature");
var signatureBuilder = new SignatureBuilder();
@ -36,7 +36,7 @@ namespace Wabbajack.Common
sigStream.Position = 0;
}
public static void Create(FileStream oldData, FileStream newData, FileStream signature, FileStream output)
public static void Create(Stream oldData, FileStream newData, FileStream signature, FileStream output)
{
CreateSignature(oldData, signature);
var db = new DeltaBuilder {ProgressReporter = reporter};

2
Wabbajack.Common/Paths.cs
View File

@ -9,7 +9,7 @@ using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Directory = System.IO.Directory;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
using Path = Alphaleonis.Win32.Filesystem.Path;

2
Wabbajack.Common/Util/AsyncLock.cs
View File

@ -12,7 +12,7 @@ namespace Wabbajack.Common
{
private readonly SemaphoreSlim _lock = new SemaphoreSlim(1, 1);
public async Task<IDisposable> Wait()
public async Task<IDisposable> WaitAsync()
{
await _lock.WaitAsync();
return Disposable.Create(() => _lock.Release());

2
Wabbajack.Common/Util/TempFolder.cs
View File

@ -14,7 +14,7 @@ namespace Wabbajack.Common
public TempFolder(bool deleteAfter = true)
{
Dir = new AbsolutePath(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()));
Dir = Path.Combine("tmp_files", Guid.NewGuid().ToString()).RelativeTo(AbsolutePath.EntryPoint);
if (!Dir.Exists)
Dir.CreateDirectory();
DeleteAfter = deleteAfter;

11
Wabbajack.Common/Utils.cs
View File

@ -427,6 +427,13 @@ namespace Wabbajack.Common
await ins.CopyToAsync(ms);
return ms.ToArray();
}
public static async Task<string> ReadAllTextAsync(this Stream ins)
{
await using var ms = new MemoryStream();
await ins.CopyToAsync(ms);
return Encoding.UTF8.GetString(ms.ToArray());
}
public static async Task<TR[]> PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue, StatusUpdateTracker updateTracker,
Func<TI, TR> f)
@ -727,7 +734,7 @@ namespace Wabbajack.Common
}
}
public static async Task CreatePatch(FileStream srcStream, Hash srcHash, FileStream destStream, Hash destHash,
public static async Task CreatePatch(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash,
FileStream patchStream)
{
await using var sigFile = new TempStream();
@ -967,7 +974,7 @@ namespace Wabbajack.Common
Status($"Deleting: {p.Line}");
});
process.Start().Wait();
await process.Start();
await result;
}

4
Wabbajack.Common/WorkQueue.cs
View File

@ -93,7 +93,7 @@ namespace Wabbajack.Common
private async Task AddNewThreadsIfNeeded(int desired)
{
using (await _lock.Wait())
using (await _lock.WaitAsync())
{
DesiredNumWorkers = desired;
while (DesiredNumWorkers > _tasks.Count)
@ -141,7 +141,7 @@ namespace Wabbajack.Common
if (DesiredNumWorkers >= _tasks.Count) continue;
// Noticed that we may need to shut down, lock and check again
using (await _lock.Wait())
using (await _lock.WaitAsync())
{
// Check if another thread shut down before this one and got us back to the desired amount already
if (DesiredNumWorkers >= _tasks.Count) continue;

3
Wabbajack.Lib/AInstaller.cs
View File

@ -172,7 +172,8 @@ namespace Wabbajack.Lib
throw new ArgumentNullException("FromFile was null");
}
var firstDest = OutputFolder.Combine(group.First().To);
await CopyFile(group.Key.StagedPath, firstDest, true);
await group.Key.StagedFile.MoveTo(firstDest);
foreach (var copy in group.Skip(1))
{

2
Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs
View File

@ -77,7 +77,7 @@ namespace Wabbajack.Lib.CompilationSteps
}
CreateBSA directive;
using (var bsa = BSADispatch.OpenRead(source.AbsolutePath))
await using (var bsa = BSADispatch.OpenRead(source.AbsolutePath))
{
directive = new CreateBSA(
state: bsa.State,

10
Wabbajack.Lib/Data.cs
View File

@ -21,7 +21,15 @@ namespace Wabbajack.Lib
Path = path;
}
public AbsolutePath AbsolutePath => File.StagedPath;
public AbsolutePath AbsolutePath
{
get
{
if (!File.IsNative)
throw new InvalidDataException("Can't get the absolute path of a non-native file");
return File.FullPath.Base;
}
}
public VirtualFile File { get; }

18
Wabbajack.Lib/Downloaders/AFKModsDownloader.cs
View File

@ -1,18 +0,0 @@
using System;
namespace Wabbajack.Lib.Downloaders
{
public class AFKModsDownloader : AbstractIPS4Downloader<AFKModsDownloader, AFKModsDownloader.State>
{
#region INeedsDownload
public override string SiteName => "AFK Mods";
public override Uri SiteURL => new Uri("https://www.afkmods.com/index.php?");
public override Uri IconUri => new Uri("https://www.afkmods.com/favicon.ico");
#endregion
public AFKModsDownloader() : base(new Uri("https://www.afkmods.com/index.php?/login/"),
"afkmods", "www.afkmods.com"){}
public class State : State<AFKModsDownloader>{}
}
}

1
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -37,7 +37,6 @@ namespace Wabbajack.Lib.Downloaders
typeof(SteamWorkshopDownloader.State),
typeof(VectorPlexusDownloader.State),
typeof(DeadlyStreamDownloader.State),
typeof(AFKModsDownloader.State),
typeof(TESAllianceDownloader.State),
typeof(BethesdaNetDownloader.State),
typeof(YouTubeDownloader.State)

5
Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs
View File

@ -21,6 +21,9 @@ namespace Wabbajack.Lib.Downloaders
private readonly string _encryptedKeyName;
private readonly string _cookieDomain;
private readonly string _cookieName;
private bool _isPrepared;
// ToDo
// Remove null assignment. Either add nullability to type, or figure way to prepare it safely
public Common.Http.Client AuthedClient { get; private set; } = null!;
@ -100,7 +103,9 @@ namespace Wabbajack.Lib.Downloaders
public async Task Prepare()
{
if (_isPrepared) return;
AuthedClient = (await GetAuthedClient()) ?? throw new NotLoggedInError(this);
_isPrepared = true;
}
public class NotLoggedInError : Exception

9
Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs
View File

@ -29,6 +29,7 @@ namespace Wabbajack.Lib.Downloaders
{
public class BethesdaNetDownloader : IUrlDownloader, INeedsLogin
{
private bool _isPrepared;
public const string DataName = "bethesda-net-data";
public ReactiveCommand<Unit, Unit> TriggerLogin { get; }
@ -70,8 +71,14 @@ namespace Wabbajack.Lib.Downloaders
public async Task Prepare()
{
if (Utils.HaveEncryptedJson(DataName)) return;
if (_isPrepared) return;
if (Utils.HaveEncryptedJson(DataName))
{
_isPrepared = true;
return;
}
await Utils.Log(new RequestBethesdaNetLogin()).Task;
_isPrepared = true;
}
public static async Task<BethesdaNetData?> Login(Game game)

1
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -23,7 +23,6 @@ namespace Wabbajack.Lib.Downloaders
new VectorPlexusDownloader(),
new DeadlyStreamDownloader(),
new BethesdaNetDownloader(),
new AFKModsDownloader(),
new TESAllianceDownloader(),
new YouTubeDownloader(),
new HTTPDownloader(),

2
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -100,7 +100,7 @@ namespace Wabbajack.Lib.Downloaders
{
if (!_prepared)
{
using var _ = await _lock.Wait();
using var _ = await _lock.WaitAsync();
// Could have become prepared while we waited for the lock
if (!_prepared)
{

6
Wabbajack.Lib/MO2Compiler.cs
View File

@ -459,13 +459,13 @@ namespace Wabbajack.Lib
await using var srcStream = srcFile.OpenRead();
await using var outputStream = IncludeFile(out var id);
entry.PatchID = id;
await using var destStream = LoadDataForTo(entry.To, absolutePaths);
await using var destStream = await LoadDataForTo(entry.To, absolutePaths);
await Utils.CreatePatch(srcStream, srcFile.Hash, destStream, entry.Hash, outputStream);
Info($"Patch size {outputStream.Length} for {entry.To}");
});
}
private FileStream LoadDataForTo(RelativePath to, Dictionary<RelativePath, AbsolutePath> absolutePaths)
private async Task<FileStream> LoadDataForTo(RelativePath to, Dictionary<RelativePath, AbsolutePath> absolutePaths)
{
if (absolutePaths.TryGetValue(to, out var absolute))
return absolute.OpenRead();
@ -475,7 +475,7 @@ namespace Wabbajack.Lib
var bsaId = (RelativePath)((string)to).Split('\\')[1];
var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsaId);
using var a = BSADispatch.OpenRead(MO2Folder.Combine(bsa.To));
await using var a = BSADispatch.OpenRead(MO2Folder.Combine(bsa.To));
var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray());
var file = a.Files.First(e => e.Path == find);
var returnStream = new TempStream();

2
Wabbajack.Lib/MO2Installer.cs
View File

@ -259,7 +259,7 @@ namespace Wabbajack.Lib
var bsaSize = bsa.FileStates.Select(state => sourceDir.Combine(state.Path).Size).Sum();
using (var a = bsa.State.MakeBuilder(bsaSize))
await using (var a = bsa.State.MakeBuilder(bsaSize))
{
var streams = await bsa.FileStates.PMap(Queue, state =>
{

4
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -52,7 +52,7 @@ namespace Wabbajack.Lib.NexusApi
private static AsyncLock _getAPIKeyLock = new AsyncLock();
private static async Task<string> GetApiKey()
{
using (await _getAPIKeyLock.Wait())
using (await _getAPIKeyLock.WaitAsync())
{
// Clean up old location
if (File.Exists(API_KEY_CACHE_FILE))
@ -114,7 +114,7 @@ namespace Wabbajack.Lib.NexusApi
key = await browser.EvaluateJavaScript(
"document.querySelector(\"input[value=wabbajack]\").parentElement.parentElement.querySelector(\"textarea.application-key\").innerHTML");
}
catch (Exception ex)
catch (Exception)
{
// ignored
}

3
Wabbajack.Test/DownloaderTests.cs
View File

@ -46,7 +46,8 @@ namespace Wabbajack.Test
[Fact]
public async Task TestAllPrepares()
{
await Task.WhenAll(DownloadDispatcher.Downloaders.Select(d => d.Prepare()));
foreach (var downloader in DownloadDispatcher.Downloaders)
await downloader.Prepare();
}
[Fact]

7
Wabbajack.Test/EndToEndTests.cs
View File

@ -105,8 +105,8 @@ namespace Wabbajack.Test
await src.CopyToAsync(utils.DownloadsFolder.Combine(filename));
await FileExtractor.ExtractAll(Queue, src,
modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName));
await using var dest = await FileExtractor.ExtractAll(Queue, src);
await dest.MoveAllTo(modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName));
}
private async Task<(AbsolutePath Download, AbsolutePath ModFolder)> DownloadAndInstall(Game game, int modId, string modName)
@ -140,7 +140,8 @@ namespace Wabbajack.Test
await src.CopyToAsync(dest);
var modFolder = utils.ModsFolder.Combine(modName);
await FileExtractor.ExtractAll(Queue, src, modFolder);
await using var files = await FileExtractor.ExtractAll(Queue, src);
await files.MoveAllTo(modFolder);
await dest.WithExtension(Consts.MetaFileExtension).WriteAllTextAsync(ini);
return (dest, modFolder);

12
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using Xunit;
@ -141,7 +142,10 @@ namespace Wabbajack.VirtualFileSystem.Test
var file = context.Index.ByFullPath[res];
var cleanup = await context.Stage(new List<VirtualFile> {file});
Assert.Equal("This is a test", await file.StagedPath.ReadAllTextAsync());
await using var stream = file.StagedFile.OpenRead();
Assert.Equal("This is a test", await stream.ReadAllTextAsync());
await cleanup();
}
@ -164,7 +168,11 @@ namespace Wabbajack.VirtualFileSystem.Test
var cleanup = await context.Stage(files);
foreach (var file in files)
Assert.Equal("This is a test", await file.StagedPath.ReadAllTextAsync());
{
await using var stream = file.StagedFile.OpenRead();
Assert.Equal("This is a test", await stream.ReadAllTextAsync());
}
await cleanup();
}

18
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -74,7 +74,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return await VirtualFile.Analyze(this, null, f, f, 0);
return await VirtualFile.Analyze(this, null, new ExtractedDiskFile(f), f, 0);
});
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@ -90,7 +90,7 @@ namespace Wabbajack.VirtualFileSystem
public async Task<IndexRoot> AddRoots(List<AbsolutePath> roots)
{
await _cleanupTask;
var native = Index.AllFiles.Where(file => file.IsNative).ToDictionary(file => file.StagedPath);
var native = Index.AllFiles.Where(file => file.IsNative).ToDictionary(file => file.FullPath.Base);
var filtered = Index.AllFiles.Where(file => ((AbsolutePath)file.Name).Exists).ToList();
@ -106,7 +106,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return await VirtualFile.Analyze(this, null, f, f, 0);
return await VirtualFile.Analyze(this, null, new ExtractedDiskFile(f), f, 0);
});
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@ -210,22 +210,22 @@ namespace Wabbajack.VirtualFileSystem
.OrderBy(f => f.Key?.NestingFactor ?? 0)
.ToList();
var paths = new List<AbsolutePath>();
var paths = new List<IAsyncDisposable>();
foreach (var group in grouped)
{
var tmpPath = ((RelativePath)Guid.NewGuid().ToString()).RelativeTo(StagingFolder);
await FileExtractor.ExtractAll(Queue, group.Key.StagedPath, tmpPath);
paths.Add(tmpPath);
var only = group.Select(f => f.RelativeName);
var extracted = await group.Key.StagedFile.ExtractAll(Queue, only);
paths.Add(extracted);
foreach (var file in group)
file.StagedPath = file.RelativeName.RelativeTo(tmpPath);
file.StagedFile = extracted[file.RelativeName];
}
return async () =>
{
foreach (var p in paths)
{
await p.DeleteDirectory();
await p.DisposeAsync();
}
};
}

51
Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs Normal file
View File

@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Compression.BSA;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public class ExtractedBSAFile : IExtractedFile
{
private readonly IFile _file;
public ExtractedBSAFile(IFile file)
{
_file = file;
}
public RelativePath Path => _file.Path;
public async Task<Hash> HashAsync()
{
await using var stream = OpenRead();
return stream.xxHash();
}
public DateTime LastModifiedUtc => DateTime.UtcNow;
public long Size => _file.Size;
public Stream OpenRead()
{
var ms = new MemoryStream();
_file.CopyDataTo(ms);
ms.Position = 0;
return ms;
}
public async Task<bool> CanExtract()
{
return false;
}
public Task<ExtractedFiles> ExtractAll(WorkQueue queue, IEnumerable<RelativePath> OnlyFiles)
{
throw new Exception("BSAs can't contain archives");
}
public async Task MoveTo(AbsolutePath path)
{
await using var fs = path.Create();
_file.CopyDataTo(fs);
}
}
}

47
Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs Normal file
View File

@ -0,0 +1,47 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public class ExtractedDiskFile : IExtractedFile
{
private AbsolutePath _path;
public ExtractedDiskFile(AbsolutePath path)
{
if (path == default)
throw new InvalidDataException("Path cannot be empty");
_path = path;
}
public async Task<Hash> HashAsync()
{
return await _path.FileHashAsync();
}
public DateTime LastModifiedUtc => _path.LastModifiedUtc;
public long Size => _path.Size;
public Stream OpenRead()
{
return _path.OpenRead();
}
public async Task<bool> CanExtract()
{
return await FileExtractor.CanExtract(_path);
}
public Task<ExtractedFiles> ExtractAll(WorkQueue queue, IEnumerable<RelativePath> onlyFiles)
{
return FileExtractor.ExtractAll(queue, _path, onlyFiles);
}
public async Task MoveTo(AbsolutePath path)
{
await _path.MoveToAsync(path, true);
_path = path;
}
}
}

64
Wabbajack.VirtualFileSystem/ExtractedFiles.cs Normal file
View File

@ -0,0 +1,64 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public class ExtractedFiles : IAsyncDisposable, IEnumerable<KeyValuePair<RelativePath, IExtractedFile>>
{
private Dictionary<RelativePath, IExtractedFile> _files;
private IAsyncDisposable _disposable;
private AbsolutePath _tempFolder;
public ExtractedFiles(Dictionary<RelativePath, IExtractedFile> files, IAsyncDisposable disposeOther)
{
_files = files;
_disposable = disposeOther;
}
public ExtractedFiles(TempFolder tempPath)
{
_files = tempPath.Dir.EnumerateFiles().ToDictionary(f => f.RelativeTo(tempPath.Dir),
f => (IExtractedFile)new ExtractedDiskFile(f));
_disposable = tempPath;
}
public async ValueTask DisposeAsync()
{
if (_disposable != null)
{
await _disposable.DisposeAsync();
_disposable = null;
}
}
public bool ContainsKey(RelativePath key)
{
return _files.ContainsKey(key);
}
public int Count => _files.Count;
public IExtractedFile this[RelativePath key] => _files[key];
public IEnumerator<KeyValuePair<RelativePath, IExtractedFile>> GetEnumerator()
{
return _files.GetEnumerator();
}
public async Task MoveAllTo(AbsolutePath folder)
{
foreach (var (key, value) in this)
{
await value.MoveTo(key.RelativeTo(folder));
}
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}

81
Wabbajack.VirtualFileSystem/FileExtractor.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Reactive.Linq;
@ -16,42 +17,43 @@ namespace Wabbajack.VirtualFileSystem
{
public class FileExtractor
{
public static async Task ExtractAll(WorkQueue queue, AbsolutePath source, AbsolutePath dest)
public static async Task<ExtractedFiles> ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable<RelativePath> OnlyFiles = null)
{
try
{
if (Consts.SupportedBSAs.Contains(source.Extension))
await ExtractAllWithBSA(queue, source, dest);
return await ExtractAllWithBSA(queue, source);
else if (source.Extension == Consts.OMOD)
ExtractAllWithOMOD(source, dest);
return ExtractAllWithOMOD(source);
else if (source.Extension == Consts.EXE)
await ExtractAllExe(source, dest);
return await ExtractAllExe(source);
else
await ExtractAllWith7Zip(source, dest);
return await ExtractAllWith7Zip(source, OnlyFiles);
}
catch (Exception ex)
{
Utils.ErrorThrow(ex, $"Error while extracting {source}");
throw new Exception();
}
}
private static async Task ExtractAllExe(AbsolutePath source, AbsolutePath dest)
private static async Task<ExtractedFiles> ExtractAllExe(AbsolutePath source)
{
var isArchive = await TestWith7z(source);
if (isArchive)
{
await ExtractAllWith7Zip(source, dest);
return;
return await ExtractAllWith7Zip(source, null);
}
var dest = new TempFolder();
Utils.Log($"Extracting {(string)source.FileName}");
var process = new ProcessHelper
{
Path = @"Extractors\innounp.exe".RelativeTo(AbsolutePath.EntryPoint),
Arguments = new object[] {"-x", "-y", "-b", $"-d\"{dest}\"", source}
Arguments = new object[] {"-x", "-y", "-b", $"-d\"{dest.Dir}\"", source}
};
@ -69,7 +71,8 @@ namespace Wabbajack.VirtualFileSystem
Utils.Status($"Extracting {source.FileName} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d));
});
await process.Start();
}
return new ExtractedFiles(dest);
}
private class OMODProgress : ICodeProgress
{
@ -91,55 +94,63 @@ namespace Wabbajack.VirtualFileSystem
}
}
private static void ExtractAllWithOMOD(AbsolutePath source, AbsolutePath dest)
private static ExtractedFiles ExtractAllWithOMOD(AbsolutePath source)
{
var dest = new TempFolder();
Utils.Log($"Extracting {(string)source.FileName}");
Framework.Settings.TempPath = (string)dest;
Framework.Settings.TempPath = (string)dest.Dir;
Framework.Settings.CodeProgress = new OMODProgress();
var omod = new OMOD((string)source);
omod.GetDataFiles();
omod.GetPlugins();
return new ExtractedFiles(dest);
}
private static async Task ExtractAllWithBSA(WorkQueue queue, AbsolutePath source, AbsolutePath dest)
private static async Task<ExtractedFiles> ExtractAllWithBSA(WorkQueue queue, AbsolutePath source)
{
try
{
using var arch = BSADispatch.OpenRead(source);
await arch.Files
.PMap(queue, f =>
{
Utils.Status($"Extracting {(string)f.Path}");
var outPath = f.Path.RelativeTo(dest);
var parent = outPath.Parent;
if (!parent.IsDirectory)
parent.CreateDirectory();
using var fs = outPath.Create();
f.CopyDataTo(fs);
});
await using var arch = BSADispatch.OpenRead(source);
var files = arch.Files.ToDictionary(f => f.Path, f => (IExtractedFile)new ExtractedBSAFile(f));
return new ExtractedFiles(files, arch);
}
catch (Exception ex)
{
Utils.ErrorThrow(ex, $"While Extracting {source}");
throw new Exception();
}
}
private static async Task ExtractAllWith7Zip(AbsolutePath source, AbsolutePath dest)
private static async Task<ExtractedFiles> ExtractAllWith7Zip(AbsolutePath source, IEnumerable<RelativePath> onlyFiles)
{
TempFile tmpFile = null;
var dest = new TempFolder();
Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}", $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe"));
var process = new ProcessHelper
{
Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),
Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest}\"", source, "-mmt=off"}
};
if (onlyFiles != null)
{
tmpFile = new TempFile();
await tmpFile.Path.WriteAllLinesAsync(onlyFiles.Select(f => (string)f).ToArray());
process.Arguments = new object[]
{
"x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, $"@\"{tmpFile.Path}\"", "-mmt=off"
};
}
else
{
process.Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, "-mmt=off"};
}
var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p =>
@ -159,12 +170,15 @@ namespace Wabbajack.VirtualFileSystem
if (exitCode != 0)
{
Utils.Error(new _7zipReturnError(exitCode, source, dest, ""));
Utils.Error(new _7zipReturnError(exitCode, source, dest.Dir, ""));
}
else
{
Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true);
}
tmpFile?.Dispose();
return new ExtractedFiles(dest);
}
/// <summary>
@ -205,9 +219,8 @@ namespace Wabbajack.VirtualFileSystem
private static Extension _exeExtension = new Extension(".exe");
public static bool MightBeArchive(AbsolutePath path)
public static bool MightBeArchive(Extension ext)
{
var ext = path.Extension;
return ext == _exeExtension || Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext);
}
}

24
Wabbajack.VirtualFileSystem/IExtractedFile.cs Normal file
View File

@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public interface IExtractedFile
{
public Task<Hash> HashAsync();
public DateTime LastModifiedUtc { get; }
public long Size { get; }
public Stream OpenRead();
public Task<bool> CanExtract();
public Task<ExtractedFiles> ExtractAll(WorkQueue queue, IEnumerable<RelativePath> Only = null);
public Task MoveTo(AbsolutePath path);
}
}

57
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -38,22 +38,21 @@ namespace Wabbajack.VirtualFileSystem
public Context Context { get; set; }
public AbsolutePath StagedPath
private IExtractedFile _stagedFile = null;
public IExtractedFile StagedFile
{
get
{
if (IsNative)
return (AbsolutePath)Name;
if (_stagedPath == null)
throw new UnstagedFileException(FullPath);
return _stagedPath;
if (IsNative) return new ExtractedDiskFile(AbsoluteName);
if (_stagedFile == null)
throw new InvalidDataException("File is unstaged");
return _stagedFile;
}
internal set
set
{
if (IsNative)
throw new CannotStageNativeFile("Cannot stage a native file");
_stagedPath = value;
_stagedFile = value;
}
}
/// <summary>
@ -129,18 +128,18 @@ namespace Wabbajack.VirtualFileSystem
itm.ThisAndAllChildrenReduced(fn);
}
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, AbsolutePath absPath,
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile,
IPath relPath, int depth = 0)
{
var hash = absPath.FileHash();
var hash = await extractedFile.HashAsync();
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(absPath))
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension))
{
var result = await TryGetContentsFromServer(hash);
if (result != null)
{
Utils.Log($"Downloaded VFS data for {(string)absPath}");
Utils.Log($"Downloaded VFS data for {relPath.FileName}");
VirtualFile Convert(IndexedVirtualFile file, IPath path, VirtualFile vparent)
{
@ -150,7 +149,7 @@ namespace Wabbajack.VirtualFileSystem
Name = path,
Parent = vparent,
Size = file.Size,
LastModified = absPath.LastModifiedUtc.AsUnixTime(),
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = file.Hash
};
@ -169,8 +168,8 @@ namespace Wabbajack.VirtualFileSystem
Context = context,
Name = relPath,
Parent = parent,
Size = absPath.Size,
LastModified = absPath.LastModifiedUtc.AsUnixTime(),
Size = extractedFile.Size,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = hash
};
@ -178,19 +177,17 @@ namespace Wabbajack.VirtualFileSystem
self.FillFullPath(depth);
if (context.UseExtendedHashes)
self.ExtendedHashes = ExtendedHashes.FromFile(absPath);
self.ExtendedHashes = ExtendedHashes.FromFile(extractedFile);
if (await FileExtractor.CanExtract(absPath))
{
await using var tempFolder = Context.GetTemporaryFolder();
await FileExtractor.ExtractAll(context.Queue, absPath, tempFolder.FullName);
if (!await extractedFile.CanExtract()) return self;
var list = await tempFolder.FullName.EnumerateFiles()
.PMap(context.Queue,
absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), depth + 1));
await using var extracted = await extractedFile.ExtractAll(context.Queue);
self.Children = list.ToImmutableList();
}
var list = await extracted
.PMap(context.Queue,
file => Analyze(context, self, file.Value, file.Key, depth + 1));
self.Children = list.ToImmutableList();
return self;
}
@ -320,9 +317,9 @@ namespace Wabbajack.VirtualFileSystem
return path;
}
public FileStream OpenRead()
public Stream OpenRead()
{
return StagedPath.OpenRead();
return StagedFile.OpenRead();
}
}
@ -333,7 +330,7 @@ namespace Wabbajack.VirtualFileSystem
public string MD5 { get; set; }
public string CRC { get; set; }
public static ExtendedHashes FromFile(AbsolutePath file)
public static ExtendedHashes FromFile(IExtractedFile file)
{
var hashes = new ExtendedHashes();
using (var stream = file.OpenRead())

2
Wabbajack/View Models/MainWindowVM.cs
View File

@ -84,7 +84,7 @@ namespace Wabbajack
.ObserveOnGuiThread()
.SelectTask(async msg =>
{
using var _ = await singleton_lock.Wait();
using var _ = await singleton_lock.WaitAsync();
try
{
await UserInterventionHandlers.Handle(msg);