diff --git a/Compression.BSA.Test/BSATests.cs b/Compression.BSA.Test/BSATests.cs index 5456d970..0708289c 100644 --- a/Compression.BSA.Test/BSATests.cs +++ b/Compression.BSA.Test/BSATests.cs @@ -81,7 +81,8 @@ namespace Compression.BSA.Test var folder = _bsaFolder.Combine(game.ToString(), modid.ToString()); await folder.DeleteDirectory(); folder.CreateDirectory(); - await FileExtractor.ExtractAll(Queue, filename, folder); + await using var files = await FileExtractor.ExtractAll(Queue, filename); + await files.MoveAllTo(folder); foreach (var bsa in folder.EnumerateFiles().Where(f => Consts.SupportedBSAs.Contains(f.Extension))) { @@ -94,7 +95,7 @@ namespace Compression.BSA.Test var tempFile = ((RelativePath)"tmp.bsa").RelativeToEntryPoint(); var size = bsa.Size; - using var a = BSADispatch.OpenRead(bsa); + await using var a = BSADispatch.OpenRead(bsa); await a.Files.PMap(Queue, file => { var absName = _tempDir.Combine(file.Path); @@ -111,7 +112,7 @@ namespace Compression.BSA.Test TestContext.WriteLine($"Building {bsa}"); - using (var w = ViaJson(a.State).MakeBuilder(size)) + await using (var w = ViaJson(a.State).MakeBuilder(size)) { var streams = await a.Files.PMap(Queue, file => { @@ -125,7 +126,7 @@ namespace Compression.BSA.Test } TestContext.WriteLine($"Verifying {bsa}"); - using var b = BSADispatch.OpenRead(tempFile); + await using var b = BSADispatch.OpenRead(tempFile); TestContext.WriteLine($"Performing A/B tests on {bsa}"); Assert.Equal(a.State.ToJson(), b.State.ToJson()); diff --git a/Compression.BSA/BA2Builder.cs b/Compression.BSA/BA2Builder.cs index e596f8df..6f991dea 100644 --- a/Compression.BSA/BA2Builder.cs +++ b/Compression.BSA/BA2Builder.cs @@ -4,6 +4,7 @@ using System.IO; using System.IO.MemoryMappedFiles; using System.Linq; using System.Text; +using System.Threading.Tasks; using ICSharpCode.SharpZipLib.Zip.Compression; using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using Wabbajack.Common; @@ -34,7 +35,7 @@ namespace Compression.BSA _slab = new DiskSlabAllocator(size); } - public void Dispose() + public async ValueTask DisposeAsync() { _slab.Dispose(); } diff --git a/Compression.BSA/BA2Reader.cs b/Compression.BSA/BA2Reader.cs index 5795a4fd..3acddeae 100644 --- a/Compression.BSA/BA2Reader.cs +++ b/Compression.BSA/BA2Reader.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; +using System.Threading.Tasks; using ICSharpCode.SharpZipLib.Zip.Compression; using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; @@ -99,7 +100,7 @@ namespace Compression.BSA } - public void Dispose() + public async ValueTask DisposeAsync() { _stream?.Dispose(); _rdr?.Dispose(); diff --git a/Compression.BSA/BSABuilder.cs b/Compression.BSA/BSABuilder.cs index 1bce94cb..8b93023b 100644 --- a/Compression.BSA/BSABuilder.cs +++ b/Compression.BSA/BSABuilder.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; +using System.Threading.Tasks; using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using K4os.Compression.LZ4; using K4os.Compression.LZ4.Streams; @@ -75,7 +76,7 @@ namespace Compression.BSA public bool HasNameBlobs => (_archiveFlags & 0x100) > 0; - public void Dispose() + public async ValueTask DisposeAsync() { _slab.Dispose(); } diff --git a/Compression.BSA/BSAReader.cs b/Compression.BSA/BSAReader.cs index 39661df4..e600c6e4 100644 --- a/Compression.BSA/BSAReader.cs +++ b/Compression.BSA/BSAReader.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Text; +using System.Threading.Tasks; using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using K4os.Compression.LZ4.Streams; using Wabbajack.Common; @@ -49,7 +50,7 @@ namespace Compression.BSA Miscellaneous = 0x100 } - public class BSAReader : IDisposable, IBSAReader + public class BSAReader : IAsyncDisposable, IBSAReader { internal uint _archiveFlags; internal uint _fileCount; @@ -113,7 +114,7 @@ namespace Compression.BSA } } - public void Dispose() + public async ValueTask DisposeAsync() { _stream.Close(); } diff --git a/Compression.BSA/IBSAReader.cs b/Compression.BSA/IBSAReader.cs index 546185a0..e9f41af5 100644 --- a/Compression.BSA/IBSAReader.cs +++ b/Compression.BSA/IBSAReader.cs @@ -5,7 +5,7 @@ using Wabbajack.Common; namespace Compression.BSA { - public interface IBSAReader : IDisposable + public interface IBSAReader : IAsyncDisposable { /// /// The files defined by the archive @@ -15,7 +15,7 @@ namespace Compression.BSA ArchiveStateObject State { get; } } - public interface IBSABuilder : IDisposable + public interface IBSABuilder : IAsyncDisposable { void AddFile(FileStateObject state, Stream src); void Build(AbsolutePath filename); diff --git a/Compression.BSA/TES3Builder.cs b/Compression.BSA/TES3Builder.cs index 4551805d..d7a9da68 100644 --- a/Compression.BSA/TES3Builder.cs +++ b/Compression.BSA/TES3Builder.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.IO; using System.Text; +using System.Threading.Tasks; using Wabbajack.Common; using File = Alphaleonis.Win32.Filesystem.File; @@ -71,7 +72,7 @@ namespace Compression.BSA } } - public void Dispose() + public async ValueTask DisposeAsync() { } } diff --git a/Compression.BSA/TES3Reader.cs b/Compression.BSA/TES3Reader.cs index 16892a6e..1a2a5a87 100644 --- a/Compression.BSA/TES3Reader.cs +++ b/Compression.BSA/TES3Reader.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Text; +using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; @@ -61,7 +62,7 @@ namespace Compression.BSA _dataOffset = br.BaseStream.Position; } - public void Dispose() + public async ValueTask DisposeAsync() { } diff --git a/Wabbajack.BuildServer/Controllers/ListValidation.cs b/Wabbajack.BuildServer/Controllers/ListValidation.cs index 2b3ab30e..ad5ae634 100644 --- a/Wabbajack.BuildServer/Controllers/ListValidation.cs +++ b/Wabbajack.BuildServer/Controllers/ListValidation.cs @@ -117,7 +117,7 @@ namespace Wabbajack.BuildServer.Controllers private static AsyncLock _findPatchLock = new AsyncLock(); private async Task<(Archive, ArchiveStatus)> TryToFix(SqlService.ValidationData data, Archive archive) { - using var _ = await _findPatchLock.Wait(); + using var _ = await _findPatchLock.WaitAsync(); var result = await _updater.GetAlternative(archive.Hash.ToHex()); return result switch diff --git a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs index 2429a979..e18379fa 100644 --- a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs @@ -63,7 +63,7 @@ namespace Wabbajack.BuildServer.Controllers Utils.Log($"Writing {ms.Length} at position {Offset} in ingest file {Key}"); long position; - using (var _ = await _writeLocks[Key].Wait()) + using (var _ = await _writeLocks[Key].WaitAsync()) { await using var file = _settings.TempPath.Combine(Key).WriteShared(); file.Position = Offset; diff --git a/Wabbajack.CLI/Verbs/DownloadUrl.cs b/Wabbajack.CLI/Verbs/DownloadUrl.cs index 6ac806c4..6fa0c4a7 100644 --- a/Wabbajack.CLI/Verbs/DownloadUrl.cs +++ b/Wabbajack.CLI/Verbs/DownloadUrl.cs @@ -32,11 +32,11 @@ namespace Wabbajack.CLI.Verbs .Debounce(TimeSpan.FromSeconds(1)) .Subscribe(s => Console.WriteLine($"Downloading {s.ProgressPercent}")); - new[] {state} + await new[] {state} .PMap(queue, async s => { await s.Download(new Archive(state: null!) {Name = Path.GetFileName(Output)}, (AbsolutePath)Output); - }).Wait(); + }); File.WriteAllLines(Output + ".meta", state.GetMetaIni()); return 0; diff --git a/Wabbajack.Common.Test/AsyncLockTests.cs b/Wabbajack.Common.Test/AsyncLockTests.cs index 48ba84de..29d55c66 100644 --- a/Wabbajack.Common.Test/AsyncLockTests.cs +++ b/Wabbajack.Common.Test/AsyncLockTests.cs @@ -13,7 +13,7 @@ namespace Wabbajack.Common.Test bool firstRun = false; var first = Task.Run(async () => { - using (await asyncLock.Wait()) + using (await asyncLock.WaitAsync()) { await Task.Delay(500); firstRun = true; @@ -22,7 +22,7 @@ namespace Wabbajack.Common.Test var second = Task.Run(async () => { await Task.Delay(200); - using (await asyncLock.Wait()) + using (await asyncLock.WaitAsync()) { Assert.True(firstRun); } @@ -41,7 +41,7 @@ namespace Wabbajack.Common.Test { return Task.Run(async () => { - using (await asyncLock.Wait()) + using (await asyncLock.WaitAsync()) { await Task.Delay(500); firstRun = true; @@ -55,7 +55,7 @@ namespace Wabbajack.Common.Test Task.Run(async () => { await Task.Delay(200); - using (await asyncLock.Wait()) + using (await asyncLock.WaitAsync()) { Assert.True(firstRun); secondRun = true; diff --git a/Wabbajack.Common/Hash.cs b/Wabbajack.Common/Hash.cs index f83ead85..ecbb319c 100644 --- a/Wabbajack.Common/Hash.cs +++ b/Wabbajack.Common/Hash.cs @@ -145,6 +145,17 @@ namespace Wabbajack.Common var value = xxHashFactory.Instance.Create(config).ComputeHash(f); return Hash.FromULong(BitConverter.ToUInt64(value.Hash)); } + + public static Hash xxHash(this Stream stream) + { + var hash = new xxHashConfig(); + hash.HashSizeInBits = 64; + hash.Seed = 0x42; + var config = new xxHashConfig {HashSizeInBits = 64}; + using var f = new StatusFileStream(stream, $"Hashing memory stream"); + var value = xxHashFactory.Instance.Create(config).ComputeHash(f); + return Hash.FromULong(BitConverter.ToUInt64(value.Hash)); + } public static Hash FileHashCached(this AbsolutePath file, bool nullOnIoError = false) { if (TryGetHashCache(file, out var foundHash)) return foundHash; diff --git a/Wabbajack.Common/OctoDiff.cs b/Wabbajack.Common/OctoDiff.cs index f03e5d09..bb17c9e8 100644 --- a/Wabbajack.Common/OctoDiff.cs +++ b/Wabbajack.Common/OctoDiff.cs @@ -28,7 +28,7 @@ namespace Wabbajack.Common return sigStream; } - private static void CreateSignature(FileStream oldData, FileStream sigStream) + private static void CreateSignature(Stream oldData, FileStream sigStream) { Utils.Status("Creating Patch Signature"); var signatureBuilder = new SignatureBuilder(); @@ -36,7 +36,7 @@ namespace Wabbajack.Common sigStream.Position = 0; } - public static void Create(FileStream oldData, FileStream newData, FileStream signature, FileStream output) + public static void Create(Stream oldData, FileStream newData, FileStream signature, FileStream output) { CreateSignature(oldData, signature); var db = new DeltaBuilder {ProgressReporter = reporter}; diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index d5fac5b3..5073ca4b 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -9,7 +9,7 @@ using System.Reflection; using System.Text; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; -using Directory = System.IO.Directory; +using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; using Path = Alphaleonis.Win32.Filesystem.Path; diff --git a/Wabbajack.Common/Util/AsyncLock.cs b/Wabbajack.Common/Util/AsyncLock.cs index 0474ece8..9b6a64b2 100644 --- a/Wabbajack.Common/Util/AsyncLock.cs +++ b/Wabbajack.Common/Util/AsyncLock.cs @@ -12,7 +12,7 @@ namespace Wabbajack.Common { private readonly SemaphoreSlim _lock = new SemaphoreSlim(1, 1); - public async Task Wait() + public async Task WaitAsync() { await _lock.WaitAsync(); return Disposable.Create(() => _lock.Release()); diff --git a/Wabbajack.Common/Util/TempFolder.cs b/Wabbajack.Common/Util/TempFolder.cs index 6a7992da..9cd37842 100644 --- a/Wabbajack.Common/Util/TempFolder.cs +++ b/Wabbajack.Common/Util/TempFolder.cs @@ -14,7 +14,7 @@ namespace Wabbajack.Common public TempFolder(bool deleteAfter = true) { - Dir = new AbsolutePath(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName())); + Dir = Path.Combine("tmp_files", Guid.NewGuid().ToString()).RelativeTo(AbsolutePath.EntryPoint); if (!Dir.Exists) Dir.CreateDirectory(); DeleteAfter = deleteAfter; diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 62530d80..24fbf909 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -427,6 +427,13 @@ namespace Wabbajack.Common await ins.CopyToAsync(ms); return ms.ToArray(); } + + public static async Task ReadAllTextAsync(this Stream ins) + { + await using var ms = new MemoryStream(); + await ins.CopyToAsync(ms); + return Encoding.UTF8.GetString(ms.ToArray()); + } public static async Task PMap(this IEnumerable coll, WorkQueue queue, StatusUpdateTracker updateTracker, Func f) @@ -727,7 +734,7 @@ namespace Wabbajack.Common } } - public static async Task CreatePatch(FileStream srcStream, Hash srcHash, FileStream destStream, Hash destHash, + public static async Task CreatePatch(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash, FileStream patchStream) { await using var sigFile = new TempStream(); @@ -967,7 +974,7 @@ namespace Wabbajack.Common Status($"Deleting: {p.Line}"); }); - process.Start().Wait(); + await process.Start(); await result; } diff --git a/Wabbajack.Common/WorkQueue.cs b/Wabbajack.Common/WorkQueue.cs index d9667669..f935b5e7 100644 --- a/Wabbajack.Common/WorkQueue.cs +++ b/Wabbajack.Common/WorkQueue.cs @@ -93,7 +93,7 @@ namespace Wabbajack.Common private async Task AddNewThreadsIfNeeded(int desired) { - using (await _lock.Wait()) + using (await _lock.WaitAsync()) { DesiredNumWorkers = desired; while (DesiredNumWorkers > _tasks.Count) @@ -141,7 +141,7 @@ namespace Wabbajack.Common if (DesiredNumWorkers >= _tasks.Count) continue; // Noticed that we may need to shut down, lock and check again - using (await _lock.Wait()) + using (await _lock.WaitAsync()) { // Check if another thread shut down before this one and got us back to the desired amount already if (DesiredNumWorkers >= _tasks.Count) continue; diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index b0cd0536..89eb8d5d 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -172,7 +172,8 @@ namespace Wabbajack.Lib throw new ArgumentNullException("FromFile was null"); } var firstDest = OutputFolder.Combine(group.First().To); - await CopyFile(group.Key.StagedPath, firstDest, true); + + await group.Key.StagedFile.MoveTo(firstDest); foreach (var copy in group.Skip(1)) { diff --git a/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs b/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs index a12e8eff..afee7680 100644 --- a/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs +++ b/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs @@ -77,7 +77,7 @@ namespace Wabbajack.Lib.CompilationSteps } CreateBSA directive; - using (var bsa = BSADispatch.OpenRead(source.AbsolutePath)) + await using (var bsa = BSADispatch.OpenRead(source.AbsolutePath)) { directive = new CreateBSA( state: bsa.State, diff --git a/Wabbajack.Lib/Data.cs b/Wabbajack.Lib/Data.cs index 2c6a63cb..1f6b6807 100644 --- a/Wabbajack.Lib/Data.cs +++ b/Wabbajack.Lib/Data.cs @@ -21,7 +21,15 @@ namespace Wabbajack.Lib Path = path; } - public AbsolutePath AbsolutePath => File.StagedPath; + public AbsolutePath AbsolutePath + { + get + { + if (!File.IsNative) + throw new InvalidDataException("Can't get the absolute path of a non-native file"); + return File.FullPath.Base; + } + } public VirtualFile File { get; } diff --git a/Wabbajack.Lib/Downloaders/AFKModsDownloader.cs b/Wabbajack.Lib/Downloaders/AFKModsDownloader.cs deleted file mode 100644 index fdeb1385..00000000 --- a/Wabbajack.Lib/Downloaders/AFKModsDownloader.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; - -namespace Wabbajack.Lib.Downloaders -{ - public class AFKModsDownloader : AbstractIPS4Downloader - { - #region INeedsDownload - public override string SiteName => "AFK Mods"; - public override Uri SiteURL => new Uri("https://www.afkmods.com/index.php?"); - public override Uri IconUri => new Uri("https://www.afkmods.com/favicon.ico"); - #endregion - - public AFKModsDownloader() : base(new Uri("https://www.afkmods.com/index.php?/login/"), - "afkmods", "www.afkmods.com"){} - - public class State : State{} - } -} diff --git a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs index d3677b1b..ebfc5b32 100644 --- a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs +++ b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs @@ -37,7 +37,6 @@ namespace Wabbajack.Lib.Downloaders typeof(SteamWorkshopDownloader.State), typeof(VectorPlexusDownloader.State), typeof(DeadlyStreamDownloader.State), - typeof(AFKModsDownloader.State), typeof(TESAllianceDownloader.State), typeof(BethesdaNetDownloader.State), typeof(YouTubeDownloader.State) diff --git a/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs b/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs index a6a4e681..acc5e3fd 100644 --- a/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs @@ -21,6 +21,9 @@ namespace Wabbajack.Lib.Downloaders private readonly string _encryptedKeyName; private readonly string _cookieDomain; private readonly string _cookieName; + + private bool _isPrepared; + // ToDo // Remove null assignment. Either add nullability to type, or figure way to prepare it safely public Common.Http.Client AuthedClient { get; private set; } = null!; @@ -100,7 +103,9 @@ namespace Wabbajack.Lib.Downloaders public async Task Prepare() { + if (_isPrepared) return; AuthedClient = (await GetAuthedClient()) ?? throw new NotLoggedInError(this); + _isPrepared = true; } public class NotLoggedInError : Exception diff --git a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs index 4602c126..47445128 100644 --- a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs +++ b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs @@ -29,6 +29,7 @@ namespace Wabbajack.Lib.Downloaders { public class BethesdaNetDownloader : IUrlDownloader, INeedsLogin { + private bool _isPrepared; public const string DataName = "bethesda-net-data"; public ReactiveCommand TriggerLogin { get; } @@ -70,8 +71,14 @@ namespace Wabbajack.Lib.Downloaders public async Task Prepare() { - if (Utils.HaveEncryptedJson(DataName)) return; + if (_isPrepared) return; + if (Utils.HaveEncryptedJson(DataName)) + { + _isPrepared = true; + return; + } await Utils.Log(new RequestBethesdaNetLogin()).Task; + _isPrepared = true; } public static async Task Login(Game game) diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index 7a2a35e9..5cffafda 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -23,7 +23,6 @@ namespace Wabbajack.Lib.Downloaders new VectorPlexusDownloader(), new DeadlyStreamDownloader(), new BethesdaNetDownloader(), - new AFKModsDownloader(), new TESAllianceDownloader(), new YouTubeDownloader(), new HTTPDownloader(), diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 087b46cf..632af6cc 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -100,7 +100,7 @@ namespace Wabbajack.Lib.Downloaders { if (!_prepared) { - using var _ = await _lock.Wait(); + using var _ = await _lock.WaitAsync(); // Could have become prepared while we waited for the lock if (!_prepared) { diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 41c00f29..1a1bf6ed 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -459,13 +459,13 @@ namespace Wabbajack.Lib await using var srcStream = srcFile.OpenRead(); await using var outputStream = IncludeFile(out var id); entry.PatchID = id; - await using var destStream = LoadDataForTo(entry.To, absolutePaths); + await using var destStream = await LoadDataForTo(entry.To, absolutePaths); await Utils.CreatePatch(srcStream, srcFile.Hash, destStream, entry.Hash, outputStream); Info($"Patch size {outputStream.Length} for {entry.To}"); }); } - private FileStream LoadDataForTo(RelativePath to, Dictionary absolutePaths) + private async Task LoadDataForTo(RelativePath to, Dictionary absolutePaths) { if (absolutePaths.TryGetValue(to, out var absolute)) return absolute.OpenRead(); @@ -475,7 +475,7 @@ namespace Wabbajack.Lib var bsaId = (RelativePath)((string)to).Split('\\')[1]; var bsa = InstallDirectives.OfType().First(b => b.TempID == bsaId); - using var a = BSADispatch.OpenRead(MO2Folder.Combine(bsa.To)); + await using var a = BSADispatch.OpenRead(MO2Folder.Combine(bsa.To)); var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray()); var file = a.Files.First(e => e.Path == find); var returnStream = new TempStream(); diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index f0647c37..473b1dba 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -259,7 +259,7 @@ namespace Wabbajack.Lib var bsaSize = bsa.FileStates.Select(state => sourceDir.Combine(state.Path).Size).Sum(); - using (var a = bsa.State.MakeBuilder(bsaSize)) + await using (var a = bsa.State.MakeBuilder(bsaSize)) { var streams = await bsa.FileStates.PMap(Queue, state => { diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index 3f33d093..10b83edb 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -52,7 +52,7 @@ namespace Wabbajack.Lib.NexusApi private static AsyncLock _getAPIKeyLock = new AsyncLock(); private static async Task GetApiKey() { - using (await _getAPIKeyLock.Wait()) + using (await _getAPIKeyLock.WaitAsync()) { // Clean up old location if (File.Exists(API_KEY_CACHE_FILE)) @@ -114,7 +114,7 @@ namespace Wabbajack.Lib.NexusApi key = await browser.EvaluateJavaScript( "document.querySelector(\"input[value=wabbajack]\").parentElement.parentElement.querySelector(\"textarea.application-key\").innerHTML"); } - catch (Exception ex) + catch (Exception) { // ignored } diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 2375f922..49c1a900 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -46,7 +46,8 @@ namespace Wabbajack.Test [Fact] public async Task TestAllPrepares() { - await Task.WhenAll(DownloadDispatcher.Downloaders.Select(d => d.Prepare())); + foreach (var downloader in DownloadDispatcher.Downloaders) + await downloader.Prepare(); } [Fact] diff --git a/Wabbajack.Test/EndToEndTests.cs b/Wabbajack.Test/EndToEndTests.cs index 6f3ad560..d2f74495 100644 --- a/Wabbajack.Test/EndToEndTests.cs +++ b/Wabbajack.Test/EndToEndTests.cs @@ -105,8 +105,8 @@ namespace Wabbajack.Test await src.CopyToAsync(utils.DownloadsFolder.Combine(filename)); - await FileExtractor.ExtractAll(Queue, src, - modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName)); + await using var dest = await FileExtractor.ExtractAll(Queue, src); + await dest.MoveAllTo(modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName)); } private async Task<(AbsolutePath Download, AbsolutePath ModFolder)> DownloadAndInstall(Game game, int modId, string modName) @@ -140,7 +140,8 @@ namespace Wabbajack.Test await src.CopyToAsync(dest); var modFolder = utils.ModsFolder.Combine(modName); - await FileExtractor.ExtractAll(Queue, src, modFolder); + await using var files = await FileExtractor.ExtractAll(Queue, src); + await files.MoveAllTo(modFolder); await dest.WithExtension(Consts.MetaFileExtension).WriteAllTextAsync(ini); return (dest, modFolder); diff --git a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs index 5059fbf8..0868ea56 100644 --- a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO.Compression; using System.Linq; +using System.Text; using System.Threading.Tasks; using Wabbajack.Common; using Xunit; @@ -141,7 +142,10 @@ namespace Wabbajack.VirtualFileSystem.Test var file = context.Index.ByFullPath[res]; var cleanup = await context.Stage(new List {file}); - Assert.Equal("This is a test", await file.StagedPath.ReadAllTextAsync()); + + await using var stream = file.StagedFile.OpenRead(); + + Assert.Equal("This is a test", await stream.ReadAllTextAsync()); await cleanup(); } @@ -164,7 +168,11 @@ namespace Wabbajack.VirtualFileSystem.Test var cleanup = await context.Stage(files); foreach (var file in files) - Assert.Equal("This is a test", await file.StagedPath.ReadAllTextAsync()); + { + await using var stream = file.StagedFile.OpenRead(); + + Assert.Equal("This is a test", await stream.ReadAllTextAsync()); + } await cleanup(); } diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index 4ee6a82b..d2ed40dc 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -74,7 +74,7 @@ namespace Wabbajack.VirtualFileSystem return found; } - return await VirtualFile.Analyze(this, null, f, f, 0); + return await VirtualFile.Analyze(this, null, new ExtractedDiskFile(f), f, 0); }); var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList()); @@ -90,7 +90,7 @@ namespace Wabbajack.VirtualFileSystem public async Task AddRoots(List roots) { await _cleanupTask; - var native = Index.AllFiles.Where(file => file.IsNative).ToDictionary(file => file.StagedPath); + var native = Index.AllFiles.Where(file => file.IsNative).ToDictionary(file => file.FullPath.Base); var filtered = Index.AllFiles.Where(file => ((AbsolutePath)file.Name).Exists).ToList(); @@ -106,7 +106,7 @@ namespace Wabbajack.VirtualFileSystem return found; } - return await VirtualFile.Analyze(this, null, f, f, 0); + return await VirtualFile.Analyze(this, null, new ExtractedDiskFile(f), f, 0); }); var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList()); @@ -210,22 +210,22 @@ namespace Wabbajack.VirtualFileSystem .OrderBy(f => f.Key?.NestingFactor ?? 0) .ToList(); - var paths = new List(); + var paths = new List(); foreach (var group in grouped) { - var tmpPath = ((RelativePath)Guid.NewGuid().ToString()).RelativeTo(StagingFolder); - await FileExtractor.ExtractAll(Queue, group.Key.StagedPath, tmpPath); - paths.Add(tmpPath); + var only = group.Select(f => f.RelativeName); + var extracted = await group.Key.StagedFile.ExtractAll(Queue, only); + paths.Add(extracted); foreach (var file in group) - file.StagedPath = file.RelativeName.RelativeTo(tmpPath); + file.StagedFile = extracted[file.RelativeName]; } return async () => { foreach (var p in paths) { - await p.DeleteDirectory(); + await p.DisposeAsync(); } }; } diff --git a/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs b/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs new file mode 100644 index 00000000..f278ccb0 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Compression.BSA; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem +{ + public class ExtractedBSAFile : IExtractedFile + { + private readonly IFile _file; + public ExtractedBSAFile(IFile file) + { + _file = file; + } + + public RelativePath Path => _file.Path; + + public async Task HashAsync() + { + await using var stream = OpenRead(); + return stream.xxHash(); + } + public DateTime LastModifiedUtc => DateTime.UtcNow; + public long Size => _file.Size; + public Stream OpenRead() + { + var ms = new MemoryStream(); + _file.CopyDataTo(ms); + ms.Position = 0; + return ms; + } + + public async Task CanExtract() + { + return false; + } + + public Task ExtractAll(WorkQueue queue, IEnumerable OnlyFiles) + { + throw new Exception("BSAs can't contain archives"); + } + + public async Task MoveTo(AbsolutePath path) + { + await using var fs = path.Create(); + _file.CopyDataTo(fs); + } + } +} diff --git a/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs b/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs new file mode 100644 index 00000000..d748ad97 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem +{ + public class ExtractedDiskFile : IExtractedFile + { + private AbsolutePath _path; + + public ExtractedDiskFile(AbsolutePath path) + { + if (path == default) + throw new InvalidDataException("Path cannot be empty"); + _path = path; + } + + public async Task HashAsync() + { + return await _path.FileHashAsync(); + } + public DateTime LastModifiedUtc => _path.LastModifiedUtc; + public long Size => _path.Size; + public Stream OpenRead() + { + return _path.OpenRead(); + } + + public async Task CanExtract() + { + return await FileExtractor.CanExtract(_path); + } + + public Task ExtractAll(WorkQueue queue, IEnumerable onlyFiles) + { + return FileExtractor.ExtractAll(queue, _path, onlyFiles); + } + + public async Task MoveTo(AbsolutePath path) + { + await _path.MoveToAsync(path, true); + _path = path; + } + } +} diff --git a/Wabbajack.VirtualFileSystem/ExtractedFiles.cs b/Wabbajack.VirtualFileSystem/ExtractedFiles.cs new file mode 100644 index 00000000..f0e8afc3 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/ExtractedFiles.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem +{ + public class ExtractedFiles : IAsyncDisposable, IEnumerable> + { + private Dictionary _files; + private IAsyncDisposable _disposable; + private AbsolutePath _tempFolder; + + public ExtractedFiles(Dictionary files, IAsyncDisposable disposeOther) + { + _files = files; + _disposable = disposeOther; + } + + public ExtractedFiles(TempFolder tempPath) + { + _files = tempPath.Dir.EnumerateFiles().ToDictionary(f => f.RelativeTo(tempPath.Dir), + f => (IExtractedFile)new ExtractedDiskFile(f)); + _disposable = tempPath; + } + + public async ValueTask DisposeAsync() + { + if (_disposable != null) + { + await _disposable.DisposeAsync(); + _disposable = null; + } + } + + public bool ContainsKey(RelativePath key) + { + return _files.ContainsKey(key); + } + + public int Count => _files.Count; + + public IExtractedFile this[RelativePath key] => _files[key]; + public IEnumerator> GetEnumerator() + { + return _files.GetEnumerator(); + } + + public async Task MoveAllTo(AbsolutePath folder) + { + foreach (var (key, value) in this) + { + await value.MoveTo(key.RelativeTo(folder)); + } + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + } +} diff --git a/Wabbajack.VirtualFileSystem/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor.cs index 8f53f144..9e916349 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Reactive.Linq; @@ -16,42 +17,43 @@ namespace Wabbajack.VirtualFileSystem { public class FileExtractor { - - public static async Task ExtractAll(WorkQueue queue, AbsolutePath source, AbsolutePath dest) + + public static async Task ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable OnlyFiles = null) { try { if (Consts.SupportedBSAs.Contains(source.Extension)) - await ExtractAllWithBSA(queue, source, dest); + return await ExtractAllWithBSA(queue, source); else if (source.Extension == Consts.OMOD) - ExtractAllWithOMOD(source, dest); + return ExtractAllWithOMOD(source); else if (source.Extension == Consts.EXE) - await ExtractAllExe(source, dest); + return await ExtractAllExe(source); else - await ExtractAllWith7Zip(source, dest); + return await ExtractAllWith7Zip(source, OnlyFiles); } catch (Exception ex) { Utils.ErrorThrow(ex, $"Error while extracting {source}"); + throw new Exception(); } } - private static async Task ExtractAllExe(AbsolutePath source, AbsolutePath dest) + private static async Task ExtractAllExe(AbsolutePath source) { var isArchive = await TestWith7z(source); if (isArchive) { - await ExtractAllWith7Zip(source, dest); - return; + return await ExtractAllWith7Zip(source, null); } + var dest = new TempFolder(); Utils.Log($"Extracting {(string)source.FileName}"); var process = new ProcessHelper { Path = @"Extractors\innounp.exe".RelativeTo(AbsolutePath.EntryPoint), - Arguments = new object[] {"-x", "-y", "-b", $"-d\"{dest}\"", source} + Arguments = new object[] {"-x", "-y", "-b", $"-d\"{dest.Dir}\"", source} }; @@ -69,7 +71,8 @@ namespace Wabbajack.VirtualFileSystem Utils.Status($"Extracting {source.FileName} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d)); }); await process.Start(); - } + return new ExtractedFiles(dest); + } private class OMODProgress : ICodeProgress { @@ -91,55 +94,70 @@ namespace Wabbajack.VirtualFileSystem } } - private static void ExtractAllWithOMOD(AbsolutePath source, AbsolutePath dest) + private static ExtractedFiles ExtractAllWithOMOD(AbsolutePath source) { + var dest = new TempFolder(); Utils.Log($"Extracting {(string)source.FileName}"); - Framework.Settings.TempPath = (string)dest; + Framework.Settings.TempPath = (string)dest.Dir; Framework.Settings.CodeProgress = new OMODProgress(); var omod = new OMOD((string)source); omod.GetDataFiles(); omod.GetPlugins(); + + return new ExtractedFiles(dest); } - private static async Task ExtractAllWithBSA(WorkQueue queue, AbsolutePath source, AbsolutePath dest) + private static async Task ExtractAllWithBSA(WorkQueue queue, AbsolutePath source) { try { - using var arch = BSADispatch.OpenRead(source); - await arch.Files - .PMap(queue, f => - { - Utils.Status($"Extracting {(string)f.Path}"); - var outPath = f.Path.RelativeTo(dest); - var parent = outPath.Parent; - - if (!parent.IsDirectory) - parent.CreateDirectory(); - - using var fs = outPath.Create(); - f.CopyDataTo(fs); - }); + await using var arch = BSADispatch.OpenRead(source); + var files = arch.Files.ToDictionary(f => f.Path, f => (IExtractedFile)new ExtractedBSAFile(f)); + return new ExtractedFiles(files, arch); } catch (Exception ex) { Utils.ErrorThrow(ex, $"While Extracting {source}"); + throw new Exception(); } } - private static async Task ExtractAllWith7Zip(AbsolutePath source, AbsolutePath dest) + private static async Task ExtractAllWith7Zip(AbsolutePath source, IEnumerable onlyFiles) { + TempFile tmpFile = null; + var dest = new TempFolder(); Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}", $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe")); - var process = new ProcessHelper { Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint), - Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest}\"", source, "-mmt=off"} + }; + if (onlyFiles != null) + { + //It's stupid that we have to do this, but 7zip's file pattern matching isn't very fuzzy + IEnumerable AllVariants(string input) + { + yield return input; + yield return "\\" + input; + } + + tmpFile = new TempFile(); + await tmpFile.Path.WriteAllLinesAsync(onlyFiles.SelectMany(f => AllVariants((string)f)).ToArray()); + process.Arguments = new object[] + { + "x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, $"@\"{tmpFile.Path}\"", "-mmt=off" + }; + } + else + { + process.Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, "-mmt=off"}; + } + var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output) .ForEachAsync(p => @@ -159,12 +177,15 @@ namespace Wabbajack.VirtualFileSystem if (exitCode != 0) { - Utils.Error(new _7zipReturnError(exitCode, source, dest, "")); + Utils.Error(new _7zipReturnError(exitCode, source, dest.Dir, "")); } else { Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true); } + + tmpFile?.Dispose(); + return new ExtractedFiles(dest); } /// @@ -205,9 +226,8 @@ namespace Wabbajack.VirtualFileSystem private static Extension _exeExtension = new Extension(".exe"); - public static bool MightBeArchive(AbsolutePath path) + public static bool MightBeArchive(Extension ext) { - var ext = path.Extension; return ext == _exeExtension || Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext); } } diff --git a/Wabbajack.VirtualFileSystem/IExtractedFile.cs b/Wabbajack.VirtualFileSystem/IExtractedFile.cs new file mode 100644 index 00000000..e37e86b6 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/IExtractedFile.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem +{ + public interface IExtractedFile + { + public Task HashAsync(); + public DateTime LastModifiedUtc { get; } + public long Size { get; } + + public Stream OpenRead(); + + public Task CanExtract(); + + public Task ExtractAll(WorkQueue queue, IEnumerable Only = null); + + public Task MoveTo(AbsolutePath path); + + } +} diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 2c99b5bb..27593fba 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -38,22 +38,21 @@ namespace Wabbajack.VirtualFileSystem public Context Context { get; set; } - public AbsolutePath StagedPath + private IExtractedFile _stagedFile = null; + public IExtractedFile StagedFile { get { - if (IsNative) - return (AbsolutePath)Name; - if (_stagedPath == null) - throw new UnstagedFileException(FullPath); - return _stagedPath; + if (IsNative) return new ExtractedDiskFile(AbsoluteName); + if (_stagedFile == null) + throw new InvalidDataException("File is unstaged"); + return _stagedFile; } - internal set + set { - if (IsNative) - throw new CannotStageNativeFile("Cannot stage a native file"); - _stagedPath = value; + _stagedFile = value; } + } /// @@ -129,18 +128,18 @@ namespace Wabbajack.VirtualFileSystem itm.ThisAndAllChildrenReduced(fn); } - public static async Task Analyze(Context context, VirtualFile parent, AbsolutePath absPath, + public static async Task Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile, IPath relPath, int depth = 0) { - var hash = absPath.FileHash(); + var hash = await extractedFile.HashAsync(); - if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(absPath)) + if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension)) { var result = await TryGetContentsFromServer(hash); if (result != null) { - Utils.Log($"Downloaded VFS data for {(string)absPath}"); + Utils.Log($"Downloaded VFS data for {relPath.FileName}"); VirtualFile Convert(IndexedVirtualFile file, IPath path, VirtualFile vparent) { @@ -150,7 +149,7 @@ namespace Wabbajack.VirtualFileSystem Name = path, Parent = vparent, Size = file.Size, - LastModified = absPath.LastModifiedUtc.AsUnixTime(), + LastModified = extractedFile.LastModifiedUtc.AsUnixTime(), LastAnalyzed = DateTime.Now.AsUnixTime(), Hash = file.Hash }; @@ -169,8 +168,8 @@ namespace Wabbajack.VirtualFileSystem Context = context, Name = relPath, Parent = parent, - Size = absPath.Size, - LastModified = absPath.LastModifiedUtc.AsUnixTime(), + Size = extractedFile.Size, + LastModified = extractedFile.LastModifiedUtc.AsUnixTime(), LastAnalyzed = DateTime.Now.AsUnixTime(), Hash = hash }; @@ -178,19 +177,17 @@ namespace Wabbajack.VirtualFileSystem self.FillFullPath(depth); if (context.UseExtendedHashes) - self.ExtendedHashes = ExtendedHashes.FromFile(absPath); + self.ExtendedHashes = ExtendedHashes.FromFile(extractedFile); - if (await FileExtractor.CanExtract(absPath)) - { - await using var tempFolder = Context.GetTemporaryFolder(); - await FileExtractor.ExtractAll(context.Queue, absPath, tempFolder.FullName); + if (!await extractedFile.CanExtract()) return self; - var list = await tempFolder.FullName.EnumerateFiles() - .PMap(context.Queue, - absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), depth + 1)); + await using var extracted = await extractedFile.ExtractAll(context.Queue); - self.Children = list.ToImmutableList(); - } + var list = await extracted + .PMap(context.Queue, + file => Analyze(context, self, file.Value, file.Key, depth + 1)); + + self.Children = list.ToImmutableList(); return self; } @@ -320,9 +317,9 @@ namespace Wabbajack.VirtualFileSystem return path; } - public FileStream OpenRead() + public Stream OpenRead() { - return StagedPath.OpenRead(); + return StagedFile.OpenRead(); } } @@ -333,7 +330,7 @@ namespace Wabbajack.VirtualFileSystem public string MD5 { get; set; } public string CRC { get; set; } - public static ExtendedHashes FromFile(AbsolutePath file) + public static ExtendedHashes FromFile(IExtractedFile file) { var hashes = new ExtendedHashes(); using (var stream = file.OpenRead()) diff --git a/Wabbajack/View Models/MainWindowVM.cs b/Wabbajack/View Models/MainWindowVM.cs index 27e5c630..273041c2 100644 --- a/Wabbajack/View Models/MainWindowVM.cs +++ b/Wabbajack/View Models/MainWindowVM.cs @@ -84,7 +84,7 @@ namespace Wabbajack .ObserveOnGuiThread() .SelectTask(async msg => { - using var _ = await singleton_lock.Wait(); + using var _ = await singleton_lock.WaitAsync(); try { await UserInterventionHandlers.Handle(msg);