From ea76d88012cac0482d0f0d2f9a7ba13bcf0b3428 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 27 May 2020 20:43:57 -0600 Subject: [PATCH] Temp files now implement IAsyncDisposable instead of IDisposable and use AbsolutePath.DeleteAsync --- Compression.BSA/BA2Builder.cs | 2 +- Compression.BSA/BSABuilder.cs | 2 +- Wabbajack.App.Test/FilePickerTests.cs | 14 +++++----- Wabbajack.Common/Util/DiskSlabAllocator.cs | 14 ++++++---- Wabbajack.Common/Util/TempFile.cs | 7 ++--- Wabbajack.Common/Util/TempStream.cs | 4 +-- .../Downloaders/BethesdaNetDownloader.cs | 2 +- .../ArchiveMaintainerTests.cs | 6 ++-- Wabbajack.Server.Test/AuthoredFilesTests.cs | 2 +- Wabbajack.Server.Test/ModlistUpdater.cs | 4 +-- .../Services/ArchiveDownloader.cs | 2 +- Wabbajack.Server/Services/ListValidator.cs | 2 +- Wabbajack.Server/Services/PatchBuilder.cs | 4 +-- Wabbajack.Test/DownloaderTests.cs | 28 +++++++++---------- Wabbajack.Test/RestartingDownloadsTests.cs | 2 +- Wabbajack.Test/SanityTests.cs | 14 +++++----- Wabbajack.VirtualFileSystem/FileExtractor.cs | 6 +++- 17 files changed, 60 insertions(+), 55 deletions(-) diff --git a/Compression.BSA/BA2Builder.cs b/Compression.BSA/BA2Builder.cs index 196df729..c182bd7c 100644 --- a/Compression.BSA/BA2Builder.cs +++ b/Compression.BSA/BA2Builder.cs @@ -37,7 +37,7 @@ namespace Compression.BSA public async ValueTask DisposeAsync() { - _slab.Dispose(); + await _slab.DisposeAsync(); } public async Task AddFile(FileStateObject state, Stream src) diff --git a/Compression.BSA/BSABuilder.cs b/Compression.BSA/BSABuilder.cs index 5ea06ba1..dbc5dc56 100644 --- a/Compression.BSA/BSABuilder.cs +++ b/Compression.BSA/BSABuilder.cs @@ -78,7 +78,7 @@ namespace Compression.BSA public async ValueTask DisposeAsync() { - _slab.Dispose(); + await _slab.DisposeAsync(); } public async Task AddFile(FileStateObject state, Stream src) { diff --git a/Wabbajack.App.Test/FilePickerTests.cs b/Wabbajack.App.Test/FilePickerTests.cs index 3bdb77a5..3dfdd991 100644 --- a/Wabbajack.App.Test/FilePickerTests.cs +++ b/Wabbajack.App.Test/FilePickerTests.cs @@ -47,7 +47,7 @@ namespace Wabbajack.Test public async Task FileNoExistsCheck_Exists() { var vm = new FilePickerVM(); - using (CreateSetFile(vm)) + await using (CreateSetFile(vm)) { vm.PathType = FilePickerVM.PathTypeOptions.File; vm.ExistCheckOption = FilePickerVM.CheckOptions.Off; @@ -76,7 +76,7 @@ namespace Wabbajack.Test public async Task ExistCheckTypeOff_Exists() { var vm = new FilePickerVM(); - using (CreateSetFile(vm)) + await using (CreateSetFile(vm)) { vm.PathType = FilePickerVM.PathTypeOptions.Off; vm.ExistCheckOption = FilePickerVM.CheckOptions.On; @@ -119,7 +119,7 @@ namespace Wabbajack.Test public async Task FileIfNotEmptyCheck_Exists() { var vm = new FilePickerVM(); - using (CreateSetFile(vm)) + await using (CreateSetFile(vm)) { vm.PathType = FilePickerVM.PathTypeOptions.File; vm.ExistCheckOption = FilePickerVM.CheckOptions.IfPathNotEmpty; @@ -148,7 +148,7 @@ namespace Wabbajack.Test public async Task FileOnExistsCheck_Exists() { var vm = new FilePickerVM(); - using (CreateSetFile(vm)) + await using (CreateSetFile(vm)) { vm.PathType = FilePickerVM.PathTypeOptions.File; vm.ExistCheckOption = FilePickerVM.CheckOptions.On; @@ -259,7 +259,7 @@ namespace Wabbajack.Test public async Task FileExistsButSetToFolder() { var vm = new FilePickerVM(); - using (CreateSetFile(vm)) + await using (CreateSetFile(vm)) { vm.PathType = FilePickerVM.PathTypeOptions.Folder; vm.ExistCheckOption = FilePickerVM.CheckOptions.On; @@ -291,7 +291,7 @@ namespace Wabbajack.Test public async Task FileWithFilters_Passes() { var vm = new FilePickerVM(); - using (CreateSetFile(vm)) + await using (CreateSetFile(vm)) { vm.PathType = FilePickerVM.PathTypeOptions.File; vm.ExistCheckOption = FilePickerVM.CheckOptions.Off; @@ -308,7 +308,7 @@ namespace Wabbajack.Test public async Task FileWithFilters_ExistsButFails() { var vm = new FilePickerVM(); - using (CreateSetFile(vm)) + await using (CreateSetFile(vm)) { vm.PathType = FilePickerVM.PathTypeOptions.File; vm.ExistCheckOption = FilePickerVM.CheckOptions.Off; diff --git a/Wabbajack.Common/Util/DiskSlabAllocator.cs b/Wabbajack.Common/Util/DiskSlabAllocator.cs index 89f3346a..5676c6cd 100644 --- a/Wabbajack.Common/Util/DiskSlabAllocator.cs +++ b/Wabbajack.Common/Util/DiskSlabAllocator.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.IO.MemoryMappedFiles; +using System.Threading.Tasks; namespace Wabbajack.Common { @@ -9,13 +10,13 @@ namespace Wabbajack.Common /// Memory allocator that stores data via memory mapping to a on-disk file. Disposing of this object /// deletes the memory mapped file /// - public class DiskSlabAllocator : IDisposable + public class DiskSlabAllocator : IAsyncDisposable { private readonly TempFile _file; private readonly MemoryMappedFile _mmap; private long _head = 0; private readonly FileStream _fileStream; - private List _allocated = new List(); + private List _allocated = new List(); private long _size; public DiskSlabAllocator(long size) @@ -44,12 +45,13 @@ namespace Wabbajack.Common } } - public void Dispose() + public async ValueTask DisposeAsync() { - _allocated.Do(s => s.Dispose()); + foreach (var allocated in _allocated) + await allocated.DisposeAsync(); _mmap.Dispose(); - _fileStream.Dispose(); - _file.Dispose(); + await _fileStream.DisposeAsync(); + await _file.DisposeAsync(); } } } diff --git a/Wabbajack.Common/Util/TempFile.cs b/Wabbajack.Common/Util/TempFile.cs index 0022151b..37145330 100644 --- a/Wabbajack.Common/Util/TempFile.cs +++ b/Wabbajack.Common/Util/TempFile.cs @@ -8,7 +8,7 @@ using AlphaPath = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Common { - public class TempFile : IDisposable + public class TempFile : IAsyncDisposable { public FileInfo File { get; private set; } public AbsolutePath Path => (AbsolutePath)File.FullName; @@ -35,12 +35,11 @@ namespace Wabbajack.Common } this.DeleteAfter = deleteAfter; } - - public void Dispose() + public async ValueTask DisposeAsync() { if (DeleteAfter) { - this.File.Delete(); + await Path.DeleteAsync(); } } } diff --git a/Wabbajack.Common/Util/TempStream.cs b/Wabbajack.Common/Util/TempStream.cs index b65e4027..5618e77c 100644 --- a/Wabbajack.Common/Util/TempStream.cs +++ b/Wabbajack.Common/Util/TempStream.cs @@ -21,13 +21,13 @@ namespace Wabbajack.Common protected override void Dispose(bool disposing) { base.Dispose(disposing); - _file.Dispose(); + _file.DisposeAsync().AsTask().Wait(); } public override async ValueTask DisposeAsync() { await base.DisposeAsync(); - _file.Dispose(); + await _file.DisposeAsync(); } } } diff --git a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs index b23dbce7..d35958a2 100644 --- a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs +++ b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs @@ -148,7 +148,7 @@ namespace Wabbajack.Lib.Downloaders public override async Task Download(Archive a, AbsolutePath destination) { var (client, info, collected) = await ResolveDownloadInfo(); - using var tf = new TempFile(); + await using var tf = new TempFile(); await using var file = tf.File.Create(); var max_chunks = info.depot_list[0].file_list[0].chunk_count; foreach (var chunk in info.depot_list[0].file_list[0].chunk_list.OrderBy(c => c.index)) diff --git a/Wabbajack.Server.Test/ArchiveMaintainerTests.cs b/Wabbajack.Server.Test/ArchiveMaintainerTests.cs index 6906255e..62423b02 100644 --- a/Wabbajack.Server.Test/ArchiveMaintainerTests.cs +++ b/Wabbajack.Server.Test/ArchiveMaintainerTests.cs @@ -16,8 +16,8 @@ namespace Wabbajack.BuildServer.Test public async Task CanIngestFiles() { var maintainer = Fixture.GetService(); - using var tf = new TempFile(); - using var tf2 = new TempFile(); + await using var tf = new TempFile(); + await using var tf2 = new TempFile(); await tf.Path.WriteAllBytesAsync(RandomData(1024)); await tf.Path.CopyToAsync(tf2.Path); @@ -35,7 +35,7 @@ namespace Wabbajack.BuildServer.Test public async Task IngestsExistingFiles() { var maintainer = Fixture.GetService(); - using var tf = new TempFile(); + await using var tf = new TempFile(); await tf.Path.WriteAllBytesAsync(RandomData(1024)); var hash = await tf.Path.FileHashAsync(); diff --git a/Wabbajack.Server.Test/AuthoredFilesTests.cs b/Wabbajack.Server.Test/AuthoredFilesTests.cs index 28b38dc3..5bce8623 100644 --- a/Wabbajack.Server.Test/AuthoredFilesTests.cs +++ b/Wabbajack.Server.Test/AuthoredFilesTests.cs @@ -20,7 +20,7 @@ namespace Wabbajack.BuildServer.Test [Fact] public async Task CanUploadDownloadAndDeleteAuthoredFiles() { - using var file = new TempFile(); + await using var file = new TempFile(); await file.Path.WriteAllBytesAsync(RandomData(Consts.UPLOADED_FILE_BLOCK_SIZE * 4 + Consts.UPLOADED_FILE_BLOCK_SIZE / 3)); var originalHash = await file.Path.FileHashAsync(); diff --git a/Wabbajack.Server.Test/ModlistUpdater.cs b/Wabbajack.Server.Test/ModlistUpdater.cs index 05b932c5..7e1c25ec 100644 --- a/Wabbajack.Server.Test/ModlistUpdater.cs +++ b/Wabbajack.Server.Test/ModlistUpdater.cs @@ -104,7 +104,7 @@ namespace Wabbajack.Server.Test await"TestEndToEndArchiveUpdating.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(newFileData); - using var tempFile = new TempFile(); + await using var tempFile = new TempFile(); var pendingRequest = DownloadDispatcher.DownloadWithPossibleUpgrade(oldArchive, tempFile.Path); for (var times = 0; await downloader.Execute() == 0 && times < 40; times ++) @@ -124,7 +124,7 @@ namespace Wabbajack.Server.Test private async Task IngestData(ArchiveMaintainer am, byte[] data) { - using var f = new TempFile(); + await using var f = new TempFile(); await f.Path.WriteAllBytesAsync(data); await am.Ingest(f.Path); } diff --git a/Wabbajack.Server/Services/ArchiveDownloader.cs b/Wabbajack.Server/Services/ArchiveDownloader.cs index 510b9e16..df39d2d2 100644 --- a/Wabbajack.Server/Services/ArchiveDownloader.cs +++ b/Wabbajack.Server/Services/ArchiveDownloader.cs @@ -60,7 +60,7 @@ namespace Wabbajack.Server.Services _logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"); await DownloadDispatcher.PrepareAll(new[] {nextDownload.Archive.State}); - using var tempPath = new TempFile(); + await using var tempPath = new TempFile(); await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path); var hash = await tempPath.Path.FileHashAsync(); diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 108758f8..0285b708 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -184,7 +184,7 @@ namespace Wabbajack.Server.Services _logger.Log(LogLevel.Information, $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}"); await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}" }); - upgrade.NewFile.Dispose(); + await upgrade.NewFile.DisposeAsync(); return (archive, ArchiveStatus.Updating); } diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index 6eb29e4f..b78a859c 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -68,8 +68,8 @@ namespace Wabbajack.Server.Services var patchName = $"{Consts.ArchiveUpdatesCDNFolder}\\{patch.Src.Archive.Hash.ToHex()}_{patch.Dest.Archive.Hash.ToHex()}"; - using var sigFile = new TempFile(); - using var patchFile = new TempFile(); + await using var sigFile = new TempFile(); + await using var patchFile = new TempFile(); await using var srcStream = await srcPath.OpenShared(); await using var destStream = await destPath.OpenShared(); await using var sigStream = await sigFile.Path.Create(); diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 99f34864..f591f514 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -69,7 +69,7 @@ namespace Wabbajack.Test var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!){Size = 20})); - using var filename = new TempFile(); + await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List{"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k" } })); Assert.False(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List{ "blerg" }})); @@ -99,7 +99,7 @@ namespace Wabbajack.Test var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!){Size = 20})); - using var filename = new TempFile(); + await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?" } })); Assert.False(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "blerg" } })); @@ -129,7 +129,7 @@ namespace Wabbajack.Test var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); - using var filename = new TempFile(); + await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List { "1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_" } })); Assert.False(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List()})); @@ -158,7 +158,7 @@ namespace Wabbajack.Test var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); - using var filename = new TempFile(); + await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "http://build.wabbajack.org/" } })); Assert.False(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); @@ -182,7 +182,7 @@ namespace Wabbajack.Test var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); - using var filename = new TempFile(); + await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "http://build.wabbajack.org/" } })); @@ -212,7 +212,7 @@ namespace Wabbajack.Test var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20 })); - using var filename = new TempFile(); + await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List {"http://www.mediafire.com/file/agiqzm1xwebczpx/"}})); @@ -241,7 +241,7 @@ namespace Wabbajack.Test Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20 })); // Exercise the cache code Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20 })); - using var filename = new TempFile(); + await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); @@ -268,7 +268,7 @@ namespace Wabbajack.Test var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); - using var filename = new TempFile(); + await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); @@ -300,7 +300,7 @@ namespace Wabbajack.Test Assert.False(await converted.Verify(new Archive(state: null!) { Size = 15})); - using var filename = new TempFile(); + await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); await converted.Download(new Archive(state: null!) { Name = "LoversLab Test.txt" }, filename.Path); @@ -327,7 +327,7 @@ namespace Wabbajack.Test */ var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); - using var filename = new TempFile(); + await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); @@ -352,7 +352,7 @@ namespace Wabbajack.Test var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); - using var filename = new TempFile(); + await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); @@ -404,7 +404,7 @@ namespace Wabbajack.Test var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); - using var filename = new TempFile(); + await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); @@ -425,7 +425,7 @@ namespace Wabbajack.Test var ini = $@"[General] directURL=https://bethesda.net/en/mods/skyrim/mod-detail/4145641"; - using var filename = new TempFile(); + await using var filename = new TempFile(); var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); @@ -463,7 +463,7 @@ namespace Wabbajack.Test Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); - using var tempFile = new TempFile(); + await using var tempFile = new TempFile(); await converted.Download(new Archive(state: null!) { Name = "yt_test.zip"}, tempFile.Path); Assert.Equal(Hash.FromBase64("kD36zbA2X9Q="), await tempFile.Path.FileHashAsync()); } diff --git a/Wabbajack.Test/RestartingDownloadsTests.cs b/Wabbajack.Test/RestartingDownloadsTests.cs index 23710795..3bf8b88f 100644 --- a/Wabbajack.Test/RestartingDownloadsTests.cs +++ b/Wabbajack.Test/RestartingDownloadsTests.cs @@ -70,7 +70,7 @@ namespace Wabbajack.Test [Fact] public async Task DownloadResume() { - using var testFile = new TempFile(); + await using var testFile = new TempFile(); using var server = new CrappyRandomServer(); var state = new HTTPDownloader.State($"http://localhost:{server.Port}/foo"); diff --git a/Wabbajack.Test/SanityTests.cs b/Wabbajack.Test/SanityTests.cs index 5337a043..2e810748 100644 --- a/Wabbajack.Test/SanityTests.cs +++ b/Wabbajack.Test/SanityTests.cs @@ -267,8 +267,8 @@ namespace Wabbajack.Test await utils.Configure(); - - using var tempFile = new TempFile(); + + await using var tempFile = new TempFile(); var bsaState = new BSAStateObject { Magic = "BSA\0", Version = 0x69, ArchiveFlags = 0x107, FileFlags = 0x0, @@ -306,8 +306,8 @@ namespace Wabbajack.Test await utils.Configure(); - - using var tempFile = new TempFile(); + + await using var tempFile = new TempFile(); var bsaState = new BSAStateObject { Magic = "BSA\0", Version = 0x69, ArchiveFlags = 0x107, FileFlags = 0x0, @@ -347,8 +347,8 @@ namespace Wabbajack.Test await utils.Configure(); - - using var tempFile = new TempFile(); + + await using var tempFile = new TempFile(); var bsaState = new BSAStateObject { Magic = "BSA\0", Version = 0x69, ArchiveFlags = 0x107, FileFlags = 0x0, @@ -389,7 +389,7 @@ namespace Wabbajack.Test }; // Create the download - using var tempFile = new TempFile(); + await using var tempFile = new TempFile(); await using (var bsa = bsaState.MakeBuilder(1024 * 1024)) { await bsa.AddFile(new BSAFileStateObject diff --git a/Wabbajack.VirtualFileSystem/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor.cs index 7d36226e..d0e99f1c 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor.cs @@ -184,7 +184,11 @@ namespace Wabbajack.VirtualFileSystem Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true); } - tmpFile?.Dispose(); + if (tmpFile != null) + { + await tmpFile.DisposeAsync(); + } + return new ExtractedFiles(dest); }