diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 32efdec1..05a2264f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -31,7 +31,13 @@ jobs: - name: Set Permissions if: runner.os != 'Windows' run: chmod -R +x Wabbajack.FileExtractor/Extractors - + + - name: Setup .NET Core SDK 8.0.x + uses: actions/setup-dotnet@v1 + with: + dotnet-version: '8.0.x' + include-prerelease: true + - name: Setup .NET Core SDK 7.0.x uses: actions/setup-dotnet@v1 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index f6a08e81..1fb7d354 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ### Changelog -#### Version - 3.3.0.0 - TBA +#### Version - 3.3.0.0 - 10/13/2023 * Fixed some UI issues arising from 3.2.0.0 changes - more informative error text, wiki link button * Added optional JSON flag for `DisplayVersionOnlyInInstallerView` to enable the installer image to only show version number. * Fixed manual downloader downloading in the OS's "Downloads" folder diff --git a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj index af71c310..31ef5b3a 100644 --- a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj +++ b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj @@ -2,10 +2,10 @@ WinExe - net7.0-windows + net8.0-windows true x64 - win10-x64 + win-x64 $(VERSION) $(VERSION) $(VERSION) diff --git a/Wabbajack.CLI.Builder/Wabbajack.CLI.Builder.csproj b/Wabbajack.CLI.Builder/Wabbajack.CLI.Builder.csproj index 87e50f36..2e823132 100644 --- a/Wabbajack.CLI.Builder/Wabbajack.CLI.Builder.csproj +++ b/Wabbajack.CLI.Builder/Wabbajack.CLI.Builder.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable enable diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index fe9ebfc9..27c0da7c 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -13,7 +13,7 @@ CS8600 CS8601 CS8618 - net7.0 + net8.0 diff --git a/Wabbajack.Common/IEnumerableExtensions.cs b/Wabbajack.Common/IEnumerableExtensions.cs index ee8141fa..3958df0d 100644 --- a/Wabbajack.Common/IEnumerableExtensions.cs +++ b/Wabbajack.Common/IEnumerableExtensions.cs @@ -58,7 +58,14 @@ public static class IEnumerableExtensions return data; } - public static IEnumerable> Partition(this IEnumerable coll, int size) + /// + /// Splits the collection into `size` parts + /// + /// + /// + /// + /// + public static IEnumerable> Partition(this IEnumerable coll, int count) { var asList = coll.ToList(); @@ -70,7 +77,30 @@ public static class IEnumerableExtensions } } - return Enumerable.Range(0, size).Select(offset => SkipEnumerable(asList, offset, size)); + return Enumerable.Range(0, count).Select(offset => SkipEnumerable(asList, offset, count)); + } + + /// + /// Split the collection into `size` parts + /// + /// + /// + /// + /// + public static IEnumerable> Batch(this IEnumerable coll, int size) + { + List current = new(); + foreach (var itm in coll) + { + current.Add(itm); + if (current.Count == size) + { + yield return current; + current = new List(); + } + } + if (current.Count > 0) + yield return current; } diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index a382ae26..0f0a7737 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable GPL-3.0-or-later $(VERSION) diff --git a/Wabbajack.Compiler.Test/Wabbajack.Compiler.Test.csproj b/Wabbajack.Compiler.Test/Wabbajack.Compiler.Test.csproj index b9b77692..ac61b703 100644 --- a/Wabbajack.Compiler.Test/Wabbajack.Compiler.Test.csproj +++ b/Wabbajack.Compiler.Test/Wabbajack.Compiler.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 false diff --git a/Wabbajack.Compiler/Wabbajack.Compiler.csproj b/Wabbajack.Compiler/Wabbajack.Compiler.csproj index 9ca4cf58..09c34ed7 100644 --- a/Wabbajack.Compiler/Wabbajack.Compiler.csproj +++ b/Wabbajack.Compiler/Wabbajack.Compiler.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/Wabbajack.Compression.BSA.Test/Wabbajack.Compression.BSA.Test.csproj b/Wabbajack.Compression.BSA.Test/Wabbajack.Compression.BSA.Test.csproj index 270a509f..7e05fc46 100644 --- a/Wabbajack.Compression.BSA.Test/Wabbajack.Compression.BSA.Test.csproj +++ b/Wabbajack.Compression.BSA.Test/Wabbajack.Compression.BSA.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 false diff --git a/Wabbajack.Compression.BSA/Wabbajack.Compression.BSA.csproj b/Wabbajack.Compression.BSA/Wabbajack.Compression.BSA.csproj index d9d11284..4e1d94e7 100644 --- a/Wabbajack.Compression.BSA/Wabbajack.Compression.BSA.csproj +++ b/Wabbajack.Compression.BSA/Wabbajack.Compression.BSA.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable true GPL-3.0-or-later diff --git a/Wabbajack.Compression.Zip.Test/Wabbajack.Compression.Zip.Test.csproj b/Wabbajack.Compression.Zip.Test/Wabbajack.Compression.Zip.Test.csproj index 6d15e005..db8aa67d 100644 --- a/Wabbajack.Compression.Zip.Test/Wabbajack.Compression.Zip.Test.csproj +++ b/Wabbajack.Compression.Zip.Test/Wabbajack.Compression.Zip.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable false diff --git a/Wabbajack.Compression.Zip/Wabbajack.Compression.Zip.csproj b/Wabbajack.Compression.Zip/Wabbajack.Compression.Zip.csproj index 01178a67..5a6bf79a 100644 --- a/Wabbajack.Compression.Zip/Wabbajack.Compression.Zip.csproj +++ b/Wabbajack.Compression.Zip/Wabbajack.Compression.Zip.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable enable diff --git a/Wabbajack.Configuration/Wabbajack.Configuration.csproj b/Wabbajack.Configuration/Wabbajack.Configuration.csproj index cfadb03d..30402ac0 100644 --- a/Wabbajack.Configuration/Wabbajack.Configuration.csproj +++ b/Wabbajack.Configuration/Wabbajack.Configuration.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable enable diff --git a/Wabbajack.DTOs.ConverterGenerators/Wabbajack.DTOs.ConverterGenerators.csproj b/Wabbajack.DTOs.ConverterGenerators/Wabbajack.DTOs.ConverterGenerators.csproj index 3831f9b2..1e804782 100644 --- a/Wabbajack.DTOs.ConverterGenerators/Wabbajack.DTOs.ConverterGenerators.csproj +++ b/Wabbajack.DTOs.ConverterGenerators/Wabbajack.DTOs.ConverterGenerators.csproj @@ -2,7 +2,7 @@ Exe - net7.0 + net8.0 GPL-3.0-or-later $(VERSION) CS8600 diff --git a/Wabbajack.DTOs.Test/Wabbajack.DTOs.Test.csproj b/Wabbajack.DTOs.Test/Wabbajack.DTOs.Test.csproj index 6bc36413..b0eeac6f 100644 --- a/Wabbajack.DTOs.Test/Wabbajack.DTOs.Test.csproj +++ b/Wabbajack.DTOs.Test/Wabbajack.DTOs.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 false diff --git a/Wabbajack.DTOs/Wabbajack.DTOs.csproj b/Wabbajack.DTOs/Wabbajack.DTOs.csproj index 04b1cc65..ad79c9a1 100644 --- a/Wabbajack.DTOs/Wabbajack.DTOs.csproj +++ b/Wabbajack.DTOs/Wabbajack.DTOs.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj b/Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj index 02afa7a1..1030ef69 100644 --- a/Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj +++ b/Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable enable diff --git a/Wabbajack.Downloaders.Dispatcher.Test/Wabbajack.Downloaders.Dispatcher.Test.csproj b/Wabbajack.Downloaders.Dispatcher.Test/Wabbajack.Downloaders.Dispatcher.Test.csproj index 9070839d..85f9a56f 100644 --- a/Wabbajack.Downloaders.Dispatcher.Test/Wabbajack.Downloaders.Dispatcher.Test.csproj +++ b/Wabbajack.Downloaders.Dispatcher.Test/Wabbajack.Downloaders.Dispatcher.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 false diff --git a/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj b/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj index 7cebb553..5dc9ecb4 100644 --- a/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj +++ b/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/Wabbajack.Downloaders.GameFile/Wabbajack.Downloaders.GameFile.csproj b/Wabbajack.Downloaders.GameFile/Wabbajack.Downloaders.GameFile.csproj index 598216a2..8f005484 100644 --- a/Wabbajack.Downloaders.GameFile/Wabbajack.Downloaders.GameFile.csproj +++ b/Wabbajack.Downloaders.GameFile/Wabbajack.Downloaders.GameFile.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable enable diff --git a/Wabbajack.Downloaders.GoogleDrive/Wabbajack.Downloaders.GoogleDrive.csproj b/Wabbajack.Downloaders.GoogleDrive/Wabbajack.Downloaders.GoogleDrive.csproj index a4637175..c22e139b 100644 --- a/Wabbajack.Downloaders.GoogleDrive/Wabbajack.Downloaders.GoogleDrive.csproj +++ b/Wabbajack.Downloaders.GoogleDrive/Wabbajack.Downloaders.GoogleDrive.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/Wabbajack.Downloaders.Http/Wabbajack.Downloaders.Http.csproj b/Wabbajack.Downloaders.Http/Wabbajack.Downloaders.Http.csproj index e0fbecdb..00088175 100644 --- a/Wabbajack.Downloaders.Http/Wabbajack.Downloaders.Http.csproj +++ b/Wabbajack.Downloaders.Http/Wabbajack.Downloaders.Http.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/Wabbajack.Downloaders.IPS4OAuth2Downloader/Wabbajack.Downloaders.IPS4OAuth2Downloader.csproj b/Wabbajack.Downloaders.IPS4OAuth2Downloader/Wabbajack.Downloaders.IPS4OAuth2Downloader.csproj index 6fdce2ea..90fc1e29 100644 --- a/Wabbajack.Downloaders.IPS4OAuth2Downloader/Wabbajack.Downloaders.IPS4OAuth2Downloader.csproj +++ b/Wabbajack.Downloaders.IPS4OAuth2Downloader/Wabbajack.Downloaders.IPS4OAuth2Downloader.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable diff --git a/Wabbajack.Downloaders.Interfaces/Wabbajack.Downloaders.Interfaces.csproj b/Wabbajack.Downloaders.Interfaces/Wabbajack.Downloaders.Interfaces.csproj index d3acee8e..3cd979ce 100644 --- a/Wabbajack.Downloaders.Interfaces/Wabbajack.Downloaders.Interfaces.csproj +++ b/Wabbajack.Downloaders.Interfaces/Wabbajack.Downloaders.Interfaces.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable GPL-3.0-or-later $(VERSION) diff --git a/Wabbajack.Downloaders.Manual/Wabbajack.Downloaders.Manual.csproj b/Wabbajack.Downloaders.Manual/Wabbajack.Downloaders.Manual.csproj index 575e7e12..6eabc26a 100644 --- a/Wabbajack.Downloaders.Manual/Wabbajack.Downloaders.Manual.csproj +++ b/Wabbajack.Downloaders.Manual/Wabbajack.Downloaders.Manual.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable enable diff --git a/Wabbajack.Downloaders.MediaFire/Wabbajack.Downloaders.MediaFire.csproj b/Wabbajack.Downloaders.MediaFire/Wabbajack.Downloaders.MediaFire.csproj index 883ad2d0..e30a6781 100644 --- a/Wabbajack.Downloaders.MediaFire/Wabbajack.Downloaders.MediaFire.csproj +++ b/Wabbajack.Downloaders.MediaFire/Wabbajack.Downloaders.MediaFire.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable diff --git a/Wabbajack.Downloaders.Mega/Wabbajack.Downloaders.Mega.csproj b/Wabbajack.Downloaders.Mega/Wabbajack.Downloaders.Mega.csproj index 2c58d3a0..3c995fe8 100644 --- a/Wabbajack.Downloaders.Mega/Wabbajack.Downloaders.Mega.csproj +++ b/Wabbajack.Downloaders.Mega/Wabbajack.Downloaders.Mega.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable diff --git a/Wabbajack.Downloaders.ModDB/Wabbajack.Downloaders.ModDB.csproj b/Wabbajack.Downloaders.ModDB/Wabbajack.Downloaders.ModDB.csproj index d32eb433..56aa86c5 100644 --- a/Wabbajack.Downloaders.ModDB/Wabbajack.Downloaders.ModDB.csproj +++ b/Wabbajack.Downloaders.ModDB/Wabbajack.Downloaders.ModDB.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable diff --git a/Wabbajack.Downloaders.Nexus/Wabbajack.Downloaders.Nexus.csproj b/Wabbajack.Downloaders.Nexus/Wabbajack.Downloaders.Nexus.csproj index 6fc137e1..31c94a03 100644 --- a/Wabbajack.Downloaders.Nexus/Wabbajack.Downloaders.Nexus.csproj +++ b/Wabbajack.Downloaders.Nexus/Wabbajack.Downloaders.Nexus.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable GPL-3.0-or-later $(VERSION) diff --git a/Wabbajack.Downloaders.VerificationCache/Wabbajack.Downloaders.VerificationCache.csproj b/Wabbajack.Downloaders.VerificationCache/Wabbajack.Downloaders.VerificationCache.csproj index 864b1a37..8be4fe8f 100644 --- a/Wabbajack.Downloaders.VerificationCache/Wabbajack.Downloaders.VerificationCache.csproj +++ b/Wabbajack.Downloaders.VerificationCache/Wabbajack.Downloaders.VerificationCache.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable enable diff --git a/Wabbajack.Downloaders.WabbajackCDN/Wabbajack.Downloaders.WabbajackCDN.csproj b/Wabbajack.Downloaders.WabbajackCDN/Wabbajack.Downloaders.WabbajackCDN.csproj index c9a9c634..f0dd21e1 100644 --- a/Wabbajack.Downloaders.WabbajackCDN/Wabbajack.Downloaders.WabbajackCDN.csproj +++ b/Wabbajack.Downloaders.WabbajackCDN/Wabbajack.Downloaders.WabbajackCDN.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/Wabbajack.Downloaders.WabbajackCDN/WabbajackCDNDownloader.cs b/Wabbajack.Downloaders.WabbajackCDN/WabbajackCDNDownloader.cs index 408941ae..95d265c3 100644 --- a/Wabbajack.Downloaders.WabbajackCDN/WabbajackCDNDownloader.cs +++ b/Wabbajack.Downloaders.WabbajackCDN/WabbajackCDNDownloader.cs @@ -80,33 +80,36 @@ public class WabbajackCDNDownloader : ADownloader, IUrlDownloader, var definition = (await GetDefinition(state, token))!; await using var fs = destination.Open(FileMode.Create, FileAccess.Write, FileShare.None); - await definition.Parts.PMapAll(async part => + await definition.Parts.PMapAll(async part => { - using var partJob = await _limiter.Begin( - $"Downloading {definition.MungedName} ({part.Index}/{definition.Size})", - part.Size, token); - var msg = MakeMessage(new Uri(state.Url + $"/parts/{part.Index}")); - using var response = await _client.SendAsync(msg, HttpCompletionOption.ResponseHeadersRead, token); - if (!response.IsSuccessStatusCode) - throw new InvalidDataException($"Bad response for part request for part {part.Index}"); - - var length = response.Content.Headers.ContentLength; - if (length != part.Size) - throw new InvalidDataException( - $"Bad part size, expected {part.Size} got {length} for part {part.Index}"); - - await using var data = await response.Content.ReadAsStreamAsync(token); - - var ms = new MemoryStream(); - var hash = await data.HashingCopy(ms, token, partJob); - ms.Position = 0; - if (hash != part.Hash) + return await CircuitBreaker.WithAutoRetryAllAsync<(MemoryStream, PartDefinition)>(_logger, async () => { - throw new Exception( - $"Invalid part hash {part.Index} got {hash} instead of {part.Hash} for {definition.MungedName}"); - } + using var partJob = await _limiter.Begin( + $"Downloading {definition.MungedName} ({part.Index}/{definition.Size})", + part.Size, token); + var msg = MakeMessage(new Uri(state.Url + $"/parts/{part.Index}")); + using var response = await _client.SendAsync(msg, HttpCompletionOption.ResponseHeadersRead, token); + if (!response.IsSuccessStatusCode) + throw new InvalidDataException($"Bad response for part request for part {part.Index}"); - return (ms, part); + var length = response.Content.Headers.ContentLength; + if (length != part.Size) + throw new InvalidDataException( + $"Bad part size, expected {part.Size} got {length} for part {part.Index}"); + + await using var data = await response.Content.ReadAsStreamAsync(token); + + var ms = new MemoryStream(); + var hash = await data.HashingCopy(ms, token, partJob); + ms.Position = 0; + if (hash != part.Hash) + { + throw new Exception( + $"Invalid part hash {part.Index} got {hash} instead of {part.Hash} for {definition.MungedName}"); + } + + return (ms, part); + }); }).Do(async rec => diff --git a/Wabbajack.FileExtractor.Test/Wabbajack.FileExtractor.Test.csproj b/Wabbajack.FileExtractor.Test/Wabbajack.FileExtractor.Test.csproj index 40378dd9..55d9b16f 100644 --- a/Wabbajack.FileExtractor.Test/Wabbajack.FileExtractor.Test.csproj +++ b/Wabbajack.FileExtractor.Test/Wabbajack.FileExtractor.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 false diff --git a/Wabbajack.FileExtractor/Wabbajack.FileExtractor.csproj b/Wabbajack.FileExtractor/Wabbajack.FileExtractor.csproj index 358a3e0a..0874c803 100644 --- a/Wabbajack.FileExtractor/Wabbajack.FileExtractor.csproj +++ b/Wabbajack.FileExtractor/Wabbajack.FileExtractor.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/Wabbajack.Hashing.PHash.Test/Wabbajack.Hashing.PHash.Test.csproj b/Wabbajack.Hashing.PHash.Test/Wabbajack.Hashing.PHash.Test.csproj index 7119afd0..290a02ed 100644 --- a/Wabbajack.Hashing.PHash.Test/Wabbajack.Hashing.PHash.Test.csproj +++ b/Wabbajack.Hashing.PHash.Test/Wabbajack.Hashing.PHash.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 false diff --git a/Wabbajack.Hashing.PHash/Wabbajack.Hashing.PHash.csproj b/Wabbajack.Hashing.PHash/Wabbajack.Hashing.PHash.csproj index 78d738ce..520d2626 100644 --- a/Wabbajack.Hashing.PHash/Wabbajack.Hashing.PHash.csproj +++ b/Wabbajack.Hashing.PHash/Wabbajack.Hashing.PHash.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/Wabbajack.Hashing.xxHash64.Benchmark/Wabbajack.Hashing.xxHash64.Benchmark.csproj b/Wabbajack.Hashing.xxHash64.Benchmark/Wabbajack.Hashing.xxHash64.Benchmark.csproj index 03dff657..ecd4d5b6 100644 --- a/Wabbajack.Hashing.xxHash64.Benchmark/Wabbajack.Hashing.xxHash64.Benchmark.csproj +++ b/Wabbajack.Hashing.xxHash64.Benchmark/Wabbajack.Hashing.xxHash64.Benchmark.csproj @@ -2,7 +2,7 @@ Exe - net7.0 + net8.0 Wabbajac.Hash.xxHash64.Benchmark true diff --git a/Wabbajack.Hashing.xxHash64.Test/Wabbajack.Hashing.xxHash64.Test.csproj b/Wabbajack.Hashing.xxHash64.Test/Wabbajack.Hashing.xxHash64.Test.csproj index 204f9d5f..3bd30960 100644 --- a/Wabbajack.Hashing.xxHash64.Test/Wabbajack.Hashing.xxHash64.Test.csproj +++ b/Wabbajack.Hashing.xxHash64.Test/Wabbajack.Hashing.xxHash64.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 false diff --git a/Wabbajack.Hashing.xxHash64/Wabbajack.Hashing.xxHash64.csproj b/Wabbajack.Hashing.xxHash64/Wabbajack.Hashing.xxHash64.csproj index f26d307e..dc80757e 100644 --- a/Wabbajack.Hashing.xxHash64/Wabbajack.Hashing.xxHash64.csproj +++ b/Wabbajack.Hashing.xxHash64/Wabbajack.Hashing.xxHash64.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/Wabbajack.IO.Async/Wabbajack.IO.Async.csproj b/Wabbajack.IO.Async/Wabbajack.IO.Async.csproj index 6836c680..8a918319 100644 --- a/Wabbajack.IO.Async/Wabbajack.IO.Async.csproj +++ b/Wabbajack.IO.Async/Wabbajack.IO.Async.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable enable diff --git a/Wabbajack.Installer.Test/Wabbajack.Installer.Test.csproj b/Wabbajack.Installer.Test/Wabbajack.Installer.Test.csproj index 590be299..6d27a0ff 100644 --- a/Wabbajack.Installer.Test/Wabbajack.Installer.Test.csproj +++ b/Wabbajack.Installer.Test/Wabbajack.Installer.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 false diff --git a/Wabbajack.Installer/Wabbajack.Installer.csproj b/Wabbajack.Installer/Wabbajack.Installer.csproj index 28277612..d76063bb 100644 --- a/Wabbajack.Installer/Wabbajack.Installer.csproj +++ b/Wabbajack.Installer/Wabbajack.Installer.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj index a47fefab..5336e142 100644 --- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj +++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj @@ -16,7 +16,7 @@ Resources\Icons\wabbajack.ico - net7.0-windows + net8.0-windows diff --git a/Wabbajack.Networking.BethesdaNet/Wabbajack.Networking.BethesdaNet.csproj b/Wabbajack.Networking.BethesdaNet/Wabbajack.Networking.BethesdaNet.csproj index 68bb36ab..ff322dc0 100644 --- a/Wabbajack.Networking.BethesdaNet/Wabbajack.Networking.BethesdaNet.csproj +++ b/Wabbajack.Networking.BethesdaNet/Wabbajack.Networking.BethesdaNet.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable enable diff --git a/Wabbajack.Networking.Discord/Wabbajack.Networking.Discord.csproj b/Wabbajack.Networking.Discord/Wabbajack.Networking.Discord.csproj index c9ab6aab..c05b4024 100644 --- a/Wabbajack.Networking.Discord/Wabbajack.Networking.Discord.csproj +++ b/Wabbajack.Networking.Discord/Wabbajack.Networking.Discord.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable GPL-3.0-or-later $(VERSION) diff --git a/Wabbajack.Networking.GitHub/Wabbajack.Networking.GitHub.csproj b/Wabbajack.Networking.GitHub/Wabbajack.Networking.GitHub.csproj index 40e65542..ebb07835 100644 --- a/Wabbajack.Networking.GitHub/Wabbajack.Networking.GitHub.csproj +++ b/Wabbajack.Networking.GitHub/Wabbajack.Networking.GitHub.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable GPL-3.0-or-later $(VERSION) diff --git a/Wabbajack.Networking.Http.Interfaces/Wabbajack.Networking.Http.Interfaces.csproj b/Wabbajack.Networking.Http.Interfaces/Wabbajack.Networking.Http.Interfaces.csproj index 5037943c..603db214 100644 --- a/Wabbajack.Networking.Http.Interfaces/Wabbajack.Networking.Http.Interfaces.csproj +++ b/Wabbajack.Networking.Http.Interfaces/Wabbajack.Networking.Http.Interfaces.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/Wabbajack.Networking.Http.Test/Wabbajack.Networking.Http.Test.csproj b/Wabbajack.Networking.Http.Test/Wabbajack.Networking.Http.Test.csproj index a20f6ab2..f0762e54 100644 --- a/Wabbajack.Networking.Http.Test/Wabbajack.Networking.Http.Test.csproj +++ b/Wabbajack.Networking.Http.Test/Wabbajack.Networking.Http.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable false diff --git a/Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj b/Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj index 2cf94a56..eb64f515 100644 --- a/Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj +++ b/Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable GPL-3.0-or-later $(VERSION) diff --git a/Wabbajack.Networking.NexusApi.Test/Wabbajack.Networking.NexusApi.Test.csproj b/Wabbajack.Networking.NexusApi.Test/Wabbajack.Networking.NexusApi.Test.csproj index fac1ed50..3b8e6b96 100644 --- a/Wabbajack.Networking.NexusApi.Test/Wabbajack.Networking.NexusApi.Test.csproj +++ b/Wabbajack.Networking.NexusApi.Test/Wabbajack.Networking.NexusApi.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 false diff --git a/Wabbajack.Networking.NexusApi/Wabbajack.Networking.NexusApi.csproj b/Wabbajack.Networking.NexusApi/Wabbajack.Networking.NexusApi.csproj index 500ad9fa..e2de94ee 100644 --- a/Wabbajack.Networking.NexusApi/Wabbajack.Networking.NexusApi.csproj +++ b/Wabbajack.Networking.NexusApi/Wabbajack.Networking.NexusApi.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable GPL-3.0-or-later $(VERSION) diff --git a/Wabbajack.Networking.Steam.Test/Wabbajack.Networking.Steam.Test.csproj b/Wabbajack.Networking.Steam.Test/Wabbajack.Networking.Steam.Test.csproj index c7e6d73b..67e78994 100644 --- a/Wabbajack.Networking.Steam.Test/Wabbajack.Networking.Steam.Test.csproj +++ b/Wabbajack.Networking.Steam.Test/Wabbajack.Networking.Steam.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable false diff --git a/Wabbajack.Networking.Steam/Wabbajack.Networking.Steam.csproj b/Wabbajack.Networking.Steam/Wabbajack.Networking.Steam.csproj index ee840362..2a505ddb 100644 --- a/Wabbajack.Networking.Steam/Wabbajack.Networking.Steam.csproj +++ b/Wabbajack.Networking.Steam/Wabbajack.Networking.Steam.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable enable diff --git a/Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj b/Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj index 55712888..8aaefb95 100644 --- a/Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj +++ b/Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/Wabbajack.Paths.IO.Test/Wabbajack.Paths.IO.Test.csproj b/Wabbajack.Paths.IO.Test/Wabbajack.Paths.IO.Test.csproj index 3948bc11..9283f503 100644 --- a/Wabbajack.Paths.IO.Test/Wabbajack.Paths.IO.Test.csproj +++ b/Wabbajack.Paths.IO.Test/Wabbajack.Paths.IO.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 false diff --git a/Wabbajack.Paths.IO/Wabbajack.Paths.IO.csproj b/Wabbajack.Paths.IO/Wabbajack.Paths.IO.csproj index 0cd88c4d..16988ee3 100644 --- a/Wabbajack.Paths.IO/Wabbajack.Paths.IO.csproj +++ b/Wabbajack.Paths.IO/Wabbajack.Paths.IO.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable GPL-3.0-or-later $(VERSION) diff --git a/Wabbajack.Paths.Test/Wabbajack.Paths.Test.csproj b/Wabbajack.Paths.Test/Wabbajack.Paths.Test.csproj index a5d4fe49..71630d2e 100644 --- a/Wabbajack.Paths.Test/Wabbajack.Paths.Test.csproj +++ b/Wabbajack.Paths.Test/Wabbajack.Paths.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 false diff --git a/Wabbajack.Paths/Wabbajack.Paths.csproj b/Wabbajack.Paths/Wabbajack.Paths.csproj index 1ffbca14..17c403db 100644 --- a/Wabbajack.Paths/Wabbajack.Paths.csproj +++ b/Wabbajack.Paths/Wabbajack.Paths.csproj @@ -1,6 +1,6 @@ - net7.0 + net8.0 enable GPL-3.0-or-later $(VERSION) diff --git a/Wabbajack.RateLimiter.Test/Wabbajack.RateLimiter.Test.csproj b/Wabbajack.RateLimiter.Test/Wabbajack.RateLimiter.Test.csproj index 3db11fcf..d09b9b77 100644 --- a/Wabbajack.RateLimiter.Test/Wabbajack.RateLimiter.Test.csproj +++ b/Wabbajack.RateLimiter.Test/Wabbajack.RateLimiter.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable false diff --git a/Wabbajack.RateLimiter/Wabbajack.RateLimiter.csproj b/Wabbajack.RateLimiter/Wabbajack.RateLimiter.csproj index 444af79a..935ceab3 100644 --- a/Wabbajack.RateLimiter/Wabbajack.RateLimiter.csproj +++ b/Wabbajack.RateLimiter/Wabbajack.RateLimiter.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable diff --git a/Wabbajack.Server.Lib/Wabbajack.Server.Lib.csproj b/Wabbajack.Server.Lib/Wabbajack.Server.Lib.csproj index 03942d81..351f19c5 100644 --- a/Wabbajack.Server.Lib/Wabbajack.Server.Lib.csproj +++ b/Wabbajack.Server.Lib/Wabbajack.Server.Lib.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable diff --git a/Wabbajack.Server/AppSettings.cs b/Wabbajack.Server/AppSettings.cs index 5c51ebeb..bbaddce7 100644 --- a/Wabbajack.Server/AppSettings.cs +++ b/Wabbajack.Server/AppSettings.cs @@ -1,4 +1,5 @@ -using Microsoft.Extensions.Configuration; +using Amazon.S3; +using Microsoft.Extensions.Configuration; using Wabbajack.Paths; namespace Wabbajack.BuildServer; @@ -26,8 +27,6 @@ public class AppSettings public string DiscordKey { get; set; } - public string AuthoredFilesFolder { get; set; } - public string PatchesFilesFolder { get; set; } public string MirrorFilesFolder { get; set; } public string NexusCacheFolder { get; set; } @@ -37,6 +36,20 @@ public class AppSettings public CouchDBSetting CesiDB { get; set; } public CouchDBSetting MetricsDB { get; set; } + + public S3Settings S3 { get; set; } +} + +public class S3Settings +{ + public string AccessKey { get; set; } + public string SecretKey { get; set; } + public string ServiceUrl { get; set; } + + public string AuthoredFilesBucket { get; set; } + public string ProxyFilesBucket { get; set; } + + public string AuthoredFilesBucketCache { get; set; } } public class CouchDBSetting diff --git a/Wabbajack.Server/Controllers/AuthorControls.cs b/Wabbajack.Server/Controllers/AuthorControls.cs index 0608c9c6..71cb8c64 100644 --- a/Wabbajack.Server/Controllers/AuthorControls.cs +++ b/Wabbajack.Server/Controllers/AuthorControls.cs @@ -147,7 +147,7 @@ public class AuthorControls : ControllerBase public async Task HomePage() { var user = User.FindFirstValue(ClaimTypes.Name); - var files = (await _authorFiles.AllAuthoredFiles()) + var files = _authorFiles.AllDefinitions .Where(af => af.Definition.Author == user) .Select(af => new { diff --git a/Wabbajack.Server/Controllers/AuthoredFiles.cs b/Wabbajack.Server/Controllers/AuthoredFiles.cs index 6abd80ed..107b7956 100644 --- a/Wabbajack.Server/Controllers/AuthoredFiles.cs +++ b/Wabbajack.Server/Controllers/AuthoredFiles.cs @@ -68,8 +68,7 @@ public class AuthoredFiles : ControllerBase $"Hashes don't match for index {index}. Sizes ({ms.Length} vs {part.Size}). Hashes ({hash} vs {part.Hash}"); ms.Position = 0; - await using var partStream = await _authoredFiles.CreatePart(definition.MungedName, (int)index); - await ms.CopyToAsync(partStream, token); + await _authoredFiles.WritePart(definition.MungedName, (int) index, ms); return Ok(part.Hash.ToBase64()); } @@ -123,7 +122,7 @@ public class AuthoredFiles : ControllerBase public async Task DeleteUpload(string serverAssignedUniqueId) { var user = User.FindFirstValue(ClaimTypes.Name); - var definition = (await _authoredFiles.AllAuthoredFiles()) + var definition = _authoredFiles.AllDefinitions .First(f => f.Definition.ServerAssignedUniqueId == serverAssignedUniqueId) .Definition; if (definition.Author != user) @@ -145,12 +144,12 @@ public class AuthoredFiles : ControllerBase [Route("")] public async Task UploadedFilesGet() { - var files = await _authoredFiles.AllAuthoredFiles(); + var files = _authoredFiles.AllDefinitions + .ToArray(); var response = _authoredFilesTemplate(new { Files = files.OrderByDescending(f => f.Updated).ToArray(), - TotalSpace = _authoredFiles.TotalSpace.Bytes().Humanize("#.##"), - FreeSpace = _authoredFiles.FreeSpace.Bytes().Humanize("#.##") + UsedSpace = _authoredFiles.UsedSpace.Bytes().Humanize("#.##"), }); return new ContentResult { @@ -172,10 +171,13 @@ public class AuthoredFiles : ControllerBase Response.Headers.ContentType = new StringValues("application/octet-stream"); Response.Headers.ContentLength = definition.Size; Response.Headers.ETag = definition.MungedName + "_direct"; - foreach (var part in definition.Parts) + + foreach (var part in definition.Parts.OrderBy(p => p.Index)) { - await using var partStream = await _authoredFiles.StreamForPart(mungedName, (int)part.Index); - await partStream.CopyToAsync(Response.Body); + await _authoredFiles.StreamForPart(mungedName, (int)part.Index, async stream => + { + await stream.CopyToAsync(Response.Body); + }); } } } \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Proxy.cs b/Wabbajack.Server/Controllers/Proxy.cs index eba4175d..918351f5 100644 --- a/Wabbajack.Server/Controllers/Proxy.cs +++ b/Wabbajack.Server/Controllers/Proxy.cs @@ -1,4 +1,7 @@ using System.Text; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; using FluentFTP.Helpers; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; @@ -10,7 +13,9 @@ using Wabbajack.Downloaders.Interfaces; using Wabbajack.DTOs; using Wabbajack.DTOs.DownloadStates; using Wabbajack.Hashing.xxHash64; +using Wabbajack.Paths; using Wabbajack.Paths.IO; +using Wabbajack.RateLimiter; using Wabbajack.VFS; namespace Wabbajack.Server.Controllers; @@ -24,46 +29,44 @@ public class Proxy : ControllerBase private readonly TemporaryFileManager _tempFileManager; private readonly AppSettings _appSettings; private readonly FileHashCache _hashCache; + private readonly IAmazonS3 _s3; + private readonly string _bucket; + + private string _redirectUrl = "https://proxy.wabbajack.org/"; + private readonly IResource _resource; - public Proxy(ILogger logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager, FileHashCache hashCache, AppSettings appSettings) + public Proxy(ILogger logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager, + FileHashCache hashCache, AppSettings appSettings, IAmazonS3 s3, IResource resource) { _logger = logger; _dispatcher = dispatcher; _tempFileManager = tempFileManager; _appSettings = appSettings; _hashCache = hashCache; + _s3 = s3; + _bucket = _appSettings.S3.ProxyFilesBucket; + _resource = resource; } [HttpHead] public async Task ProxyHead(CancellationToken token, [FromQuery] Uri uri, [FromQuery] string? name, [FromQuery] string? hash) { - var shouldMatch = hash != null ? Hash.FromHex(hash) : default; - _logger.LogInformation("Got proxy head request for {Uri}", uri); - var state = _dispatcher.Parse(uri); var cacheName = (await Encoding.UTF8.GetBytes(uri.ToString()).Hash()).ToHex(); - var cacheFile = _appSettings.ProxyPath.Combine(cacheName); - - if (!cacheFile.FileExists()) - return NotFound(); - - if (shouldMatch != default) - if (await _hashCache.FileHashCachedAsync(cacheFile, token) != shouldMatch) - return NotFound(); - - return Ok(); - + return new RedirectResult(_redirectUrl + cacheName); } [HttpGet] public async Task ProxyGet(CancellationToken token, [FromQuery] Uri uri, [FromQuery] string? name, [FromQuery] string? hash) { + + Hash hashResult = default; var shouldMatch = hash != null ? Hash.FromHex(hash) : default; _logger.LogInformation("Got proxy request for {Uri}", uri); var state = _dispatcher.Parse(uri); var cacheName = (await Encoding.UTF8.GetBytes(uri.ToString()).Hash()).ToHex(); - var cacheFile = _appSettings.ProxyPath.Combine(cacheName); + var cacheFile = await GetCacheEntry(cacheName); if (state == null) { @@ -84,26 +87,27 @@ public class Proxy : ControllerBase return BadRequest(new {Type = "Downloader is not IProxyable", Downloader = downloader.GetType().FullName}); } - if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(4)) + if (cacheFile != null && (DateTime.UtcNow - cacheFile.LastModified) > TimeSpan.FromHours(4)) { try { var verify = await _dispatcher.Verify(archive, token); if (verify) - cacheFile.Touch(); + await TouchCacheEntry(cacheName); } catch (Exception ex) { - _logger.LogInformation(ex, "When trying to verify cached file ({Hash}) {Url}", cacheFile.FileName, uri); - cacheFile.Touch(); + _logger.LogInformation(ex, "When trying to verify cached file ({Hash}) {Url}", + cacheFile.Hash, uri); + await TouchCacheEntry(cacheName); } } - if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(24)) + if (cacheFile != null && (DateTime.Now - cacheFile.LastModified) > TimeSpan.FromHours(24)) { try { - cacheFile.Delete(); + await DeleteCacheEntry(cacheName); } catch (Exception ex) { @@ -112,18 +116,15 @@ public class Proxy : ControllerBase } - if (cacheFile.FileExists()) + var redirectUrl = _redirectUrl + cacheName + "?response-content-disposition=attachment;filename=" + (name ?? "unknown"); + if (cacheFile != null) { if (hash != default) { - var hashResult = await _hashCache.FileHashCachedAsync(cacheFile, token); - if (hashResult != shouldMatch) + if (cacheFile.Hash != shouldMatch) return BadRequest(new {Type = "Unmatching Hashes", Expected = shouldMatch.ToHex(), Found = hashResult.ToHex()}); } - var ret = new PhysicalFileResult(cacheFile.ToString(), "application/octet-stream"); - if (name != null) - ret.FileDownloadName = name; - return ret; + return new RedirectResult(redirectUrl); } _logger.LogInformation("Downloading proxy request for {Uri}", uri); @@ -131,38 +132,97 @@ public class Proxy : ControllerBase var tempFile = _tempFileManager.CreateFile(deleteOnDispose:false); var proxyDownloader = _dispatcher.Downloader(archive) as IProxyable; - await using (var of = tempFile.Path.Open(FileMode.Create, FileAccess.Write, FileShare.None)) - { - Response.StatusCode = 200; - if (name != null) - { - Response.Headers.Add(HeaderNames.ContentDisposition, $"attachment; filename=\"{name}\""); - } - Response.Headers.Add( HeaderNames.ContentType, "application/octet-stream" ); - - var result = await proxyDownloader!.DownloadStream(archive, async s => { - return await s.HashingCopy(async m => - { - var strmA = of.WriteAsync(m, token); - await Response.Body.WriteAsync(m, token); - await Response.Body.FlushAsync(token); - await strmA; - }, token); }, - token); - - - if (hash != default && result != shouldMatch) + using var job = await _resource.Begin("Downloading file", 0, token); + hashResult = await proxyDownloader!.Download(archive, tempFile.Path, job, token); + + + if (hash != default && hashResult != shouldMatch) + { + if (tempFile.Path.FileExists()) + tempFile.Path.Delete(); + return NotFound(); + } + + await PutCacheEntry(tempFile.Path, cacheName, hashResult); + + _logger.LogInformation("Returning proxy request for {Uri}", uri); + return new RedirectResult(redirectUrl); + } + + private async Task GetCacheEntry(string name) + { + GetObjectMetadataResponse info; + try + { + info = await _s3.GetObjectMetadataAsync(new GetObjectMetadataRequest() { - if (tempFile.Path.FileExists()) - tempFile.Path.Delete(); - } + BucketName = _bucket, + Key = name, + }); + } + catch (Exception _) + { + return null; } + if (info.HttpStatusCode == System.Net.HttpStatusCode.NotFound) + return null; + + if (info.Metadata["WJ-Hash"] == null) + return null; + + if (!Hash.TryGetFromHex(info.Metadata["WJ-Hash"], out var hash)) + return null; + + return new CacheStatus + { + LastModified = info.LastModified, + Size = info.ContentLength, + Hash = hash + }; + } + + private async Task TouchCacheEntry(string name) + { + await _s3.CopyObjectAsync(new CopyObjectRequest() + { + SourceBucket = _bucket, + DestinationBucket = _bucket, + SourceKey = name, + DestinationKey = name, + MetadataDirective = S3MetadataDirective.REPLACE, + }); + } + + private async Task PutCacheEntry(AbsolutePath path, string name, Hash hash) + { + var obj = new PutObjectRequest + { + BucketName = _bucket, + Key = name, + FilePath = path.ToString(), + ContentType = "application/octet-stream", + DisablePayloadSigning = true + }; + obj.Metadata.Add("WJ-Hash", hash.ToHex()); + await _s3.PutObjectAsync(obj); + } + + private async Task DeleteCacheEntry(string name) + { + await _s3.DeleteObjectAsync(new DeleteObjectRequest + { + BucketName = _bucket, + Key = name + }); + } - await tempFile.Path.MoveToAsync(cacheFile, true, token); - - _logger.LogInformation("Returning proxy request for {Uri} {Size}", uri, cacheFile.Size().FileSizeToString()); - return new EmptyResult(); + record CacheStatus + { + public DateTime LastModified { get; init; } + public long Size { get; init; } + + public Hash Hash { get; init; } } } \ No newline at end of file diff --git a/Wabbajack.Server/DataModels/AuthorFiles.cs b/Wabbajack.Server/DataModels/AuthorFiles.cs index 5f76efb9..a4c3bc6c 100644 --- a/Wabbajack.Server/DataModels/AuthorFiles.cs +++ b/Wabbajack.Server/DataModels/AuthorFiles.cs @@ -1,6 +1,11 @@ +using System.Collections.Concurrent; +using System.Diagnostics; using System.IO.Compression; using System.Web; +using Amazon.S3; +using Amazon.S3.Model; using Microsoft.Extensions.Logging; +using Microsoft.IO; using Wabbajack.BuildServer; using Wabbajack.Common; using Wabbajack.DTOs.CDN; @@ -16,88 +21,213 @@ public class AuthorFiles private readonly ILogger _logger; private readonly AppSettings _settings; private readonly DTOSerializer _dtos; - private Dictionary _byServerId = new(); + private ConcurrentDictionary _byServerId = new(); + private readonly IAmazonS3 _s3; + private readonly ConcurrentDictionary _fileCache; + private readonly string _bucketName; + private ConcurrentDictionary _allObjects = new(); + private HashSet _mangledNames; + private readonly RecyclableMemoryStreamManager _streamPool; + private readonly HttpClient _httpClient; + private readonly AbsolutePath _cacheFile; - public AbsolutePath AuthorFilesLocation => _settings.AuthoredFilesFolder.ToAbsolutePath(); - - public AuthorFiles(ILogger logger, AppSettings settings, DTOSerializer dtos) + private Uri _baseUri => new($"https://authored-files.wabbajack.org/"); + + public AuthorFiles(ILogger logger, AppSettings settings, DTOSerializer dtos, IAmazonS3 s3, HttpClient client) { + _httpClient = client; + _s3 = s3; _logger = logger; _settings = settings; _dtos = dtos; + _fileCache = new ConcurrentDictionary(); + _bucketName = settings.S3.AuthoredFilesBucket; + _ = PrimeCache(); + _streamPool = new RecyclableMemoryStreamManager(); + _cacheFile = _settings.S3.AuthoredFilesBucketCache.ToAbsolutePath(); } - public IEnumerable AllDefinitions => AuthorFilesLocation.EnumerateFiles("definition.json.gz"); - - /// - /// Total unused space available for authored files - /// - public long FreeSpace => new DriveInfo(AuthorFilesLocation.ToString()).AvailableFreeSpace; - - /// - /// Total space available for authored files - /// - public long TotalSpace => new DriveInfo(AuthorFilesLocation.ToString()).TotalSize; - - /// - /// - /// - /// - - public async Task AllAuthoredFiles() + private async Task PrimeCache() { - var defs = new List(); - foreach (var file in AllDefinitions) + try { - defs.Add(new FileDefinitionMetadata + if (!_cacheFile.FileExists()) { - Definition = await ReadDefinition(file), - Updated = file.LastModifiedUtc() + var allObjects = await AllObjects().ToArrayAsync(); + foreach (var obje in allObjects) + { + _allObjects.TryAdd(obje.Key.ToRelativePath(), obje.LastModified.ToFileTimeUtc()); + } + SaveBucketCacheFile(_cacheFile); + } + else + { + LoadBucketCacheFile(_cacheFile); + } + + + _mangledNames = _allObjects + .Where(f => f.Key.EndsWith("definition.json.gz")) + .Select(f => f.Key.Parent) + .ToHashSet(); + + await Parallel.ForEachAsync(_mangledNames, async (name, _) => + { + if (!_allObjects.TryGetValue(name.Combine("definition.json.gz"), out var value)) + return; + + _logger.LogInformation("Priming {Name}", name); + var definition = await PrimeDefinition(name); + var metadata = new FileDefinitionMetadata() + { + Definition = definition, + Updated = DateTime.FromFileTimeUtc(value) + }; + _fileCache.TryAdd(definition.MungedName, metadata); + _byServerId.TryAdd(definition.ServerAssignedUniqueId!, definition); }); + + _logger.LogInformation("Finished priming cache, {Count} files {Size} GB cached", _fileCache.Count, + _fileCache.Sum(s => s.Value.Definition.Size) / (1024 * 1024 * 1024)); + + } + catch (Exception ex) + { + _logger.LogCritical(ex, "Failed to prime cache"); + } + } + + private void SaveBucketCacheFile(AbsolutePath cacheFile) + { + using var file = cacheFile.Open(FileMode.Create, FileAccess.Write); + using var sw = new StreamWriter(file); + foreach(var entry in _allObjects) + { + sw.WriteLine($"{entry.Key}||{entry.Value}"); + } + } + + private void LoadBucketCacheFile(AbsolutePath cacheFile) + { + using var file = cacheFile.Open(FileMode.Open, FileAccess.Read); + using var sr = new StreamReader(file); + while (!sr.EndOfStream) + { + var line = sr.ReadLine(); + var parts = line!.Split("||"); + _allObjects.TryAdd(parts[0].ToRelativePath(), long.Parse(parts[1])); + } + } + + private async Task PrimeDefinition(RelativePath name) + { + return await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () => + { + var uri = _baseUri + $"{name}/definition.json.gz"; + using var response = await _httpClient.GetAsync(uri); + return await ReadDefinition(await response.Content.ReadAsStreamAsync()); + }); + } + + private async IAsyncEnumerable AllObjects() + { + var sw = Stopwatch.StartNew(); + var total = 0; + _logger.Log(LogLevel.Information, "Listing all objects in S3"); + var results = await _s3.ListObjectsV2Async(new ListObjectsV2Request() + { + BucketName = _bucketName, + }); + TOP: + total += results.S3Objects.Count; + _logger.Log(LogLevel.Information, "Got {S3ObjectsCount} objects, {Total} total", results.S3Objects.Count, total); + foreach (var result in results.S3Objects) + { + yield return result; } - _byServerId = defs.ToDictionary(f => f.Definition.ServerAssignedUniqueId!, f => f.Definition); - return defs.ToArray(); + if (results.IsTruncated) + { + results = await _s3.ListObjectsV2Async(new ListObjectsV2Request + { + ContinuationToken = results.NextContinuationToken, + BucketName = _bucketName, + }); + goto TOP; + } + _logger.LogInformation("Finished listing all objects in S3 in {Elapsed}", sw.Elapsed); } - public async Task StreamForPart(string mungedName, int part) + public IEnumerable AllDefinitions => _fileCache.Values; + + /// + /// Used space in bytes + /// + public long UsedSpace => _fileCache.Sum(s => s.Value.Definition.Size); + + public async Task StreamForPart(string mungedName, int part, Func func) { - return AuthorFilesLocation.Combine(mungedName, "parts", part.ToString()).Open(FileMode.Open); + var definition = _fileCache[mungedName].Definition; + + if (part >= definition.Parts.Length) + throw new ArgumentOutOfRangeException(nameof(part)); + + var uri = _baseUri + $"{mungedName}/parts/{part}"; + using var response = await _httpClient.GetAsync(uri); + await func(await response.Content.ReadAsStreamAsync()); } - public async Task CreatePart(string mungedName, int part) + public async Task WritePart(string mungedName, int part, Stream ms) { - return AuthorFilesLocation.Combine(mungedName, "parts", part.ToString()).Open(FileMode.Create, FileAccess.Write, FileShare.None); + await _s3.PutObjectAsync(new PutObjectRequest + { + BucketName = _bucketName, + Key = mungedName.ToRelativePath().Combine("parts", part.ToString()).ToString().Replace("\\", "/"), + InputStream = ms, + DisablePayloadSigning = true, + ContentType = "application/octet-stream" + }); } public async Task WriteDefinition(FileDefinition definition) { - var path = AuthorFilesLocation.Combine(definition.MungedName, "definition.json.gz"); - path.Parent.CreateDirectory(); - path.Parent.Combine("parts").CreateDirectory(); - await using var ms = new MemoryStream(); await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true)) { await _dtos.Serialize(definition, gz); } - - await path.WriteAllBytesAsync(ms.ToArray()); + ms.Position = 0; + + await _s3.PutObjectAsync(new PutObjectRequest + { + BucketName = _bucketName, + Key = definition.MungedName.ToRelativePath().Combine("definition.json.gz").ToString().Replace("\\", "/"), + InputStream = ms, + DisablePayloadSigning = true, + ContentType = "application/octet-stream" + }); + _fileCache.TryAdd(definition.MungedName, new FileDefinitionMetadata + { + Definition = definition, + Updated = DateTime.UtcNow + }); + _byServerId.TryAdd(definition.ServerAssignedUniqueId!, definition); } public async Task ReadDefinition(string mungedName) { - return await ReadDefinition(AuthorFilesLocation.Combine(mungedName, "definition.json.gz")); + return _fileCache[mungedName].Definition; } public bool IsDefinition(string mungedName) { - return AuthorFilesLocation.Combine(mungedName, "definition.json.gz").FileExists(); + return _fileCache.ContainsKey(mungedName); } - private async Task ReadDefinition(AbsolutePath file) + + private async Task ReadDefinition(Stream stream) { - var gz = new GZipStream(new MemoryStream(await file.ReadAllBytesAsync()), CompressionMode.Decompress); + var gz = new GZipStream(stream, CompressionMode.Decompress); var definition = (await _dtos.DeserializeAsync(gz))!; return definition; } @@ -111,15 +241,33 @@ public class AuthorFiles public async Task DeleteFile(FileDefinition definition) { - var folder = AuthorFilesLocation.Combine(definition.MungedName); - folder.DeleteDirectory(); + var allFiles = _allObjects.Where(f => f.Key.TopParent.ToString() == definition.MungedName) + .Select(f => f.Key).ToList(); + foreach (var batch in allFiles.Batch(512)) + { + var batchedArray = batch.ToHashSet(); + _logger.LogInformation("Deleting {Count} files for prefix {Prefix}", batchedArray.Count, definition.MungedName); + await _s3.DeleteObjectsAsync(new DeleteObjectsRequest + { + BucketName = _bucketName, + + Objects = batchedArray.Select(f => new KeyVersion + { + Key = f.ToString().Replace("\\", "/") + }).ToList() + }); + foreach (var key in batchedArray) + { + _allObjects.TryRemove(key, out _); + } + } + + _byServerId.TryRemove(definition.ServerAssignedUniqueId!, out _); + _fileCache.TryRemove(definition.MungedName, out _); } - public async Task ReadDefinitionForServerId(string serverAssignedUniqueId) + public async ValueTask ReadDefinitionForServerId(string serverAssignedUniqueId) { - if (_byServerId.TryGetValue(serverAssignedUniqueId, out var found)) - return found; - await AllAuthoredFiles(); return _byServerId[serverAssignedUniqueId]; } diff --git a/Wabbajack.Server/Resources/Reports/AuthoredFiles.html b/Wabbajack.Server/Resources/Reports/AuthoredFiles.html index 42a4ed20..a7a216d7 100644 --- a/Wabbajack.Server/Resources/Reports/AuthoredFiles.html +++ b/Wabbajack.Server/Resources/Reports/AuthoredFiles.html @@ -11,7 +11,7 @@

Authored Files:

-

{{$.FreeSpace}} remaining of {{$.TotalSpace}}

+

{{$.UsedSpace}}

@@ -28,7 +28,7 @@ - + {{/each}}
{{$.HumanSize}} {{$.Definition.Author}} {{$.Updated}}(Slow) HTTP Direct Link(Slow) HTTP Direct Link
diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 30ba2c14..d67d0726 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -5,6 +5,9 @@ using System.Runtime.InteropServices; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading.Tasks; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.Util.Internal; using cesi.DTOs; using CouchDB.Driver; using CouchDB.Driver.Options; @@ -22,6 +25,7 @@ using Nettle.Compiler; using Newtonsoft.Json; using Octokit; using Wabbajack.BuildServer; +using Wabbajack.Configuration; using Wabbajack.Downloaders; using Wabbajack.Downloaders.VerificationCache; using Wabbajack.DTOs; @@ -39,10 +43,11 @@ using Wabbajack.Server.Services; using Wabbajack.Services.OSIntegrated.TokenProviders; using Wabbajack.Networking.WabbajackClientApi; using Wabbajack.Paths.IO; -using Wabbajack.Server.DTOs; using Wabbajack.VFS; using YamlDotNet.Serialization.NamingConventions; using Client = Wabbajack.Networking.GitHub.Client; +using Metric = Wabbajack.Server.DTOs.Metric; +using SettingsManager = Wabbajack.Services.OSIntegrated.SettingsManager; namespace Wabbajack.Server; @@ -93,6 +98,16 @@ public class Startup services.AddSingleton(); services.AddAllSingleton(); services.AddDownloadDispatcher(useLoginDownloaders:false, useProxyCache:false); + services.AddSingleton(s => + { + var appSettings = s.GetRequiredService(); + var settings = new BasicAWSCredentials(appSettings.S3.AccessKey, + appSettings.S3.SecretKey); + return new AmazonS3Client(settings, new AmazonS3Config + { + ServiceURL = appSettings.S3.ServiceUrl, + }); + }); services.AddTransient(s => { var settings = s.GetRequiredService(); @@ -141,6 +156,20 @@ public class Startup }); services.AddDTOSerializer(); services.AddDTOConverters(); + + services.AddSingleton(s => new Wabbajack.Services.OSIntegrated.Configuration + { + EncryptedDataLocation = KnownFolders.WabbajackAppLocal.Combine("encrypted"), + ModListsDownloadLocation = KnownFolders.EntryPoint.Combine("downloaded_mod_lists"), + SavedSettingsLocation = KnownFolders.WabbajackAppLocal.Combine("saved_settings"), + LogLocation = KnownFolders.LauncherAwarePath.Combine("logs"), + ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache") + }); + + + services.AddSingleton(); + services.AddSingleton(s => Wabbajack.Services.OSIntegrated.ServiceExtensions.GetAppSettings(s, MainSettings.SettingsFileName)); + services.AddResponseCompression(options => { options.Providers.Add(); @@ -243,5 +272,7 @@ public class Startup // Trigger the internal update code app.ApplicationServices.GetRequiredService(); app.ApplicationServices.GetRequiredService(); + + app.ApplicationServices.GetRequiredService(); } } \ No newline at end of file diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index c31158e1..7f7cffad 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable enable Exe @@ -12,6 +12,7 @@ + @@ -22,6 +23,7 @@ + diff --git a/Wabbajack.Server/appsettings.json b/Wabbajack.Server/appsettings.json index 79381d45..1acb7a81 100644 --- a/Wabbajack.Server/appsettings.json +++ b/Wabbajack.Server/appsettings.json @@ -28,6 +28,14 @@ "Database": "metrics", "Username": "wabbajack", "Password": "password" + }, + "S3": { + "AccessKey": "<>", + "SecretKey": "<>", + "ServiceUrl": "<>", + "ProxyFilesBucket": "proxy-files", + "AuthoredFilesBucket": "authored-files", + "AuthoredFilesBucketCache": "c:\\tmp\\bucket-cache.txt" } }, "AllowedHosts": "*" diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index d89cf4e2..29873e83 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -98,17 +98,7 @@ public static class ServiceExtensions service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount}); - MainSettings GetAppSettings(IServiceProvider provider, string name) - { - var settingsManager = provider.GetRequiredService(); - var settings = Task.Run(() => settingsManager.Load(name)).Result; - if (settings.Upgrade()) - { - settingsManager.Save(MainSettings.SettingsFileName, settings).FireAndForget(); - } - return settings; - } Func> GetResourceSettings(IServiceProvider provider, string name) { @@ -234,6 +224,18 @@ public static class ServiceExtensions return service; } + + public static MainSettings GetAppSettings(IServiceProvider provider, string name) + { + var settingsManager = provider.GetRequiredService(); + var settings = Task.Run(() => settingsManager.Load(name)).Result; + if (settings.Upgrade()) + { + settingsManager.Save(MainSettings.SettingsFileName, settings).FireAndForget(); + } + + return settings; + } private static void CleanAllTempData(AbsolutePath path) { diff --git a/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj b/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj index 2223ea6d..0a61e5a8 100644 --- a/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj +++ b/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/Wabbajack.VFS.Interfaces/Wabbajack.VFS.Interfaces.csproj b/Wabbajack.VFS.Interfaces/Wabbajack.VFS.Interfaces.csproj index 5b854cb5..a7bbfd87 100644 --- a/Wabbajack.VFS.Interfaces/Wabbajack.VFS.Interfaces.csproj +++ b/Wabbajack.VFS.Interfaces/Wabbajack.VFS.Interfaces.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable enable diff --git a/Wabbajack.VFS.Test/Wabbajack.VFS.Test.csproj b/Wabbajack.VFS.Test/Wabbajack.VFS.Test.csproj index 29e3421c..9a11a2dd 100644 --- a/Wabbajack.VFS.Test/Wabbajack.VFS.Test.csproj +++ b/Wabbajack.VFS.Test/Wabbajack.VFS.Test.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 false diff --git a/Wabbajack.VFS/Wabbajack.VFS.csproj b/Wabbajack.VFS/Wabbajack.VFS.csproj index a4501f7c..50dd72e6 100644 --- a/Wabbajack.VFS/Wabbajack.VFS.csproj +++ b/Wabbajack.VFS/Wabbajack.VFS.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable $(VERSION) GPL-3.0-or-later diff --git a/buildall.bat b/buildall.bat index 010056f8..f587ea41 100644 --- a/buildall.bat +++ b/buildall.bat @@ -7,12 +7,12 @@ mkdir c:\tmp\publish-wj dotnet clean dotnet restore -dotnet publish Wabbajack.App.Wpf\Wabbajack.App.Wpf.csproj --runtime win10-x64 --configuration Release /p:Platform=x64 -o c:\tmp\publish-wj\app /p:PublishReadyToRun=true /p:PublishSingleFile=true /p:IncludeNativeLibrariesForSelfExtract=true --self-contained /p:DebugType=embedded -dotnet publish Wabbajack.Launcher\Wabbajack.Launcher.csproj --runtime win10-x64 --configuration Release /p:Platform=x64 -o c:\tmp\publish-wj\launcher /p:PublishReadyToRun=true /p:PublishSingleFile=true /p:IncludeNativeLibrariesForSelfExtract=true --self-contained /p:DebugType=embedded -dotnet publish c:\oss\Wabbajack\Wabbajack.CLI\Wabbajack.CLI.csproj --runtime win10-x64 --configuration Release /p:Platform=x64 -o c:\tmp\publish-wj\app --self-contained /p:DebugType=embedded -"C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64\signtool.exe" sign /t http://timestamp.sectigo.com c:\tmp\publish-wj\app\Wabbajack.exe -"C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64\signtool.exe" sign /t http://timestamp.sectigo.com c:\tmp\publish-wj\launcher\Wabbajack.exe -"C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64\signtool.exe" sign /t http://timestamp.sectigo.com c:\tmp\publish-wj\app\wabbajack-cli.exe +dotnet publish Wabbajack.App.Wpf\Wabbajack.App.Wpf.csproj --runtime win-x64 --configuration Release /p:Platform=x64 -o c:\tmp\publish-wj\app /p:PublishReadyToRun=true /p:PublishSingleFile=true /p:IncludeNativeLibrariesForSelfExtract=true --self-contained /p:DebugType=embedded +dotnet publish Wabbajack.Launcher\Wabbajack.Launcher.csproj --runtime win-x64 --configuration Release /p:Platform=x64 -o c:\tmp\publish-wj\launcher /p:PublishReadyToRun=true /p:PublishSingleFile=true /p:IncludeNativeLibrariesForSelfExtract=true --self-contained /p:DebugType=embedded +dotnet publish c:\oss\Wabbajack\Wabbajack.CLI\Wabbajack.CLI.csproj --runtime win-x64 --configuration Release /p:Platform=x64 -o c:\tmp\publish-wj\app --self-contained /p:DebugType=embedded +"C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64\signtool.exe" sign /fd sha256 /tr http://ts.ssl.com /td sha256 /sha1 8c26a8e0bf3e70eb89721cc4d86a87137153ccba c:\tmp\publish-wj\app\Wabbajack.exe +"C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64\signtool.exe" sign /fd sha256 /tr http://ts.ssl.com /td sha256 /sha1 8c26a8e0bf3e70eb89721cc4d86a87137153ccba c:\tmp\publish-wj\launcher\Wabbajack.exe +"C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64\signtool.exe" sign /fd sha256 /tr http://ts.ssl.com /td sha256 /sha1 8c26a8e0bf3e70eb89721cc4d86a87137153ccba c:\tmp\publish-wj\app\wabbajack-cli.exe "c:\Program Files\7-Zip\7z.exe" a c:\tmp\publish-wj\%VERSION%.zip c:\tmp\publish-wj\app\* copy c:\tmp\publish-wj\launcher\Wabbajack.exe c:\tmp\publish-wj\Wabbajack.exe