Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into pre-release

Browse Source 
# Conflicts:
#	CHANGELOG.md
#	Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj
#	Wabbajack.CLI.Builder/Wabbajack.CLI.Builder.csproj
#	Wabbajack.CLI/Wabbajack.CLI.csproj
#	Wabbajack.Common/Wabbajack.Common.csproj
#	Wabbajack.Compiler.Test/Wabbajack.Compiler.Test.csproj
#	Wabbajack.Compiler/Wabbajack.Compiler.csproj
#	Wabbajack.Compression.BSA.Test/Wabbajack.Compression.BSA.Test.csproj
#	Wabbajack.Compression.BSA/Wabbajack.Compression.BSA.csproj
#	Wabbajack.Compression.Zip.Test/Wabbajack.Compression.Zip.Test.csproj
#	Wabbajack.Compression.Zip/Wabbajack.Compression.Zip.csproj
#	Wabbajack.Configuration/Wabbajack.Configuration.csproj
#	Wabbajack.DTOs.ConverterGenerators/Wabbajack.DTOs.ConverterGenerators.csproj
#	Wabbajack.DTOs.Test/Wabbajack.DTOs.Test.csproj
#	Wabbajack.DTOs/Wabbajack.DTOs.csproj
#	Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj
#	Wabbajack.Downloaders.Dispatcher.Test/Wabbajack.Downloaders.Dispatcher.Test.csproj
#	Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj
#	Wabbajack.Downloaders.GameFile/Wabbajack.Downloaders.GameFile.csproj
#	Wabbajack.Downloaders.GoogleDrive/Wabbajack.Downloaders.GoogleDrive.csproj
#	Wabbajack.Downloaders.Http/Wabbajack.Downloaders.Http.csproj
#	Wabbajack.Downloaders.IPS4OAuth2Downloader/Wabbajack.Downloaders.IPS4OAuth2Downloader.csproj
#	Wabbajack.Downloaders.Interfaces/Wabbajack.Downloaders.Interfaces.csproj
#	Wabbajack.Downloaders.Manual/Wabbajack.Downloaders.Manual.csproj
#	Wabbajack.Downloaders.MediaFire/Wabbajack.Downloaders.MediaFire.csproj
#	Wabbajack.Downloaders.Mega/Wabbajack.Downloaders.Mega.csproj
#	Wabbajack.Downloaders.ModDB/Wabbajack.Downloaders.ModDB.csproj
#	Wabbajack.Downloaders.Nexus/Wabbajack.Downloaders.Nexus.csproj
#	Wabbajack.Downloaders.VerificationCache/Wabbajack.Downloaders.VerificationCache.csproj
#	Wabbajack.Downloaders.WabbajackCDN/Wabbajack.Downloaders.WabbajackCDN.csproj
#	Wabbajack.FileExtractor.Test/Wabbajack.FileExtractor.Test.csproj
#	Wabbajack.FileExtractor/Wabbajack.FileExtractor.csproj
#	Wabbajack.Hashing.PHash.Test/Wabbajack.Hashing.PHash.Test.csproj
#	Wabbajack.Hashing.PHash/Wabbajack.Hashing.PHash.csproj
#	Wabbajack.Hashing.xxHash64.Benchmark/Wabbajack.Hashing.xxHash64.Benchmark.csproj
#	Wabbajack.Hashing.xxHash64.Test/Wabbajack.Hashing.xxHash64.Test.csproj
#	Wabbajack.Hashing.xxHash64/Wabbajack.Hashing.xxHash64.csproj
#	Wabbajack.IO.Async/Wabbajack.IO.Async.csproj
#	Wabbajack.Installer.Test/Wabbajack.Installer.Test.csproj
#	Wabbajack.Installer/Wabbajack.Installer.csproj
#	Wabbajack.Launcher/Wabbajack.Launcher.csproj
#	Wabbajack.Networking.BethesdaNet/Wabbajack.Networking.BethesdaNet.csproj
#	Wabbajack.Networking.Discord/Wabbajack.Networking.Discord.csproj
#	Wabbajack.Networking.GitHub/Wabbajack.Networking.GitHub.csproj
#	Wabbajack.Networking.Http.Interfaces/Wabbajack.Networking.Http.Interfaces.csproj
#	Wabbajack.Networking.Http.Test/Wabbajack.Networking.Http.Test.csproj
#	Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj
#	Wabbajack.Networking.NexusApi.Test/Wabbajack.Networking.NexusApi.Test.csproj
#	Wabbajack.Networking.NexusApi/Wabbajack.Networking.NexusApi.csproj
#	Wabbajack.Networking.Steam.Test/Wabbajack.Networking.Steam.Test.csproj
#	Wabbajack.Networking.Steam/Wabbajack.Networking.Steam.csproj
#	Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj
#	Wabbajack.Paths.IO.Test/Wabbajack.Paths.IO.Test.csproj
#	Wabbajack.Paths.IO/Wabbajack.Paths.IO.csproj
#	Wabbajack.Paths.Test/Wabbajack.Paths.Test.csproj
#	Wabbajack.Paths/Wabbajack.Paths.csproj
#	Wabbajack.RateLimiter.Test/Wabbajack.RateLimiter.Test.csproj
#	Wabbajack.RateLimiter/Wabbajack.RateLimiter.csproj
#	Wabbajack.Server.Lib/Wabbajack.Server.Lib.csproj
#	Wabbajack.Server/Wabbajack.Server.csproj
#	Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
#	Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj
#	Wabbajack.VFS.Interfaces/Wabbajack.VFS.Interfaces.csproj
#	Wabbajack.VFS.Test/Wabbajack.VFS.Test.csproj
#	Wabbajack.VFS/Wabbajack.VFS.csproj
This commit is contained in:
EzioTheDeadPoet 2023-10-21 18:07:10 +02:00
commit ecd5b4a2c9
77 changed files with 535 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/tests.yaml vendored
View File

@ -32,6 +32,12 @@ jobs:
if: runner.os != 'Windows'
run: chmod -R +x Wabbajack.FileExtractor/Extractors
- name: Setup .NET Core SDK 8.0.x
uses: actions/setup-dotnet@v1
with:
dotnet-version: '8.0.x'
include-prerelease: true
- name: Setup .NET Core SDK 7.0.x
uses: actions/setup-dotnet@v1
with:

2
CHANGELOG.md
View File

@ -1,6 +1,6 @@
### Changelog
#### Version - 3.3.0.0 - TBA
#### Version - 3.3.0.0 - 10/13/2023
* Fixed some UI issues arising from 3.2.0.0 changes - more informative error text, wiki link button
* Added optional JSON flag for `DisplayVersionOnlyInInstallerView` to enable the installer image to only show version number.
* Fixed manual downloader downloading in the OS's "Downloads" folder

4
Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj
View File

@ -2,10 +2,10 @@
<PropertyGroup>
<OutputType>WinExe</OutputType>
<TargetFramework>net7.0-windows</TargetFramework>
<TargetFramework>net8.0-windows</TargetFramework>
<UseWPF>true</UseWPF>
<Platforms>x64</Platforms>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>
<Version>$(VERSION)</Version>
<AssemblyVersion>$(VERSION)</AssemblyVersion>
<FileVersion>$(VERSION)</FileVersion>

2
Wabbajack.CLI.Builder/Wabbajack.CLI.Builder.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.CLI/Wabbajack.CLI.csproj
View File

@ -13,7 +13,7 @@
<NoWarn>CS8600</NoWarn>
<NoWarn>CS8601</NoWarn>
<NoWarn>CS8618</NoWarn>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
</PropertyGroup>
<ItemGroup>

34
Wabbajack.Common/IEnumerableExtensions.cs
View File

@ -58,7 +58,14 @@ public static class IEnumerableExtensions
return data;
}
public static IEnumerable<IEnumerable<T>> Partition<T>(this IEnumerable<T> coll, int size)
/// <summary>
/// Splits the collection into `size` parts
/// </summary>
/// <param name="coll"></param>
/// <param name="count"></param>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public static IEnumerable<IEnumerable<T>> Partition<T>(this IEnumerable<T> coll, int count)
{
var asList = coll.ToList();
@ -70,7 +77,30 @@ public static class IEnumerableExtensions
}
}
return Enumerable.Range(0, size).Select(offset => SkipEnumerable(asList, offset, size));
return Enumerable.Range(0, count).Select(offset => SkipEnumerable(asList, offset, count));
}
/// <summary>
/// Split the collection into `size` parts
/// </summary>
/// <param name="coll"></param>
/// <param name="size"></param>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public static IEnumerable<IEnumerable<T>> Batch<T>(this IEnumerable<T> coll, int size)
{
List<T> current = new();
foreach (var itm in coll)
{
current.Add(itm);
if (current.Count == size)
{
yield return current;
current = new List<T>();
}
}
if (current.Count > 0)
yield return current;
}

2
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>
<Version>$(VERSION)</Version>

2
Wabbajack.Compiler.Test/Wabbajack.Compiler.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>

2
Wabbajack.Compiler/Wabbajack.Compiler.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

2
Wabbajack.Compression.BSA.Test/Wabbajack.Compression.BSA.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>

2
Wabbajack.Compression.BSA/Wabbajack.Compression.BSA.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

2
Wabbajack.Compression.Zip.Test/Wabbajack.Compression.Zip.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>

2
Wabbajack.Compression.Zip/Wabbajack.Compression.Zip.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.Configuration/Wabbajack.Configuration.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.DTOs.ConverterGenerators/Wabbajack.DTOs.ConverterGenerators.csproj
View File

@ -2,7 +2,7 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>
<Version>$(VERSION)</Version>
<NoWarn>CS8600</NoWarn>

2
Wabbajack.DTOs.Test/Wabbajack.DTOs.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>

2
Wabbajack.DTOs/Wabbajack.DTOs.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

2
Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.Downloaders.Dispatcher.Test/Wabbajack.Downloaders.Dispatcher.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>

2
Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

2
Wabbajack.Downloaders.GameFile/Wabbajack.Downloaders.GameFile.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.Downloaders.GoogleDrive/Wabbajack.Downloaders.GoogleDrive.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

2
Wabbajack.Downloaders.Http/Wabbajack.Downloaders.Http.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

2
Wabbajack.Downloaders.IPS4OAuth2Downloader/Wabbajack.Downloaders.IPS4OAuth2Downloader.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.Downloaders.Interfaces/Wabbajack.Downloaders.Interfaces.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>
<Version>$(VERSION)</Version>

2
Wabbajack.Downloaders.Manual/Wabbajack.Downloaders.Manual.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.Downloaders.MediaFire/Wabbajack.Downloaders.MediaFire.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.Downloaders.Mega/Wabbajack.Downloaders.Mega.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.Downloaders.ModDB/Wabbajack.Downloaders.ModDB.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.Downloaders.Nexus/Wabbajack.Downloaders.Nexus.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>
<Version>$(VERSION)</Version>

2
Wabbajack.Downloaders.VerificationCache/Wabbajack.Downloaders.VerificationCache.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.Downloaders.WabbajackCDN/Wabbajack.Downloaders.WabbajackCDN.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

5
Wabbajack.Downloaders.WabbajackCDN/WabbajackCDNDownloader.cs
View File

@ -80,7 +80,9 @@ public class WabbajackCDNDownloader : ADownloader<WabbajackCDN>, IUrlDownloader,
var definition = (await GetDefinition(state, token))!;
await using var fs = destination.Open(FileMode.Create, FileAccess.Write, FileShare.None);
await definition.Parts.PMapAll(async part =>
await definition.Parts.PMapAll<PartDefinition, (MemoryStream, PartDefinition)>(async part =>
{
return await CircuitBreaker.WithAutoRetryAllAsync<(MemoryStream, PartDefinition)>(_logger, async () =>
{
using var partJob = await _limiter.Begin(
$"Downloading {definition.MungedName} ({part.Index}/{definition.Size})",
@ -107,6 +109,7 @@ public class WabbajackCDNDownloader : ADownloader<WabbajackCDN>, IUrlDownloader,
}
return (ms, part);
});
}).Do(async rec =>

2
Wabbajack.FileExtractor.Test/Wabbajack.FileExtractor.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>

2
Wabbajack.FileExtractor/Wabbajack.FileExtractor.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

2
Wabbajack.Hashing.PHash.Test/Wabbajack.Hashing.PHash.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>

2
Wabbajack.Hashing.PHash/Wabbajack.Hashing.PHash.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

2
Wabbajack.Hashing.xxHash64.Benchmark/Wabbajack.Hashing.xxHash64.Benchmark.csproj
View File

@ -2,7 +2,7 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<RootNamespace>Wabbajac.Hash.xxHash64.Benchmark</RootNamespace>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>

2
Wabbajack.Hashing.xxHash64.Test/Wabbajack.Hashing.xxHash64.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>

2
Wabbajack.Hashing.xxHash64/Wabbajack.Hashing.xxHash64.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

2
Wabbajack.IO.Async/Wabbajack.IO.Async.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.Installer.Test/Wabbajack.Installer.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>

2
Wabbajack.Installer/Wabbajack.Installer.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

2
Wabbajack.Launcher/Wabbajack.Launcher.csproj
View File

@ -16,7 +16,7 @@
</ItemGroup>
<PropertyGroup>
<ApplicationIcon>Resources\Icons\wabbajack.ico</ApplicationIcon>
<TargetFramework>net7.0-windows</TargetFramework>
<TargetFramework>net8.0-windows</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Avalonia.Themes.Fluent" Version="11.0.0-preview4" />

2
Wabbajack.Networking.BethesdaNet/Wabbajack.Networking.BethesdaNet.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.Networking.Discord/Wabbajack.Networking.Discord.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>
<Version>$(VERSION)</Version>

2
Wabbajack.Networking.GitHub/Wabbajack.Networking.GitHub.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>
<Version>$(VERSION)</Version>

2
Wabbajack.Networking.Http.Interfaces/Wabbajack.Networking.Http.Interfaces.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

2
Wabbajack.Networking.Http.Test/Wabbajack.Networking.Http.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>

2
Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>
<Version>$(VERSION)</Version>

2
Wabbajack.Networking.NexusApi.Test/Wabbajack.Networking.NexusApi.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>

2
Wabbajack.Networking.NexusApi/Wabbajack.Networking.NexusApi.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>
<Version>$(VERSION)</Version>

2
Wabbajack.Networking.Steam.Test/Wabbajack.Networking.Steam.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>

2
Wabbajack.Networking.Steam/Wabbajack.Networking.Steam.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

2
Wabbajack.Paths.IO.Test/Wabbajack.Paths.IO.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>

2
Wabbajack.Paths.IO/Wabbajack.Paths.IO.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>
<Version>$(VERSION)</Version>

2
Wabbajack.Paths.Test/Wabbajack.Paths.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>

2
Wabbajack.Paths/Wabbajack.Paths.csproj
View File

@ -1,6 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>
<Version>$(VERSION)</Version>

2
Wabbajack.RateLimiter.Test/Wabbajack.RateLimiter.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>

2
Wabbajack.RateLimiter/Wabbajack.RateLimiter.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.Server.Lib/Wabbajack.Server.Lib.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>

19
Wabbajack.Server/AppSettings.cs
View File

@ -1,4 +1,5 @@
using Microsoft.Extensions.Configuration;
using Amazon.S3;
using Microsoft.Extensions.Configuration;
using Wabbajack.Paths;
namespace Wabbajack.BuildServer;
@ -26,8 +27,6 @@ public class AppSettings
public string DiscordKey { get; set; }
public string AuthoredFilesFolder { get; set; }
public string PatchesFilesFolder { get; set; }
public string MirrorFilesFolder { get; set; }
public string NexusCacheFolder { get; set; }
@ -37,6 +36,20 @@ public class AppSettings
public CouchDBSetting CesiDB { get; set; }
public CouchDBSetting MetricsDB { get; set; }
public S3Settings S3 { get; set; }
}
public class S3Settings
{
public string AccessKey { get; set; }
public string SecretKey { get; set; }
public string ServiceUrl { get; set; }
public string AuthoredFilesBucket { get; set; }
public string ProxyFilesBucket { get; set; }
public string AuthoredFilesBucketCache { get; set; }
}
public class CouchDBSetting

2
Wabbajack.Server/Controllers/AuthorControls.cs
View File

@ -147,7 +147,7 @@ public class AuthorControls : ControllerBase
public async Task<IActionResult> HomePage()
{
var user = User.FindFirstValue(ClaimTypes.Name);
var files = (await _authorFiles.AllAuthoredFiles())
var files = _authorFiles.AllDefinitions
.Where(af => af.Definition.Author == user)
.Select(af => new
{

20
Wabbajack.Server/Controllers/AuthoredFiles.cs
View File

@ -68,8 +68,7 @@ public class AuthoredFiles : ControllerBase
$"Hashes don't match for index {index}. Sizes ({ms.Length} vs {part.Size}). Hashes ({hash} vs {part.Hash}");
ms.Position = 0;
await using var partStream = await _authoredFiles.CreatePart(definition.MungedName, (int)index);
await ms.CopyToAsync(partStream, token);
await _authoredFiles.WritePart(definition.MungedName, (int) index, ms);
return Ok(part.Hash.ToBase64());
}
@ -123,7 +122,7 @@ public class AuthoredFiles : ControllerBase
public async Task<IActionResult> DeleteUpload(string serverAssignedUniqueId)
{
var user = User.FindFirstValue(ClaimTypes.Name);
var definition = (await _authoredFiles.AllAuthoredFiles())
var definition = _authoredFiles.AllDefinitions
.First(f => f.Definition.ServerAssignedUniqueId == serverAssignedUniqueId)
.Definition;
if (definition.Author != user)
@ -145,12 +144,12 @@ public class AuthoredFiles : ControllerBase
[Route("")]
public async Task<ContentResult> UploadedFilesGet()
{
var files = await _authoredFiles.AllAuthoredFiles();
var files = _authoredFiles.AllDefinitions
.ToArray();
var response = _authoredFilesTemplate(new
{
Files = files.OrderByDescending(f => f.Updated).ToArray(),
TotalSpace = _authoredFiles.TotalSpace.Bytes().Humanize("#.##"),
FreeSpace = _authoredFiles.FreeSpace.Bytes().Humanize("#.##")
UsedSpace = _authoredFiles.UsedSpace.Bytes().Humanize("#.##"),
});
return new ContentResult
{
@ -172,10 +171,13 @@ public class AuthoredFiles : ControllerBase
Response.Headers.ContentType = new StringValues("application/octet-stream");
Response.Headers.ContentLength = definition.Size;
Response.Headers.ETag = definition.MungedName + "_direct";
foreach (var part in definition.Parts)
foreach (var part in definition.Parts.OrderBy(p => p.Index))
{
await using var partStream = await _authoredFiles.StreamForPart(mungedName, (int)part.Index);
await partStream.CopyToAsync(Response.Body);
await _authoredFiles.StreamForPart(mungedName, (int)part.Index, async stream =>
{
await stream.CopyToAsync(Response.Body);
});
}
}
}

164
Wabbajack.Server/Controllers/Proxy.cs
View File

@ -1,4 +1,7 @@
using System.Text;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
using FluentFTP.Helpers;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
@ -10,7 +13,9 @@ using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
using Wabbajack.VFS;
namespace Wabbajack.Server.Controllers;
@ -24,46 +29,44 @@ public class Proxy : ControllerBase
private readonly TemporaryFileManager _tempFileManager;
private readonly AppSettings _appSettings;
private readonly FileHashCache _hashCache;
private readonly IAmazonS3 _s3;
private readonly string _bucket;
public Proxy(ILogger<Proxy> logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager, FileHashCache hashCache, AppSettings appSettings)
private string _redirectUrl = "https://proxy.wabbajack.org/";
private readonly IResource<DownloadDispatcher> _resource;
public Proxy(ILogger<Proxy> logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager,
FileHashCache hashCache, AppSettings appSettings, IAmazonS3 s3, IResource<DownloadDispatcher> resource)
{
_logger = logger;
_dispatcher = dispatcher;
_tempFileManager = tempFileManager;
_appSettings = appSettings;
_hashCache = hashCache;
_s3 = s3;
_bucket = _appSettings.S3.ProxyFilesBucket;
_resource = resource;
}
[HttpHead]
public async Task<IActionResult> ProxyHead(CancellationToken token, [FromQuery] Uri uri, [FromQuery] string? name,
[FromQuery] string? hash)
{
var shouldMatch = hash != null ? Hash.FromHex(hash) : default;
_logger.LogInformation("Got proxy head request for {Uri}", uri);
var state = _dispatcher.Parse(uri);
var cacheName = (await Encoding.UTF8.GetBytes(uri.ToString()).Hash()).ToHex();
var cacheFile = _appSettings.ProxyPath.Combine(cacheName);
if (!cacheFile.FileExists())
return NotFound();
if (shouldMatch != default)
if (await _hashCache.FileHashCachedAsync(cacheFile, token) != shouldMatch)
return NotFound();
return Ok();
return new RedirectResult(_redirectUrl + cacheName);
}
[HttpGet]
public async Task<IActionResult> ProxyGet(CancellationToken token, [FromQuery] Uri uri, [FromQuery] string? name, [FromQuery] string? hash)
{
Hash hashResult = default;
var shouldMatch = hash != null ? Hash.FromHex(hash) : default;
_logger.LogInformation("Got proxy request for {Uri}", uri);
var state = _dispatcher.Parse(uri);
var cacheName = (await Encoding.UTF8.GetBytes(uri.ToString()).Hash()).ToHex();
var cacheFile = _appSettings.ProxyPath.Combine(cacheName);
var cacheFile = await GetCacheEntry(cacheName);
if (state == null)
{
@ -84,26 +87,27 @@ public class Proxy : ControllerBase
return BadRequest(new {Type = "Downloader is not IProxyable", Downloader = downloader.GetType().FullName});
}
if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(4))
if (cacheFile != null && (DateTime.UtcNow - cacheFile.LastModified) > TimeSpan.FromHours(4))
{
try
{
var verify = await _dispatcher.Verify(archive, token);
if (verify)
cacheFile.Touch();
await TouchCacheEntry(cacheName);
}
catch (Exception ex)
{
_logger.LogInformation(ex, "When trying to verify cached file ({Hash}) {Url}", cacheFile.FileName, uri);
cacheFile.Touch();
_logger.LogInformation(ex, "When trying to verify cached file ({Hash}) {Url}",
cacheFile.Hash, uri);
await TouchCacheEntry(cacheName);
}
}
if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(24))
if (cacheFile != null && (DateTime.Now - cacheFile.LastModified) > TimeSpan.FromHours(24))
{
try
{
cacheFile.Delete();
await DeleteCacheEntry(cacheName);
}
catch (Exception ex)
{
@ -112,18 +116,15 @@ public class Proxy : ControllerBase
}
if (cacheFile.FileExists())
var redirectUrl = _redirectUrl + cacheName + "?response-content-disposition=attachment;filename=" + (name ?? "unknown");
if (cacheFile != null)
{
if (hash != default)
{
var hashResult = await _hashCache.FileHashCachedAsync(cacheFile, token);
if (hashResult != shouldMatch)
if (cacheFile.Hash != shouldMatch)
return BadRequest(new {Type = "Unmatching Hashes", Expected = shouldMatch.ToHex(), Found = hashResult.ToHex()});
}
var ret = new PhysicalFileResult(cacheFile.ToString(), "application/octet-stream");
if (name != null)
ret.FileDownloadName = name;
return ret;
return new RedirectResult(redirectUrl);
}
_logger.LogInformation("Downloading proxy request for {Uri}", uri);
@ -131,38 +132,97 @@ public class Proxy : ControllerBase
var tempFile = _tempFileManager.CreateFile(deleteOnDispose:false);
var proxyDownloader = _dispatcher.Downloader(archive) as IProxyable;
await using (var of = tempFile.Path.Open(FileMode.Create, FileAccess.Write, FileShare.None))
{
Response.StatusCode = 200;
if (name != null)
{
Response.Headers.Add(HeaderNames.ContentDisposition, $"attachment; filename=\"{name}\"");
}
Response.Headers.Add( HeaderNames.ContentType, "application/octet-stream" );
var result = await proxyDownloader!.DownloadStream(archive, async s => {
return await s.HashingCopy(async m =>
{
var strmA = of.WriteAsync(m, token);
await Response.Body.WriteAsync(m, token);
await Response.Body.FlushAsync(token);
await strmA;
}, token); },
token);
using var job = await _resource.Begin("Downloading file", 0, token);
hashResult = await proxyDownloader!.Download(archive, tempFile.Path, job, token);
if (hash != default && result != shouldMatch)
if (hash != default && hashResult != shouldMatch)
{
if (tempFile.Path.FileExists())
tempFile.Path.Delete();
}
return NotFound();
}
await PutCacheEntry(tempFile.Path, cacheName, hashResult);
await tempFile.Path.MoveToAsync(cacheFile, true, token);
_logger.LogInformation("Returning proxy request for {Uri}", uri);
return new RedirectResult(redirectUrl);
}
_logger.LogInformation("Returning proxy request for {Uri} {Size}", uri, cacheFile.Size().FileSizeToString());
return new EmptyResult();
private async Task<CacheStatus?> GetCacheEntry(string name)
{
GetObjectMetadataResponse info;
try
{
info = await _s3.GetObjectMetadataAsync(new GetObjectMetadataRequest()
{
BucketName = _bucket,
Key = name,
});
}
catch (Exception _)
{
return null;
}
if (info.HttpStatusCode == System.Net.HttpStatusCode.NotFound)
return null;
if (info.Metadata["WJ-Hash"] == null)
return null;
if (!Hash.TryGetFromHex(info.Metadata["WJ-Hash"], out var hash))
return null;
return new CacheStatus
{
LastModified = info.LastModified,
Size = info.ContentLength,
Hash = hash
};
}
private async Task TouchCacheEntry(string name)
{
await _s3.CopyObjectAsync(new CopyObjectRequest()
{
SourceBucket = _bucket,
DestinationBucket = _bucket,
SourceKey = name,
DestinationKey = name,
MetadataDirective = S3MetadataDirective.REPLACE,
});
}
private async Task PutCacheEntry(AbsolutePath path, string name, Hash hash)
{
var obj = new PutObjectRequest
{
BucketName = _bucket,
Key = name,
FilePath = path.ToString(),
ContentType = "application/octet-stream",
DisablePayloadSigning = true
};
obj.Metadata.Add("WJ-Hash", hash.ToHex());
await _s3.PutObjectAsync(obj);
}
private async Task DeleteCacheEntry(string name)
{
await _s3.DeleteObjectAsync(new DeleteObjectRequest
{
BucketName = _bucket,
Key = name
});
}
record CacheStatus
{
public DateTime LastModified { get; init; }
public long Size { get; init; }
public Hash Hash { get; init; }
}
}

242
Wabbajack.Server/DataModels/AuthorFiles.cs
View File

@ -1,6 +1,11 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using System.IO.Compression;
using System.Web;
using Amazon.S3;
using Amazon.S3.Model;
using Microsoft.Extensions.Logging;
using Microsoft.IO;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.DTOs.CDN;
@ -16,88 +21,213 @@ public class AuthorFiles
private readonly ILogger<AuthorFiles> _logger;
private readonly AppSettings _settings;
private readonly DTOSerializer _dtos;
private Dictionary<string, FileDefinition> _byServerId = new();
private ConcurrentDictionary<string, FileDefinition> _byServerId = new();
private readonly IAmazonS3 _s3;
private readonly ConcurrentDictionary<string,FileDefinitionMetadata> _fileCache;
private readonly string _bucketName;
private ConcurrentDictionary<RelativePath, long> _allObjects = new();
private HashSet<RelativePath> _mangledNames;
private readonly RecyclableMemoryStreamManager _streamPool;
private readonly HttpClient _httpClient;
private readonly AbsolutePath _cacheFile;
public AbsolutePath AuthorFilesLocation => _settings.AuthoredFilesFolder.ToAbsolutePath();
private Uri _baseUri => new($"https://authored-files.wabbajack.org/");
public AuthorFiles(ILogger<AuthorFiles> logger, AppSettings settings, DTOSerializer dtos)
public AuthorFiles(ILogger<AuthorFiles> logger, AppSettings settings, DTOSerializer dtos, IAmazonS3 s3, HttpClient client)
{
_httpClient = client;
_s3 = s3;
_logger = logger;
_settings = settings;
_dtos = dtos;
_fileCache = new ConcurrentDictionary<string, FileDefinitionMetadata>();
_bucketName = settings.S3.AuthoredFilesBucket;
_ = PrimeCache();
_streamPool = new RecyclableMemoryStreamManager();
_cacheFile = _settings.S3.AuthoredFilesBucketCache.ToAbsolutePath();
}
public IEnumerable<AbsolutePath> AllDefinitions => AuthorFilesLocation.EnumerateFiles("definition.json.gz");
/// <summary>
/// Total unused space available for authored files
/// </summary>
public long FreeSpace => new DriveInfo(AuthorFilesLocation.ToString()).AvailableFreeSpace;
/// <summary>
/// Total space available for authored files
/// </summary>
public long TotalSpace => new DriveInfo(AuthorFilesLocation.ToString()).TotalSize;
/// <summary>
///
/// </summary>
/// <returns></returns>
public async Task<FileDefinitionMetadata[]> AllAuthoredFiles()
private async Task PrimeCache()
{
var defs = new List<FileDefinitionMetadata>();
foreach (var file in AllDefinitions)
try
{
defs.Add(new FileDefinitionMetadata
if (!_cacheFile.FileExists())
{
Definition = await ReadDefinition(file),
Updated = file.LastModifiedUtc()
var allObjects = await AllObjects().ToArrayAsync();
foreach (var obje in allObjects)
{
_allObjects.TryAdd(obje.Key.ToRelativePath(), obje.LastModified.ToFileTimeUtc());
}
SaveBucketCacheFile(_cacheFile);
}
else
{
LoadBucketCacheFile(_cacheFile);
}
_mangledNames = _allObjects
.Where(f => f.Key.EndsWith("definition.json.gz"))
.Select(f => f.Key.Parent)
.ToHashSet();
await Parallel.ForEachAsync(_mangledNames, async (name, _) =>
{
if (!_allObjects.TryGetValue(name.Combine("definition.json.gz"), out var value))
return;
_logger.LogInformation("Priming {Name}", name);
var definition = await PrimeDefinition(name);
var metadata = new FileDefinitionMetadata()
{
Definition = definition,
Updated = DateTime.FromFileTimeUtc(value)
};
_fileCache.TryAdd(definition.MungedName, metadata);
_byServerId.TryAdd(definition.ServerAssignedUniqueId!, definition);
});
_logger.LogInformation("Finished priming cache, {Count} files {Size} GB cached", _fileCache.Count,
_fileCache.Sum(s => s.Value.Definition.Size) / (1024 * 1024 * 1024));
}
catch (Exception ex)
{
_logger.LogCritical(ex, "Failed to prime cache");
}
}
private void SaveBucketCacheFile(AbsolutePath cacheFile)
{
using var file = cacheFile.Open(FileMode.Create, FileAccess.Write);
using var sw = new StreamWriter(file);
foreach(var entry in _allObjects)
{
sw.WriteLine($"{entry.Key}||{entry.Value}");
}
}
private void LoadBucketCacheFile(AbsolutePath cacheFile)
{
using var file = cacheFile.Open(FileMode.Open, FileAccess.Read);
using var sr = new StreamReader(file);
while (!sr.EndOfStream)
{
var line = sr.ReadLine();
var parts = line!.Split("||");
_allObjects.TryAdd(parts[0].ToRelativePath(), long.Parse(parts[1]));
}
}
private async Task<FileDefinition> PrimeDefinition(RelativePath name)
{
return await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () =>
{
var uri = _baseUri + $"{name}/definition.json.gz";
using var response = await _httpClient.GetAsync(uri);
return await ReadDefinition(await response.Content.ReadAsStreamAsync());
});
}
_byServerId = defs.ToDictionary(f => f.Definition.ServerAssignedUniqueId!, f => f.Definition);
return defs.ToArray();
private async IAsyncEnumerable<S3Object> AllObjects()
{
var sw = Stopwatch.StartNew();
var total = 0;
_logger.Log(LogLevel.Information, "Listing all objects in S3");
var results = await _s3.ListObjectsV2Async(new ListObjectsV2Request()
{
BucketName = _bucketName,
});
TOP:
total += results.S3Objects.Count;
_logger.Log(LogLevel.Information, "Got {S3ObjectsCount} objects, {Total} total", results.S3Objects.Count, total);
foreach (var result in results.S3Objects)
{
yield return result;
}
public async Task<Stream> StreamForPart(string mungedName, int part)
if (results.IsTruncated)
{
return AuthorFilesLocation.Combine(mungedName, "parts", part.ToString()).Open(FileMode.Open);
results = await _s3.ListObjectsV2Async(new ListObjectsV2Request
{
ContinuationToken = results.NextContinuationToken,
BucketName = _bucketName,
});
goto TOP;
}
_logger.LogInformation("Finished listing all objects in S3 in {Elapsed}", sw.Elapsed);
}
public async Task<Stream> CreatePart(string mungedName, int part)
public IEnumerable<FileDefinitionMetadata> AllDefinitions => _fileCache.Values;
/// <summary>
/// Used space in bytes
/// </summary>
public long UsedSpace => _fileCache.Sum(s => s.Value.Definition.Size);
public async Task StreamForPart(string mungedName, int part, Func<Stream, Task> func)
{
return AuthorFilesLocation.Combine(mungedName, "parts", part.ToString()).Open(FileMode.Create, FileAccess.Write, FileShare.None);
var definition = _fileCache[mungedName].Definition;
if (part >= definition.Parts.Length)
throw new ArgumentOutOfRangeException(nameof(part));
var uri = _baseUri + $"{mungedName}/parts/{part}";
using var response = await _httpClient.GetAsync(uri);
await func(await response.Content.ReadAsStreamAsync());
}
public async Task WritePart(string mungedName, int part, Stream ms)
{
await _s3.PutObjectAsync(new PutObjectRequest
{
BucketName = _bucketName,
Key = mungedName.ToRelativePath().Combine("parts", part.ToString()).ToString().Replace("\\", "/"),
InputStream = ms,
DisablePayloadSigning = true,
ContentType = "application/octet-stream"
});
}
public async Task WriteDefinition(FileDefinition definition)
{
var path = AuthorFilesLocation.Combine(definition.MungedName, "definition.json.gz");
path.Parent.CreateDirectory();
path.Parent.Combine("parts").CreateDirectory();
await using var ms = new MemoryStream();
await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
{
await _dtos.Serialize(definition, gz);
}
ms.Position = 0;
await path.WriteAllBytesAsync(ms.ToArray());
await _s3.PutObjectAsync(new PutObjectRequest
{
BucketName = _bucketName,
Key = definition.MungedName.ToRelativePath().Combine("definition.json.gz").ToString().Replace("\\", "/"),
InputStream = ms,
DisablePayloadSigning = true,
ContentType = "application/octet-stream"
});
_fileCache.TryAdd(definition.MungedName, new FileDefinitionMetadata
{
Definition = definition,
Updated = DateTime.UtcNow
});
_byServerId.TryAdd(definition.ServerAssignedUniqueId!, definition);
}
public async Task<FileDefinition> ReadDefinition(string mungedName)
{
return await ReadDefinition(AuthorFilesLocation.Combine(mungedName, "definition.json.gz"));
return _fileCache[mungedName].Definition;
}
public bool IsDefinition(string mungedName)
{
return AuthorFilesLocation.Combine(mungedName, "definition.json.gz").FileExists();
return _fileCache.ContainsKey(mungedName);
}
private async Task<FileDefinition> ReadDefinition(AbsolutePath file)
private async Task<FileDefinition> ReadDefinition(Stream stream)
{
var gz = new GZipStream(new MemoryStream(await file.ReadAllBytesAsync()), CompressionMode.Decompress);
var gz = new GZipStream(stream, CompressionMode.Decompress);
var definition = (await _dtos.DeserializeAsync<FileDefinition>(gz))!;
return definition;
}
@ -111,15 +241,33 @@ public class AuthorFiles
public async Task DeleteFile(FileDefinition definition)
{
var folder = AuthorFilesLocation.Combine(definition.MungedName);
folder.DeleteDirectory();
var allFiles = _allObjects.Where(f => f.Key.TopParent.ToString() == definition.MungedName)
.Select(f => f.Key).ToList();
foreach (var batch in allFiles.Batch(512))
{
var batchedArray = batch.ToHashSet();
_logger.LogInformation("Deleting {Count} files for prefix {Prefix}", batchedArray.Count, definition.MungedName);
await _s3.DeleteObjectsAsync(new DeleteObjectsRequest
{
BucketName = _bucketName,
Objects = batchedArray.Select(f => new KeyVersion
{
Key = f.ToString().Replace("\\", "/")
}).ToList()
});
foreach (var key in batchedArray)
{
_allObjects.TryRemove(key, out _);
}
}
public async Task<FileDefinition> ReadDefinitionForServerId(string serverAssignedUniqueId)
_byServerId.TryRemove(definition.ServerAssignedUniqueId!, out _);
_fileCache.TryRemove(definition.MungedName, out _);
}
public async ValueTask<FileDefinition> ReadDefinitionForServerId(string serverAssignedUniqueId)
{
if (_byServerId.TryGetValue(serverAssignedUniqueId, out var found))
return found;
await AllAuthoredFiles();
return _byServerId[serverAssignedUniqueId];
}

4
Wabbajack.Server/Resources/Reports/AuthoredFiles.html
View File

@ -11,7 +11,7 @@
</head>
<body>
<H1>Authored Files:</H1>
<H3>{{$.FreeSpace}} remaining of {{$.TotalSpace}} </H3>
<H3>{{$.UsedSpace}}</H3>
<table id="inlined-data" class="table table-striped table-bordered" style="width:100%" >
<thead>
<tr>
@ -28,7 +28,7 @@
<td>{{$.HumanSize}}</td>
<td>{{$.Definition.Author}}</td>
<td>{{$.Updated}}</td>
<td><a href='/authored_files/direct_link/{{$.Definition.MungedName}}'>(Slow) HTTP Direct Link</a></td>
<td><a href='https://workers.wabbajack.workers.dev/authored_files/stream/{{$.Definition.MungedName}}'>(Slow) HTTP Direct Link</a></td>
</tr>
{{/each}}
</table>

33
Wabbajack.Server/Startup.cs
View File

@ -5,6 +5,9 @@ using System.Runtime.InteropServices;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.Util.Internal;
using cesi.DTOs;
using CouchDB.Driver;
using CouchDB.Driver.Options;
@ -22,6 +25,7 @@ using Nettle.Compiler;
using Newtonsoft.Json;
using Octokit;
using Wabbajack.BuildServer;
using Wabbajack.Configuration;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.VerificationCache;
using Wabbajack.DTOs;
@ -39,10 +43,11 @@ using Wabbajack.Server.Services;
using Wabbajack.Services.OSIntegrated.TokenProviders;
using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.Paths.IO;
using Wabbajack.Server.DTOs;
using Wabbajack.VFS;
using YamlDotNet.Serialization.NamingConventions;
using Client = Wabbajack.Networking.GitHub.Client;
using Metric = Wabbajack.Server.DTOs.Metric;
using SettingsManager = Wabbajack.Services.OSIntegrated.SettingsManager;
namespace Wabbajack.Server;
@ -93,6 +98,16 @@ public class Startup
services.AddSingleton<TarLog>();
services.AddAllSingleton<IHttpDownloader, SingleThreadedDownloader>();
services.AddDownloadDispatcher(useLoginDownloaders:false, useProxyCache:false);
services.AddSingleton<IAmazonS3>(s =>
{
var appSettings = s.GetRequiredService<AppSettings>();
var settings = new BasicAWSCredentials(appSettings.S3.AccessKey,
appSettings.S3.SecretKey);
return new AmazonS3Client(settings, new AmazonS3Config
{
ServiceURL = appSettings.S3.ServiceUrl,
});
});
services.AddTransient(s =>
{
var settings = s.GetRequiredService<AppSettings>();
@ -141,6 +156,20 @@ public class Startup
});
services.AddDTOSerializer();
services.AddDTOConverters();
services.AddSingleton(s => new Wabbajack.Services.OSIntegrated.Configuration
{
EncryptedDataLocation = KnownFolders.WabbajackAppLocal.Combine("encrypted"),
ModListsDownloadLocation = KnownFolders.EntryPoint.Combine("downloaded_mod_lists"),
SavedSettingsLocation = KnownFolders.WabbajackAppLocal.Combine("saved_settings"),
LogLocation = KnownFolders.LauncherAwarePath.Combine("logs"),
ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache")
});
services.AddSingleton<SettingsManager>();
services.AddSingleton<MainSettings>(s => Wabbajack.Services.OSIntegrated.ServiceExtensions.GetAppSettings(s, MainSettings.SettingsFileName));
services.AddResponseCompression(options =>
{
options.Providers.Add<BrotliCompressionProvider>();
@ -243,5 +272,7 @@ public class Startup
// Trigger the internal update code
app.ApplicationServices.GetRequiredService<NexusCacheManager>();
app.ApplicationServices.GetRequiredService<DiscordBackend>();
app.ApplicationServices.GetRequiredService<AuthorFiles>();
}
}

4
Wabbajack.Server/Wabbajack.Server.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<OutputType>Exe</OutputType>
@ -12,6 +12,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.7.205.9" />
<PackageReference Include="cesi.DTOs" Version="1.0.0" />
<PackageReference Include="Chronic.Core" Version="0.4.0" />
<PackageReference Include="Dapper" Version="2.0.123" />
@ -22,6 +23,7 @@
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.11" />
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.3" />
<PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="2.3.2" />
<PackageReference Include="Nettle" Version="1.3.0" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.5" />
</ItemGroup>

8
Wabbajack.Server/appsettings.json
View File

@ -28,6 +28,14 @@
"Database": "metrics",
"Username": "wabbajack",
"Password": "password"
},
"S3": {
"AccessKey": "<>",
"SecretKey": "<>",
"ServiceUrl": "<>",
"ProxyFilesBucket": "proxy-files",
"AuthoredFilesBucket": "authored-files",
"AuthoredFilesBucketCache": "c:\\tmp\\bucket-cache.txt"
}
},
"AllowedHosts": "*"

22
Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
View File

@ -98,17 +98,7 @@ public static class ServiceExtensions
service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount});
MainSettings GetAppSettings(IServiceProvider provider, string name)
{
var settingsManager = provider.GetRequiredService<SettingsManager>();
var settings = Task.Run(() => settingsManager.Load<MainSettings>(name)).Result;
if (settings.Upgrade())
{
settingsManager.Save(MainSettings.SettingsFileName, settings).FireAndForget();
}
return settings;
}
Func<Task<(int MaxTasks, long MaxThroughput)>> GetResourceSettings(IServiceProvider provider, string name)
{
@ -235,6 +225,18 @@ public static class ServiceExtensions
return service;
}
public static MainSettings GetAppSettings(IServiceProvider provider, string name)
{
var settingsManager = provider.GetRequiredService<SettingsManager>();
var settings = Task.Run(() => settingsManager.Load<MainSettings>(name)).Result;
if (settings.Upgrade())
{
settingsManager.Save(MainSettings.SettingsFileName, settings).FireAndForget();
}
return settings;
}
private static void CleanAllTempData(AbsolutePath path)
{
// Get directories first and cache them, this freezes the directories were looking at

2
Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

2
Wabbajack.VFS.Interfaces/Wabbajack.VFS.Interfaces.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Wabbajack.VFS.Test/Wabbajack.VFS.Test.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>

2
Wabbajack.VFS/Wabbajack.VFS.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<Version>$(VERSION)</Version>
<PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>

12
buildall.bat
View File

@ -7,12 +7,12 @@ mkdir c:\tmp\publish-wj
dotnet clean
dotnet restore
dotnet publish Wabbajack.App.Wpf\Wabbajack.App.Wpf.csproj --runtime win10-x64 --configuration Release /p:Platform=x64 -o c:\tmp\publish-wj\app /p:PublishReadyToRun=true /p:PublishSingleFile=true /p:IncludeNativeLibrariesForSelfExtract=true --self-contained /p:DebugType=embedded
dotnet publish Wabbajack.Launcher\Wabbajack.Launcher.csproj --runtime win10-x64 --configuration Release /p:Platform=x64 -o c:\tmp\publish-wj\launcher /p:PublishReadyToRun=true /p:PublishSingleFile=true /p:IncludeNativeLibrariesForSelfExtract=true --self-contained /p:DebugType=embedded
dotnet publish c:\oss\Wabbajack\Wabbajack.CLI\Wabbajack.CLI.csproj --runtime win10-x64 --configuration Release /p:Platform=x64 -o c:\tmp\publish-wj\app --self-contained /p:DebugType=embedded
"C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64\signtool.exe" sign /t http://timestamp.sectigo.com c:\tmp\publish-wj\app\Wabbajack.exe
"C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64\signtool.exe" sign /t http://timestamp.sectigo.com c:\tmp\publish-wj\launcher\Wabbajack.exe
"C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64\signtool.exe" sign /t http://timestamp.sectigo.com c:\tmp\publish-wj\app\wabbajack-cli.exe
dotnet publish Wabbajack.App.Wpf\Wabbajack.App.Wpf.csproj --runtime win-x64 --configuration Release /p:Platform=x64 -o c:\tmp\publish-wj\app /p:PublishReadyToRun=true /p:PublishSingleFile=true /p:IncludeNativeLibrariesForSelfExtract=true --self-contained /p:DebugType=embedded
dotnet publish Wabbajack.Launcher\Wabbajack.Launcher.csproj --runtime win-x64 --configuration Release /p:Platform=x64 -o c:\tmp\publish-wj\launcher /p:PublishReadyToRun=true /p:PublishSingleFile=true /p:IncludeNativeLibrariesForSelfExtract=true --self-contained /p:DebugType=embedded
dotnet publish c:\oss\Wabbajack\Wabbajack.CLI\Wabbajack.CLI.csproj --runtime win-x64 --configuration Release /p:Platform=x64 -o c:\tmp\publish-wj\app --self-contained /p:DebugType=embedded
"C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64\signtool.exe" sign /fd sha256 /tr http://ts.ssl.com /td sha256 /sha1 8c26a8e0bf3e70eb89721cc4d86a87137153ccba c:\tmp\publish-wj\app\Wabbajack.exe
"C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64\signtool.exe" sign /fd sha256 /tr http://ts.ssl.com /td sha256 /sha1 8c26a8e0bf3e70eb89721cc4d86a87137153ccba c:\tmp\publish-wj\launcher\Wabbajack.exe
"C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64\signtool.exe" sign /fd sha256 /tr http://ts.ssl.com /td sha256 /sha1 8c26a8e0bf3e70eb89721cc4d86a87137153ccba c:\tmp\publish-wj\app\wabbajack-cli.exe
"c:\Program Files\7-Zip\7z.exe" a c:\tmp\publish-wj\%VERSION%.zip c:\tmp\publish-wj\app\*
copy c:\tmp\publish-wj\launcher\Wabbajack.exe c:\tmp\publish-wj\Wabbajack.exe