From 88cdebccda0cdb3d072f74c9f15851792b828100 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 12 Oct 2021 21:59:54 -0600 Subject: [PATCH] WIP fixed game downloader --- Wabbajack.App/Controls/BrowseItemViewModel.cs | 1 + Wabbajack.App/Screens/BrowseViewModel.cs | 1 + .../Screens/CompilerConfigurationViewModel.cs | 3 +- .../Screens/StandardInstallationViewModel.cs | 1 + Wabbajack.Compiler/ACompiler.cs | 33 +++++---- Wabbajack.Compiler/MO2Compiler.cs | 1 + .../ServiceExtensions.cs | 1 + .../Wabbajack.Downloaders.Dispatcher.csproj | 1 + .../GameFileDownloader.cs | 68 +++++++++++++++++++ .../GameLocator.cs | 2 +- .../IGameLocator.cs | 2 +- .../ServiceExtensions.cs | 21 ++++++ .../Wabbajack.Downloaders.GameFile.csproj | 27 ++++++++ Wabbajack.Downloaders.Mega/MegaDownloader.cs | 24 ++++--- Wabbajack.Installer/AInstaller.cs | 1 + Wabbajack.Installer/StandardInstaller.cs | 1 + .../Wabbajack.Installer.csproj | 1 + .../Client.cs | 2 +- .../ServiceExtensions.cs | 1 + .../StubbedGameLocator.cs | 1 + Wabbajack.sln | 7 ++ 21 files changed, 176 insertions(+), 24 deletions(-) create mode 100644 Wabbajack.Downloaders.GameFile/GameFileDownloader.cs rename {Wabbajack.Installer => Wabbajack.Downloaders.GameFile}/GameLocator.cs (98%) rename {Wabbajack.Installer => Wabbajack.Downloaders.GameFile}/IGameLocator.cs (86%) create mode 100644 Wabbajack.Downloaders.GameFile/ServiceExtensions.cs create mode 100644 Wabbajack.Downloaders.GameFile/Wabbajack.Downloaders.GameFile.csproj diff --git a/Wabbajack.App/Controls/BrowseItemViewModel.cs b/Wabbajack.App/Controls/BrowseItemViewModel.cs index beb0bad1..774bdda1 100644 --- a/Wabbajack.App/Controls/BrowseItemViewModel.cs +++ b/Wabbajack.App/Controls/BrowseItemViewModel.cs @@ -17,6 +17,7 @@ using Wabbajack.App.Messages; using Wabbajack.App.ViewModels; using Wabbajack.Common; using Wabbajack.Downloaders; +using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; using Wabbajack.DTOs.JsonConverters; using Wabbajack.Installer; diff --git a/Wabbajack.App/Screens/BrowseViewModel.cs b/Wabbajack.App/Screens/BrowseViewModel.cs index 04041a0a..749d68c4 100644 --- a/Wabbajack.App/Screens/BrowseViewModel.cs +++ b/Wabbajack.App/Screens/BrowseViewModel.cs @@ -24,6 +24,7 @@ using Wabbajack.Networking.WabbajackClientApi; using DynamicData.Binding; using Microsoft.Extensions.DependencyInjection; using Wabbajack.Downloaders; +using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs.JsonConverters; using Wabbajack.Installer; using Wabbajack.Paths; diff --git a/Wabbajack.App/Screens/CompilerConfigurationViewModel.cs b/Wabbajack.App/Screens/CompilerConfigurationViewModel.cs index 4dc316e9..cc5b0655 100644 --- a/Wabbajack.App/Screens/CompilerConfigurationViewModel.cs +++ b/Wabbajack.App/Screens/CompilerConfigurationViewModel.cs @@ -87,7 +87,8 @@ public class CompilerConfigurationViewModel : ViewModelBase, IReceiverMarker Downloads = Downloads, Source = BasePath, Game = BaseGame.Game, - Profile = SelectedProfile + Profile = SelectedProfile, + UseGamePaths = true }; MessageBus.Instance.Send(new StartCompilation(settings)); diff --git a/Wabbajack.App/Screens/StandardInstallationViewModel.cs b/Wabbajack.App/Screens/StandardInstallationViewModel.cs index ead47d73..9ac63a44 100644 --- a/Wabbajack.App/Screens/StandardInstallationViewModel.cs +++ b/Wabbajack.App/Screens/StandardInstallationViewModel.cs @@ -19,6 +19,7 @@ using Wabbajack.App.Screens; using Wabbajack.App.Utilities; using Wabbajack.App.ViewModels.SubViewModels; using Wabbajack.Common; +using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.JsonConverters; diff --git a/Wabbajack.Compiler/ACompiler.cs b/Wabbajack.Compiler/ACompiler.cs index 73568fec..b35973ac 100644 --- a/Wabbajack.Compiler/ACompiler.cs +++ b/Wabbajack.Compiler/ACompiler.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Compiler.CompilationSteps; using Wabbajack.Downloaders; +using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.DownloadStates; @@ -195,7 +196,8 @@ namespace Wabbajack.Compiler return new IndexedArchive( _vfs.Index.ByRootPath[path.Combine(state.GameFile)]) { - Name = state.GameFile.ToString().Replace("/", "_").Replace("\\", "_") + Name = state.GameFile.ToString().Replace("/", "_").Replace("\\", "_"), + State = state }; })); } @@ -515,7 +517,7 @@ namespace Wabbajack.Compiler public async Task ResolveArchive(IndexedArchive archive) { - if (archive.IniData == null) + if (archive.State == null && archive.IniData == null) { _logger.LogWarning( "No download metadata found for {archive}, please use MO2 to query info or add a .meta file and try again.", @@ -523,31 +525,38 @@ namespace Wabbajack.Compiler return null; } - var state = await _dispatcher.ResolveArchive(archive.IniData!["General"].ToDictionary(d => d.KeyName, d => d.Value)); - - if (state == null) + IDownloadState? state; + if (archive.State == null) { - _logger.LogWarning("{archive} could not be handled by any of the downloaders", archive.Name); - return null; + state = await _dispatcher.ResolveArchive(archive.IniData!["General"] + .ToDictionary(d => d.KeyName, d => d.Value)); + + if (state == null) + { + _logger.LogWarning("{archive} could not be handled by any of the downloaders", archive.Name); + return null; + } + + } + else + { + state = archive.State; } var result = new Archive { - State = state, + State = state!, Name = archive.Name ?? "", Hash = archive.File.Hash, Size = archive.File.Size }; - var downloader = _dispatcher.Downloader(result); - await downloader.Prepare(); - var token = new CancellationTokenSource(); token.CancelAfter(_settings.MaxVerificationTime); if (!await _dispatcher.Verify(result, token.Token)) { _logger.LogWarning( - "Unable to resolve link for {Archive}. If this is hosted on the Nexus the file may have been removed.", archive.State!.PrimaryKeyString); + "Unable to resolve link for {Archive}. If this is hosted on the Nexus the file may have been removed.", result.State!.PrimaryKeyString); } result.Meta = "[General]\n" + string.Join("\n", _dispatcher.MetaIni(result)); diff --git a/Wabbajack.Compiler/MO2Compiler.cs b/Wabbajack.Compiler/MO2Compiler.cs index a86be9d1..a3f533f1 100644 --- a/Wabbajack.Compiler/MO2Compiler.cs +++ b/Wabbajack.Compiler/MO2Compiler.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Compiler.CompilationSteps; using Wabbajack.Downloaders; +using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.JsonConverters; diff --git a/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs b/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs index 678fece3..7445c372 100644 --- a/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs +++ b/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs @@ -22,6 +22,7 @@ namespace Wabbajack.Downloaders .AddNexusDownloader() .AddIPS4OAuth2Downloaders() .AddWabbajackCDNDownloader() + .AddGameFileDownloader() .AddSingleton(); } } diff --git a/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj b/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj index ff0417e7..168f0d6d 100644 --- a/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj +++ b/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj @@ -8,6 +8,7 @@ + diff --git a/Wabbajack.Downloaders.GameFile/GameFileDownloader.cs b/Wabbajack.Downloaders.GameFile/GameFileDownloader.cs new file mode 100644 index 00000000..6cb6ff63 --- /dev/null +++ b/Wabbajack.Downloaders.GameFile/GameFileDownloader.cs @@ -0,0 +1,68 @@ +using Wabbajack.Downloaders.Interfaces; +using Wabbajack.DTOs; +using Wabbajack.DTOs.DownloadStates; +using Wabbajack.DTOs.Validation; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; +using Wabbajack.RateLimiter; +using Wabbajack.VFS; + +namespace Wabbajack.Downloaders.GameFile; + +public class GameFileDownloader : ADownloader +{ + private readonly IGameLocator _locator; + private readonly FileHashCache _hashCache; + + public GameFileDownloader(IGameLocator locator, FileHashCache hashCache) + { + _locator = locator; + _hashCache = hashCache; + + } + + public override Task Download(Archive archive, GameFileSource state, AbsolutePath destination, IJob job, CancellationToken token) + { + throw new NotImplementedException(); + } + + public override Task Prepare() + { + return Task.FromResult(true); + } + + public override bool IsAllowed(ServerAllowList allowList, IDownloadState state) + { + return true; + } + + public override IDownloadState? Resolve(IReadOnlyDictionary iniData) + { + if (!iniData.TryGetValue("gameName", out var gameName) || !iniData.TryGetValue("gameFile", out var gameFile) || + !GameRegistry.TryGetByFuzzyName(gameName, out var game)) return null; + + return new GameFileSource + { + Game = game.Game, + GameFile = gameFile.ToRelativePath() + }; + + } + + public override Priority Priority => Priority.Normal; + public override async Task Verify(Archive archive, GameFileSource archiveState, IJob job, CancellationToken token) + { + var fp = archiveState.GameFile.RelativeTo(_locator.GameLocation(archiveState.Game)); + if (!fp.FileExists()) return false; + return await _hashCache.FileHashCachedAsync(fp, token) == archive.Hash; + } + public override IEnumerable MetaIni(Archive a, GameFileSource state) + { + return new[] + { + $"gameName={state.Game}", + $"gameFile={state.GameFile}" + }; + } +} \ No newline at end of file diff --git a/Wabbajack.Installer/GameLocator.cs b/Wabbajack.Downloaders.GameFile/GameLocator.cs similarity index 98% rename from Wabbajack.Installer/GameLocator.cs rename to Wabbajack.Downloaders.GameFile/GameLocator.cs index 0ce4b042..7f74ba62 100644 --- a/Wabbajack.Installer/GameLocator.cs +++ b/Wabbajack.Downloaders.GameFile/GameLocator.cs @@ -9,7 +9,7 @@ using Microsoft.Extensions.Logging; using Wabbajack.DTOs; using Wabbajack.Paths; -namespace Wabbajack.Installer +namespace Wabbajack.Downloaders.GameFile { public class GameLocator : IGameLocator { diff --git a/Wabbajack.Installer/IGameLocator.cs b/Wabbajack.Downloaders.GameFile/IGameLocator.cs similarity index 86% rename from Wabbajack.Installer/IGameLocator.cs rename to Wabbajack.Downloaders.GameFile/IGameLocator.cs index 4d2b5a85..63c3e00a 100644 --- a/Wabbajack.Installer/IGameLocator.cs +++ b/Wabbajack.Downloaders.GameFile/IGameLocator.cs @@ -1,7 +1,7 @@ using Wabbajack.DTOs; using Wabbajack.Paths; -namespace Wabbajack.Installer +namespace Wabbajack.Downloaders.GameFile { public interface IGameLocator { diff --git a/Wabbajack.Downloaders.GameFile/ServiceExtensions.cs b/Wabbajack.Downloaders.GameFile/ServiceExtensions.cs new file mode 100644 index 00000000..65cc8e03 --- /dev/null +++ b/Wabbajack.Downloaders.GameFile/ServiceExtensions.cs @@ -0,0 +1,21 @@ +using Microsoft.Extensions.DependencyInjection; +using Wabbajack.Downloaders.GameFile; +using Wabbajack.Downloaders.Interfaces; +using Wabbajack.DTOs; + +namespace Wabbajack.Downloaders +{ + public static class ServiceExtensions + { + public static IServiceCollection AddGameFileDownloader(this IServiceCollection services) + { + return services.AddAllSingleton, + GameFileDownloader>(); + } + + public static IServiceCollection AddStandardGameLocator(this IServiceCollection services) + { + return services.AddAllSingleton(); + } + } +} \ No newline at end of file diff --git a/Wabbajack.Downloaders.GameFile/Wabbajack.Downloaders.GameFile.csproj b/Wabbajack.Downloaders.GameFile/Wabbajack.Downloaders.GameFile.csproj new file mode 100644 index 00000000..5368195e --- /dev/null +++ b/Wabbajack.Downloaders.GameFile/Wabbajack.Downloaders.GameFile.csproj @@ -0,0 +1,27 @@ + + + + net6.0 + enable + enable + + + + + + + + + + ..\..\..\Program Files\dotnet\packs\Microsoft.AspNetCore.App.Ref\6.0.0-preview.6.21355.2\ref\net6.0\Microsoft.Extensions.Logging.Abstractions.dll + + + + + + + + + + + diff --git a/Wabbajack.Downloaders.Mega/MegaDownloader.cs b/Wabbajack.Downloaders.Mega/MegaDownloader.cs index 70af10c4..ca629a0f 100644 --- a/Wabbajack.Downloaders.Mega/MegaDownloader.cs +++ b/Wabbajack.Downloaders.Mega/MegaDownloader.cs @@ -71,16 +71,24 @@ namespace Wabbajack.Downloaders.ModDB { if (!_apiClient.IsLoggedIn) await _apiClient.LoginAsync(); - - try + + for (var times = 0; times < 5; times ++) { - var node = await _apiClient.GetNodeFromLinkAsync(archiveState.Url); - return node != null; - } - catch (Exception) - { - return false; + try + { + var node = await _apiClient.GetNodeFromLinkAsync(archiveState.Url); + if (node != null) + return true; + } + catch (Exception) + { + return false; + } + + await Task.Delay(TimeSpan.FromMilliseconds(500), token); } + + return false; } public override IEnumerable MetaIni(Archive a, Mega state) diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index eda6e3d4..764d55e8 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Downloaders; +using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.DownloadStates; diff --git a/Wabbajack.Installer/StandardInstaller.cs b/Wabbajack.Installer/StandardInstaller.cs index 50b001eb..c91f928e 100644 --- a/Wabbajack.Installer/StandardInstaller.cs +++ b/Wabbajack.Installer/StandardInstaller.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Compression.BSA; using Wabbajack.Downloaders; +using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.DownloadStates; diff --git a/Wabbajack.Installer/Wabbajack.Installer.csproj b/Wabbajack.Installer/Wabbajack.Installer.csproj index 326380e0..313b91ee 100644 --- a/Wabbajack.Installer/Wabbajack.Installer.csproj +++ b/Wabbajack.Installer/Wabbajack.Installer.csproj @@ -9,6 +9,7 @@ + diff --git a/Wabbajack.Networking.WabbajackClientApi/Client.cs b/Wabbajack.Networking.WabbajackClientApi/Client.cs index a2e64a41..0c1f95ac 100644 --- a/Wabbajack.Networking.WabbajackClientApi/Client.cs +++ b/Wabbajack.Networking.WabbajackClientApi/Client.cs @@ -89,7 +89,7 @@ namespace Wabbajack.Networking.WabbajackClientApi public async Task GetGameArchives(Game game, string version) { var url = $"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}.json"; - return await _client.GetFromJsonAsync(url) ?? Array.Empty(); + return await _client.GetFromJsonAsync(url, _dtos.Options) ?? Array.Empty(); } public async Task GetArchivesForHash(Hash hash) diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index f12a87da..c20b57d3 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Compiler; using Wabbajack.Downloaders; +using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; using Wabbajack.DTOs.Logins; using Wabbajack.Installer; diff --git a/Wabbajack.Services.OSIntegrated/StubbedGameLocator.cs b/Wabbajack.Services.OSIntegrated/StubbedGameLocator.cs index 717d188c..15912cdb 100644 --- a/Wabbajack.Services.OSIntegrated/StubbedGameLocator.cs +++ b/Wabbajack.Services.OSIntegrated/StubbedGameLocator.cs @@ -1,3 +1,4 @@ +using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; using Wabbajack.Installer; using Wabbajack.Paths; diff --git a/Wabbajack.sln b/Wabbajack.sln index 29f94b2b..feb693da 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -112,6 +112,8 @@ ProjectSection(SolutionItems) = preProject nuget.config = nuget.config EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Downloaders.GameFile", "Wabbajack.Downloaders.GameFile\Wabbajack.Downloaders.GameFile.csproj", "{4F252332-CA77-41DE-95A8-9DF38A81D675}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -310,6 +312,10 @@ Global {DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A}.Debug|Any CPU.Build.0 = Debug|Any CPU {DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A}.Release|Any CPU.ActiveCfg = Release|Any CPU {DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A}.Release|Any CPU.Build.0 = Release|Any CPU + {4F252332-CA77-41DE-95A8-9DF38A81D675}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4F252332-CA77-41DE-95A8-9DF38A81D675}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4F252332-CA77-41DE-95A8-9DF38A81D675}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4F252332-CA77-41DE-95A8-9DF38A81D675}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {4057B668-8595-44FE-9805-007B284A838F} = {98B731EE-4FC0-4482-A069-BCBA25497871} @@ -345,5 +351,6 @@ Global {5D792FA8-8120-4A5C-A969-2258D3D05F1A} = {F677890D-5109-43BC-97C7-C4CD47C8EE0C} {29AC8A68-D5EC-43F5-B2CC-72A75545E418} = {98B731EE-4FC0-4482-A069-BCBA25497871} {DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A} = {F01F8595-5FD7-4506-8469-F4A5522DACC1} + {4F252332-CA77-41DE-95A8-9DF38A81D675} = {98B731EE-4FC0-4482-A069-BCBA25497871} EndGlobalSection EndGlobal