WIP fixed game downloader

Wabbajack.App/Controls/BrowseItemViewModel.cs

@@ -17,6 +17,7 @@ using Wabbajack.App.Messages;
 using Wabbajack.App.ViewModels;
 using Wabbajack.Common;
 using Wabbajack.Downloaders;
+using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.JsonConverters;
 using Wabbajack.Installer;

Wabbajack.App/Screens/BrowseViewModel.cs

@@ -24,6 +24,7 @@ using Wabbajack.Networking.WabbajackClientApi;
 using DynamicData.Binding;
 using Microsoft.Extensions.DependencyInjection;
 using Wabbajack.Downloaders;
+using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs.JsonConverters;
 using Wabbajack.Installer;
 using Wabbajack.Paths;

Wabbajack.App/Screens/CompilerConfigurationViewModel.cs

@@ -87,7 +87,8 @@ public class CompilerConfigurationViewModel : ViewModelBase, IReceiverMarker
             Downloads = Downloads,
             Source = BasePath,
             Game = BaseGame.Game,
-            Profile = SelectedProfile
+            Profile = SelectedProfile,
+            UseGamePaths = true
         };
         
         MessageBus.Instance.Send(new StartCompilation(settings));

Wabbajack.App/Screens/StandardInstallationViewModel.cs

@@ -19,6 +19,7 @@ using Wabbajack.App.Screens;
 using Wabbajack.App.Utilities;
 using Wabbajack.App.ViewModels.SubViewModels;
 using Wabbajack.Common;
+using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.DownloadStates;
 using Wabbajack.DTOs.JsonConverters;

Wabbajack.Compiler/ACompiler.cs

@@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging;
 using Wabbajack.Common;
 using Wabbajack.Compiler.CompilationSteps;
 using Wabbajack.Downloaders;
+using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.Directives;
 using Wabbajack.DTOs.DownloadStates;
@@ -195,7 +196,8 @@ namespace Wabbajack.Compiler
                             return new IndexedArchive(
                                 _vfs.Index.ByRootPath[path.Combine(state.GameFile)])
                             {
-                                Name = state.GameFile.ToString().Replace("/", "_").Replace("\\", "_")
+                                Name = state.GameFile.ToString().Replace("/", "_").Replace("\\", "_"),
+                                State = state
                             };
                         }));
                     }
@@ -515,7 +517,7 @@ namespace Wabbajack.Compiler
 
         public async Task<Archive?> ResolveArchive(IndexedArchive archive)
         {
-            if (archive.IniData == null)
+            if (archive.State == null && archive.IniData == null)
             {
                 _logger.LogWarning(
                     "No download metadata found for {archive}, please use MO2 to query info or add a .meta file and try again.",
@@ -523,31 +525,38 @@ namespace Wabbajack.Compiler
                 return null;
             }
 
-            var state = await _dispatcher.ResolveArchive(archive.IniData!["General"].ToDictionary(d => d.KeyName, d => d.Value));
+            IDownloadState? state;
-
+            if (archive.State == null)
-            if (state == null)
             {
-                _logger.LogWarning("{archive} could not be handled by any of the downloaders", archive.Name);
+                state = await _dispatcher.ResolveArchive(archive.IniData!["General"]
-                return null;
+                    .ToDictionary(d => d.KeyName, d => d.Value));
+
+                if (state == null)
+                {
+                    _logger.LogWarning("{archive} could not be handled by any of the downloaders", archive.Name);
+                    return null;
+                }
+
+            }
+            else
+            {
+                state = archive.State;
             }
 
             var result = new Archive
             {
-                State = state,
+                State = state!,
                 Name = archive.Name ?? "", 
                 Hash = archive.File.Hash, 
                 Size = archive.File.Size
             };
 
-            var downloader = _dispatcher.Downloader(result);
-            await downloader.Prepare();
-
             var token = new CancellationTokenSource();
             token.CancelAfter(_settings.MaxVerificationTime);
             if (!await _dispatcher.Verify(result, token.Token))
             {
                 _logger.LogWarning(
-                    "Unable to resolve link for {Archive}. If this is hosted on the Nexus the file may have been removed.", archive.State!.PrimaryKeyString);
+                    "Unable to resolve link for {Archive}. If this is hosted on the Nexus the file may have been removed.", result.State!.PrimaryKeyString);
             }
 
             result.Meta = "[General]\n" + string.Join("\n", _dispatcher.MetaIni(result));

Wabbajack.Compiler/MO2Compiler.cs

@@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging;
 using Wabbajack.Common;
 using Wabbajack.Compiler.CompilationSteps;
 using Wabbajack.Downloaders;
+using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.Directives;
 using Wabbajack.DTOs.JsonConverters;

Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs

@@ -22,6 +22,7 @@ namespace Wabbajack.Downloaders
                 .AddNexusDownloader()
                 .AddIPS4OAuth2Downloaders()
                 .AddWabbajackCDNDownloader()
+                .AddGameFileDownloader()
                 .AddSingleton<DownloadDispatcher>();
         }
     }

Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj

@@ -8,6 +8,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <ProjectReference Include="..\Wabbajack.Downloaders.GameFile\Wabbajack.Downloaders.GameFile.csproj" />
         <ProjectReference Include="..\Wabbajack.Downloaders.GoogleDrive\Wabbajack.Downloaders.GoogleDrive.csproj" />
         <ProjectReference Include="..\Wabbajack.Downloaders.Http\Wabbajack.Downloaders.Http.csproj" />
         <ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />

Wabbajack.Downloaders.GameFile/GameFileDownloader.cs

@@ -0,0 +1,68 @@
+using Wabbajack.Downloaders.Interfaces;
+using Wabbajack.DTOs;
+using Wabbajack.DTOs.DownloadStates;
+using Wabbajack.DTOs.Validation;
+using Wabbajack.Hashing.xxHash64;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+using Wabbajack.RateLimiter;
+using Wabbajack.VFS;
+
+namespace Wabbajack.Downloaders.GameFile;
+
+public class GameFileDownloader : ADownloader<GameFileSource>
+{
+    private readonly IGameLocator _locator;
+    private readonly FileHashCache _hashCache;
+
+    public GameFileDownloader(IGameLocator locator, FileHashCache hashCache)
+    {
+        _locator = locator;
+        _hashCache = hashCache;
+
+    }
+    
+    public override Task<Hash> Download(Archive archive, GameFileSource state, AbsolutePath destination, IJob job, CancellationToken token)
+    {
+        throw new NotImplementedException();
+    }
+
+    public override Task<bool> Prepare()
+    {
+        return Task.FromResult(true);
+    }
+
+    public override bool IsAllowed(ServerAllowList allowList, IDownloadState state)
+    {
+        return true;
+    }
+
+    public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
+    {
+        if (!iniData.TryGetValue("gameName", out var gameName) || !iniData.TryGetValue("gameFile", out var gameFile) ||
+            !GameRegistry.TryGetByFuzzyName(gameName, out var game)) return null;
+
+        return new GameFileSource
+        {
+            Game = game.Game,
+            GameFile = gameFile.ToRelativePath()
+        };
+
+    }
+
+    public override Priority Priority => Priority.Normal;
+    public override async Task<bool> Verify(Archive archive, GameFileSource archiveState, IJob job, CancellationToken token)
+    {
+        var fp = archiveState.GameFile.RelativeTo(_locator.GameLocation(archiveState.Game));
+        if (!fp.FileExists()) return false;
+        return await _hashCache.FileHashCachedAsync(fp, token) == archive.Hash;
+    }
+    public override IEnumerable<string> MetaIni(Archive a, GameFileSource state)
+    {
+        return new[]
+        {
+            $"gameName={state.Game}",
+            $"gameFile={state.GameFile}"
+        };
+    }
+}

Wabbajack.Downloaders.GameFile/GameLocator.cs

@@ -9,7 +9,7 @@ using Microsoft.Extensions.Logging;
 using Wabbajack.DTOs;
 using Wabbajack.Paths;
 
-namespace Wabbajack.Installer
+namespace Wabbajack.Downloaders.GameFile
 {
     public class GameLocator : IGameLocator
     {

Wabbajack.Downloaders.GameFile/IGameLocator.cs

@@ -1,7 +1,7 @@
 using Wabbajack.DTOs;
 using Wabbajack.Paths;
 
-namespace Wabbajack.Installer
+namespace Wabbajack.Downloaders.GameFile
 {
     public interface IGameLocator
     {

Wabbajack.Downloaders.GameFile/ServiceExtensions.cs

@@ -0,0 +1,21 @@
+using Microsoft.Extensions.DependencyInjection;
+using Wabbajack.Downloaders.GameFile;
+using Wabbajack.Downloaders.Interfaces;
+using Wabbajack.DTOs;
+
+namespace Wabbajack.Downloaders
+{
+    public static class ServiceExtensions
+    {
+        public static IServiceCollection AddGameFileDownloader(this IServiceCollection services)
+        {
+            return services.AddAllSingleton<IDownloader, IDownloader<DTOs.DownloadStates.GameFileSource>, 
+                GameFileDownloader>();
+        }
+
+        public static IServiceCollection AddStandardGameLocator(this IServiceCollection services)
+        {
+            return services.AddAllSingleton<IGameLocator, GameLocator>();
+        }
+    }
+}

Wabbajack.Downloaders.GameFile/Wabbajack.Downloaders.GameFile.csproj

@@ -0,0 +1,27 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net6.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
+      <ProjectReference Include="..\Wabbajack.VFS\Wabbajack.VFS.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <Reference Include="Microsoft.Extensions.Logging.Abstractions, Version=6.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60">
+        <HintPath>..\..\..\Program Files\dotnet\packs\Microsoft.AspNetCore.App.Ref\6.0.0-preview.6.21355.2\ref\net6.0\Microsoft.Extensions.Logging.Abstractions.dll</HintPath>
+      </Reference>
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Include="GameFinder.StoreHandlers.EGS" Version="1.7.1" />
+      <PackageReference Include="GameFinder.StoreHandlers.GOG" Version="1.7.1" />
+      <PackageReference Include="GameFinder.StoreHandlers.Origin" Version="1.7.1" />
+      <PackageReference Include="GameFinder.StoreHandlers.Steam" Version="1.7.1" />
+    </ItemGroup>
+
+</Project>

Wabbajack.Downloaders.Mega/MegaDownloader.cs

@@ -72,15 +72,23 @@ namespace Wabbajack.Downloaders.ModDB
             if (!_apiClient.IsLoggedIn)
                 await _apiClient.LoginAsync();
 
-            try
+            for (var times = 0; times < 5; times ++)
             {
-                var node = await _apiClient.GetNodeFromLinkAsync(archiveState.Url);
+                try
-                return node != null;
+                {
-            }
+                    var node = await _apiClient.GetNodeFromLinkAsync(archiveState.Url);
-            catch (Exception)
+                    if (node != null) 
-            {
+                        return true;
-                return false;
+                }
+                catch (Exception)
+                {
+                    return false;
+                }
+
+                await Task.Delay(TimeSpan.FromMilliseconds(500), token);
             }
+
+            return false;
         }
 
         public override IEnumerable<string> MetaIni(Archive a, Mega state)

Wabbajack.Installer/AInstaller.cs

@@ -10,6 +10,7 @@ using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Wabbajack.Common;
 using Wabbajack.Downloaders;
+using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.Directives;
 using Wabbajack.DTOs.DownloadStates;

Wabbajack.Installer/StandardInstaller.cs

@@ -13,6 +13,7 @@ using Microsoft.Extensions.Logging;
 using Wabbajack.Common;
 using Wabbajack.Compression.BSA;
 using Wabbajack.Downloaders;
+using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.Directives;
 using Wabbajack.DTOs.DownloadStates;

Wabbajack.Installer/Wabbajack.Installer.csproj

@@ -9,6 +9,7 @@
 
     <ItemGroup>
         <ProjectReference Include="..\Wabbajack.Downloaders.Dispatcher\Wabbajack.Downloaders.Dispatcher.csproj" />
+        <ProjectReference Include="..\Wabbajack.Downloaders.GameFile\Wabbajack.Downloaders.GameFile.csproj" />
         <ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
         <ProjectReference Include="..\Wabbajack.FileExtractor\Wabbajack.FileExtractor.csproj" />
         <ProjectReference Include="..\Wabbajack.Networking.WabbajackClientApi\Wabbajack.Networking.WabbajackClientApi.csproj" />

Wabbajack.Networking.WabbajackClientApi/Client.cs

@@ -89,7 +89,7 @@ namespace Wabbajack.Networking.WabbajackClientApi
         public async Task<Archive[]> GetGameArchives(Game game, string version)
         {
             var url = $"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}.json";
-            return await _client.GetFromJsonAsync<Archive[]>(url) ?? Array.Empty<Archive>();
+            return await _client.GetFromJsonAsync<Archive[]>(url, _dtos.Options) ?? Array.Empty<Archive>();
         }
         
         public async Task<Archive[]> GetArchivesForHash(Hash hash)

Wabbajack.Services.OSIntegrated/ServiceExtensions.cs

@@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging;
 using Wabbajack.Common;
 using Wabbajack.Compiler;
 using Wabbajack.Downloaders;
+using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.Logins;
 using Wabbajack.Installer;

Wabbajack.Services.OSIntegrated/StubbedGameLocator.cs

@@ -1,3 +1,4 @@
+using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
 using Wabbajack.Installer;
 using Wabbajack.Paths;

Wabbajack.sln

@@ -112,6 +112,8 @@ ProjectSection(SolutionItems) = preProject
 	nuget.config = nuget.config
 EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Downloaders.GameFile", "Wabbajack.Downloaders.GameFile\Wabbajack.Downloaders.GameFile.csproj", "{4F252332-CA77-41DE-95A8-9DF38A81D675}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -310,6 +312,10 @@ Global
 		{DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4F252332-CA77-41DE-95A8-9DF38A81D675}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4F252332-CA77-41DE-95A8-9DF38A81D675}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4F252332-CA77-41DE-95A8-9DF38A81D675}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4F252332-CA77-41DE-95A8-9DF38A81D675}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{4057B668-8595-44FE-9805-007B284A838F} = {98B731EE-4FC0-4482-A069-BCBA25497871}
@@ -345,5 +351,6 @@ Global
 		{5D792FA8-8120-4A5C-A969-2258D3D05F1A} = {F677890D-5109-43BC-97C7-C4CD47C8EE0C}
 		{29AC8A68-D5EC-43F5-B2CC-72A75545E418} = {98B731EE-4FC0-4482-A069-BCBA25497871}
 		{DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A} = {F01F8595-5FD7-4506-8469-F4A5522DACC1}
+		{4F252332-CA77-41DE-95A8-9DF38A81D675} = {98B731EE-4FC0-4482-A069-BCBA25497871}
 	EndGlobalSection
 EndGlobal