Merge branch 'main' into dependabot/nuget/Octokit-4.0.1

.github/workflows/tests.yaml

@@ -39,12 +39,7 @@ jobs:
           include-prerelease: true
 
       - name: Test
-        run: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:EnableWindowsTargeting=true --filter "Category!=FlakeyNetwork"
-        
-      - uses: codecov/codecov-action@v3
-        with:
-          flags: unittests, ${{runner.os}}
-          verbose: true 
+        run: dotnet test /p:EnableWindowsTargeting=true --filter "Category!=FlakeyNetwork"
         
       #- name: Upload Test Folder on Failure
       #  if: ${{ failure() }}

CHANGELOG.md

@@ -1,5 +1,18 @@
 ### Changelog
 
+#### Version - 3.0.4.0 - TBD
+* upgrade GameFinder to 2.2.1
+
+#### Version - 3.0.3.1 - 10/30/2022
+* Fix file verification issues for CreatedBSAs
+* Fix files during verification where CreatedDate > LastModified
+
+#### Version - 3.0.3.0 - 10/26/2022
+* Verify hashes of all installed files
+* Verify contents of BSAs during installation
+* Provide a new CLI command for verifying a installed modlist
+* When downloading from one Nexus CDN server fails, WJ will now try alternate Nexus servers
+
 #### Version - 3.0.2.3 - 10/19/2022
 * HOTFIX: revert GameFinder library to 1.8 until it's a bit more forgiving of corrupt files
 

Wabbajack.App.Wpf/View Models/MainWindowVM.cs

@@ -134,7 +134,7 @@ namespace Wabbajack
                 var assembly = Assembly.GetExecutingAssembly();
                 var location = assembly.Location;
                 if (string.IsNullOrWhiteSpace(location))
-                    location = Process.GetCurrentProcess().MainModule?.FileName ?? "";
+                    location = Process.GetCurrentProcess().MainModule?.FileName ?? throw new Exception("Assembly location is unavailable!");
 
                 _logger.LogInformation("App Location: {Location}", assembly.Location);
                 var fvi = FileVersionInfo.GetVersionInfo(location);
@@ -149,10 +149,9 @@ namespace Wabbajack
                 // setup file association
                 try
                 {
-                    var applicationRegistrationService =
-                        _serviceProvider.GetRequiredService<IApplicationRegistrationService>();
+                    var applicationRegistrationService = _serviceProvider.GetRequiredService<IApplicationRegistrationService>();
 
-                    var applicationInfo = new ApplicationInfo(assembly);
+                    var applicationInfo = new ApplicationInfo("Wabbajack", "Wabbajack", "Wabbajack", location);
                     applicationInfo.SupportedExtensions.Add("wabbajack");
                     applicationRegistrationService.RegisterApplication(applicationInfo);
                 }

Wabbajack.CLI/Verbs/VerifyModlistInstall.cs

@@ -55,60 +55,57 @@ public class VerifyModlistInstall
 
         _logger.LogInformation("Scanning files");
         var errors = await list.Directives.PMapAllBatchedAsync(_limiter, async directive => 
-        {
-            if (directive is ArchiveMeta)
-                return null;
-
-            if (directive is RemappedInlineFile)
-                return null;
-
-            if (directive.To.InFolder(Consts.BSACreationDir))
-                return null;
-
-            var dest = directive.To.RelativeTo(installFolder);
-            if (!dest.FileExists())
             {
-                return new Result
+                if (!(directive is CreateBSA || directive.IsDeterministic))
+                    return null;
+                
+                if (directive.To.InFolder(Consts.BSACreationDir))
+                    return null;
+
+                var dest = directive.To.RelativeTo(installFolder);
+                if (!dest.FileExists())
                 {
-                    Path = directive.To,
-                    Message = $"File does not exist directive {directive.GetType()}"
-                };
-            }
+                    return new Result
+                    {
+                        Path = directive.To,
+                        Message = $"File does not exist directive {directive.GetType()}"
+                    };
+                }
+
+                if (Consts.KnownModifiedFiles.Contains(directive.To.FileName))
+                    return null;
+
+                if (directive is CreateBSA bsa)
+                {
+                    return await VerifyBSA(dest, bsa, byTo, token);
+                }
+
+                if (dest.Size() != directive.Size)
+                {
+                    return new Result
+                    {
+                        Path = directive.To,
+                        Message = $"Sizes do not match got {dest.Size()} expected {directive.Size}"
+                    };
+                }
+
+                if (directive.Size > (1024 * 1024 * 128))
+                {
+                    _logger.LogInformation("Hashing {Size} file at {Path}", directive.Size.ToFileSizeString(),
+                        directive.To);
+                }
+
+                var hash = await AbsolutePathExtensions.Hash(dest, token);
+                if (hash != directive.Hash)
+                {
+                    return new Result
+                    {
+                        Path = directive.To,
+                        Message = $"Hashes do not match, got {hash} expected {directive.Hash}"
+                    };
+                }
 
-            if (Consts.KnownModifiedFiles.Contains(directive.To.FileName))
                 return null;
-
-            if (directive is CreateBSA bsa)
-            {
-                return await VerifyBSA(dest, bsa, byTo, token);
-            }
-
-            if (dest.Size() != directive.Size)
-            {
-                return new Result
-                {
-                    Path = directive.To,
-                    Message = $"Sizes do not match got {dest.Size()} expected {directive.Size}"
-                };
-            }
-
-            if (directive.Size > (1024 * 1024 * 128))
-            {
-                _logger.LogInformation("Hashing {Size} file at {Path}", directive.Size.ToFileSizeString(),
-                    directive.To);
-            }
-
-            var hash = await AbsolutePathExtensions.Hash(dest, token);
-            if (hash != directive.Hash)
-            {
-                return new Result
-                {
-                    Path = directive.To,
-                    Message = $"Hashes do not match, got {hash} expected {directive.Hash}"
-                };
-            }
-
-            return null;
         }).Where(r => r != null)
             .ToList();
         
@@ -150,6 +147,9 @@ public class VerifyModlistInstall
             var astate = bsa.FileStates.First(f => f.Path == state.Path);
             var srcDirective = byTo[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)];
 
+            if (!srcDirective.IsDeterministic)
+                continue;
+
             if (srcDirective.Hash != hash)
             {
                 return new Result

Wabbajack.DTOs/ModList/Directive.cs

@@ -1,3 +1,4 @@
+using System.Text.Json.Serialization;
 using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Paths;
 
@@ -12,4 +13,6 @@ public abstract class Directive
     ///     location the file will be copied to, relative to the install path.
     /// </summary>
     public RelativePath To { get; set; }
+
+    [JsonIgnore] public virtual bool IsDeterministic => true;
 }

Wabbajack.DTOs/ModList/Directives/ArchiveMeta.cs

@@ -8,4 +8,6 @@ namespace Wabbajack.DTOs.Directives;
 public class ArchiveMeta : Directive
 {
     public RelativePath SourceDataID { get; set; }
+
+    public override bool IsDeterministic => false;
 }

Wabbajack.DTOs/ModList/Directives/CreateBSA.cs

@@ -13,4 +13,6 @@ public class CreateBSA : Directive
     public RelativePath TempID { get; set; }
     public IArchive State { get; set; }
     public AFile[] FileStates { get; set; } = Array.Empty<AFile>();
+
+    public override bool IsDeterministic => false;
 }

Wabbajack.DTOs/ModList/Directives/RemappedInlineFile.cs

@@ -9,4 +9,5 @@ namespace Wabbajack.DTOs.Directives;
 [JsonAlias("RemappedInlineFile, Wabbajack.Lib")]
 public class RemappedInlineFile : InlineFile
 {
+    public override bool IsDeterministic => false;
 }

Wabbajack.DTOs/ModList/Directives/TransformedTexture.cs

@@ -11,4 +11,5 @@ public class TransformedTexture : FromArchive
     ///     The file to apply to the source file to patch it
     /// </summary>
     public ImageState ImageState { get; set; } = new();
+    public override bool IsDeterministic => false;
 }

Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs

@@ -9,7 +9,6 @@ namespace Wabbajack.Downloaders.Dispatcher.Test;
 
 public class VerificationCacheTests
 {
-    private readonly TemporaryFileManager _temp;
     private readonly ILogger<VerificationCache.VerificationCache> _logger;
 
     public VerificationCacheTests(ILogger<VerificationCache.VerificationCache> logger)

Wabbajack.Downloaders.GameFile/GameLocator.cs

@@ -1,3 +1,6 @@
+using System.Runtime.InteropServices;
+using GameFinder.Common;
+using GameFinder.RegistryUtils;
 using GameFinder.StoreHandlers.EGS;
 using GameFinder.StoreHandlers.GOG;
 using GameFinder.StoreHandlers.Origin;
@@ -5,67 +8,120 @@ using GameFinder.StoreHandlers.Steam;
 using Microsoft.Extensions.Logging;
 using Wabbajack.DTOs;
 using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
 
 namespace Wabbajack.Downloaders.GameFile;
 
 public class GameLocator : IGameLocator
 {
-    private readonly EGSHandler? _egs;
+    private readonly SteamHandler _steam;
     private readonly GOGHandler? _gog;
+    private readonly EGSHandler? _egs;
+    private readonly OriginHandler? _origin;
+
+    private readonly Dictionary<int, AbsolutePath> _steamGames = new();
+    private readonly Dictionary<long, AbsolutePath> _gogGames = new();
+    private readonly Dictionary<string, AbsolutePath> _egsGames = new(StringComparer.OrdinalIgnoreCase);
+    private readonly Dictionary<string, AbsolutePath> _originGames = new(StringComparer.OrdinalIgnoreCase);
+
     private readonly Dictionary<Game, AbsolutePath> _locationCache;
     private readonly ILogger<GameLocator> _logger;
-    private readonly OriginHandler? _origin;
-    private readonly SteamHandler _steam;
 
     public GameLocator(ILogger<GameLocator> logger)
     {
         _logger = logger;
-        _steam = new SteamHandler(logger);
 
-        if (OperatingSystem.IsWindows())
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
         {
-            _origin = new OriginHandler(true, false, logger);
-            _gog = new GOGHandler(logger);
+            var windowsRegistry = new WindowsRegistry();
 
-            _egs = new EGSHandler(logger);
+            _steam = new SteamHandler(windowsRegistry);
+            _gog = new GOGHandler(windowsRegistry);
+            _egs = new EGSHandler(windowsRegistry);
+            _origin = new OriginHandler();
+        }
+        else
+        {
+            _steam = new SteamHandler(null);
         }
 
         _locationCache = new Dictionary<Game, AbsolutePath>();
 
+        FindAllGames();
+    }
+
+    private void FindAllGames()
+    {
         try
         {
-            _steam.FindAllGames();
+            FindStoreGames(_steam, _steamGames, game => game.Path);
         }
-        catch (Exception ex)
+        catch (Exception e)
         {
-            _logger.LogError(ex, "While finding Steam games");
+            _logger.LogError(e, "While finding games installed with Steam");
         }
 
         try
         {
-            _origin?.FindAllGames();
+            FindStoreGames(_gog, _gogGames, game => game.Path);
         }
-        catch (Exception ex)
+        catch (Exception e)
         {
-            _logger.LogInformation(ex, "While finding Origin games");
+            _logger.LogError(e, "While finding games installed with GOG Galaxy");
         }
 
         try
         {
-            _gog?.FindAllGames();
+            FindStoreGames(_egs, _egsGames, game => game.InstallLocation);
         }
-        catch (Exception ex)
+        catch (Exception e)
         {
-            _logger.LogInformation(ex, "While finding GoG games");
+            _logger.LogError(e, "While finding games installed with the Epic Games Store");
         }
 
         try
         {
-            _egs?.FindAllGames();
+            FindStoreGames(_origin, _originGames, game => game.InstallPath);
         }
-        catch (Exception ex)
+        catch (Exception e)
         {
-            _logger.LogInformation(ex, "While finding Epic Games");
+            _logger.LogError(e, "While finding games installed with Origin");
+        }
+    }
+
+    private void FindStoreGames<TGame, TId>(
+        AHandler<TGame, TId>? handler,
+        Dictionary<TId, AbsolutePath> paths,
+        Func<TGame, string> getPath)
+        where TGame : class
+    {
+        if (handler is null) return;
+
+        var games = handler.FindAllGamesById(out var errors);
+
+        foreach (var (id, game) in games)
+        {
+            try
+            {
+                var path = getPath(game).ToAbsolutePath();
+                if (!path.DirectoryExists())
+                {
+                    _logger.LogError("Game does not exist: {Game}", game);
+                    continue;
+                }
+
+                paths[id] = path;
+                _logger.LogDebug("Found {Game}", game);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "While locating {Game}", game);
+            }
+        }
+
+        foreach (var error in errors)
+        {
+            _logger.LogError("{Error}", error);
         }
     }
 
@@ -102,68 +158,32 @@ public class GameLocator : IGameLocator
     {
         var metaData = game.MetaData();
 
-        try
+        foreach (var id in metaData.SteamIDs)
         {
-            foreach (var steamGame in _steam.Games.Where(steamGame => metaData.SteamIDs.Contains(steamGame.ID)))
-            {
-                path = steamGame!.Path.ToAbsolutePath();
-                return true;
-            }
-        }
-        catch (Exception ex)
-        {
-            _logger.LogInformation(ex, "While finding {Game} from Steam", game);
+            if (!_steamGames.TryGetValue(id, out var found)) continue;
+            path = found;
+            return true;
         }
 
-        try
+        foreach (var id in metaData.GOGIDs)
         {
-            if (_gog != null)
-            {
-                foreach (var gogGame in _gog.Games.Where(gogGame => metaData.GOGIDs.Contains(gogGame.GameID)))
-                {
-                    path = gogGame!.Path.ToAbsolutePath();
-                    return true;
-                }
-            }
-        }
-        catch (Exception ex)
-        {
-            _logger.LogInformation(ex, "While finding {Game} from GoG", game);
+            if (!_gogGames.TryGetValue(id, out var found)) continue;
+            path = found;
+            return true;
         }
 
-        try
+        foreach (var id in metaData.EpicGameStoreIDs)
         {
-            if (_egs != null)
-            {
-                foreach (var egsGame in _egs.Games.Where(egsGame =>
-                             metaData.EpicGameStoreIDs.Contains(egsGame.CatalogItemId)))
-                {
-                    path = egsGame!.Path.ToAbsolutePath();
-                    return true;
-                }
-            }
-        }
-        catch (Exception ex)
-        {
-            _logger.LogInformation(ex, "While finding {Game} from Epic", game);
+            if (!_egsGames.TryGetValue(id, out var found)) continue;
+            path = found;
+            return true;
         }
 
-
-        try
+        foreach (var id in metaData.OriginIDs)
         {
-            if (_origin != null)
-            {
-                foreach (var originGame in _origin.Games.Where(originGame =>
-                             metaData.EpicGameStoreIDs.Contains(originGame.Id)))
-                {
-                    path = originGame.Path.ToAbsolutePath();
-                    return true;
-                }
-            }
-        }
-        catch (Exception ex)
-        {
-            _logger.LogInformation(ex, "While finding {Game} from Origin", game);
+            if (!_originGames.TryGetValue(id, out var found)) continue;
+            path = found;
+            return true;
         }
 
         path = default;

Wabbajack.Downloaders.GameFile/Wabbajack.Downloaders.GameFile.csproj

@@ -18,10 +18,10 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="GameFinder.StoreHandlers.EGS" Version="1.8.0" />
-        <PackageReference Include="GameFinder.StoreHandlers.GOG" Version="1.8.0" />
-        <PackageReference Include="GameFinder.StoreHandlers.Origin" Version="1.8.0" />
-        <PackageReference Include="GameFinder.StoreHandlers.Steam" Version="1.8.0" />
+        <PackageReference Include="GameFinder.StoreHandlers.EGS" Version="2.2.1" />
+        <PackageReference Include="GameFinder.StoreHandlers.GOG" Version="2.2.1" />
+        <PackageReference Include="GameFinder.StoreHandlers.Origin" Version="2.2.1" />
+        <PackageReference Include="GameFinder.StoreHandlers.Steam" Version="2.2.1" />
     </ItemGroup>
 
 </Project>

Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs

@@ -40,7 +40,8 @@ public class MediaFireDownloader : ADownloader<DTOs.DownloadStates.MediaFire>, I
 
     public override bool IsAllowed(ServerAllowList allowList, IDownloadState state)
     {
-        return true;
+        var mediaFireState = (DTOs.DownloadStates.MediaFire) state;
+        return allowList.AllowedPrefixes.Any(p => mediaFireState.Url.ToString().StartsWith(p, StringComparison.OrdinalIgnoreCase));
     }
 
     public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)

Wabbajack.Installer/StandardInstaller.cs

@@ -316,7 +316,7 @@ public class StandardInstaller : AInstaller<StandardInstaller>
                 var astate = bsa.FileStates.First(f => f.Path == state.Path);
                 var srcDirective = indexedByDestination[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)];
                 //DX10Files are lossy
-                if (astate is not BA2DX10File) 
+                if (astate is not BA2DX10File && srcDirective.IsDeterministic) 
                     ThrowOnNonMatchingHash(bsa, srcDirective, astate, hash);
                 return (srcDirective, hash);
             }).ToHashSet();

Wabbajack.Networking.NexusApi/NexusApi.cs

@@ -33,8 +33,7 @@ public class NexusApi
     private (ValidateInfo info, ResponseMetadata header) _lastValidatedInfo; 
 
     public NexusApi(ITokenProvider<NexusApiState> apiKey, ILogger<NexusApi> logger, HttpClient client,
-        IResource<HttpClient> limiter,
-        ApplicationInfo appInfo, JsonSerializerOptions jsonOptions)
+        IResource<HttpClient> limiter, ApplicationInfo appInfo, JsonSerializerOptions jsonOptions)
     {
         ApiKey = apiKey;
         _logger = logger;

Wabbajack.Paths.IO/AbsolutePathExtensions.cs

@@ -56,6 +56,11 @@ public static class AbsolutePathExtensions
         return new FileInfo(file.ToNativePath()).LastWriteTimeUtc;
     }
     
+    public static DateTime CreatedUtc(this AbsolutePath file)
+    {
+        return new FileInfo(file.ToNativePath()).CreationTimeUtc;
+    }
+
     public static DateTime LastModified(this AbsolutePath file)
     {
         return new FileInfo(file.ToNativePath()).LastWriteTime;

Wabbajack.VFS/FileHashCache.cs

@@ -92,6 +92,10 @@ public class FileHashCache
         if (result == default || result.Hash == default)
             return default;
         
+        // Fix for strange issue where dates are messed up on some systems
+        if (file.LastModifiedUtc() < file.CreatedUtc())
+            file.Touch();
+
         if (result.LastModified == file.LastModifiedUtc().ToFileTimeUtc())
         {
             return result.Hash;

Wabbajack.sln

@@ -111,6 +111,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".solutionItems", ".solution
 	ProjectSection(SolutionItems) = preProject
 		.gitignore = .gitignore
 		nuget.config = nuget.config
+		CHANGELOG.md = CHANGELOG.md
+		README.md = README.md
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Downloaders.GameFile", "Wabbajack.Downloaders.GameFile\Wabbajack.Downloaders.GameFile.csproj", "{4F252332-CA77-41DE-95A8-9DF38A81D675}"