Merge branch 'main' into dependabot/nuget/DynamicData-7.12.1

This commit is contained in:
Timothy Baldridge 2022-10-31 20:27:34 -06:00 committed by GitHub
commit 26e290d2a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 208 additions and 162 deletions

View File

@ -39,12 +39,7 @@ jobs:
include-prerelease: true
- name: Test
run: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:EnableWindowsTargeting=true --filter "Category!=FlakeyNetwork"
- uses: codecov/codecov-action@v3
with:
flags: unittests, ${{runner.os}}
verbose: true
run: dotnet test /p:EnableWindowsTargeting=true --filter "Category!=FlakeyNetwork"
#- name: Upload Test Folder on Failure
# if: ${{ failure() }}

View File

@ -1,5 +1,18 @@
### Changelog
#### Version - 3.0.4.0 - TBD
* upgrade GameFinder to 2.2.1
#### Version - 3.0.3.1 - 10/30/2022
* Fix file verification issues for CreatedBSAs
* Fix files during verification where CreatedDate > LastModified
#### Version - 3.0.3.0 - 10/26/2022
* Verify hashes of all installed files
* Verify contents of BSAs during installation
* Provide a new CLI command for verifying a installed modlist
* When downloading from one Nexus CDN server fails, WJ will now try alternate Nexus servers
#### Version - 3.0.2.3 - 10/19/2022
* HOTFIX: revert GameFinder library to 1.8 until it's a bit more forgiving of corrupt files

View File

@ -66,10 +66,10 @@ namespace Wabbajack
public ICommand OpenSettingsCommand { get; }
public string VersionDisplay { get; }
[Reactive]
public string ResourceStatus { get; set; }
[Reactive]
public string AppName { get; set; }
@ -98,7 +98,7 @@ namespace Wabbajack
MessageBus.Current.Listen<NavigateToGlobal>()
.Subscribe(m => HandleNavigateTo(m.Screen))
.DisposeWith(CompositeDisposable);
MessageBus.Current.Listen<NavigateTo>()
.Subscribe(m => HandleNavigateTo(m.ViewModel))
.DisposeWith(CompositeDisposable);
@ -106,7 +106,7 @@ namespace Wabbajack
MessageBus.Current.Listen<NavigateBack>()
.Subscribe(HandleNavigateBack)
.DisposeWith(CompositeDisposable);
MessageBus.Current.Listen<SpawnBrowserWindow>()
.ObserveOnGuiThread()
.Subscribe(HandleSpawnBrowserWindow)
@ -116,7 +116,7 @@ namespace Wabbajack
.Select(r => string.Join(", ", r.Where(r => r.Throughput > 0)
.Select(s => $"{s.Name} - {s.Throughput.ToFileSizeString()}/sec")))
.BindToStrict(this, view => view.ResourceStatus);
if (IsStartingFromModlist(out var path))
{
@ -134,25 +134,24 @@ namespace Wabbajack
var assembly = Assembly.GetExecutingAssembly();
var location = assembly.Location;
if (string.IsNullOrWhiteSpace(location))
location = Process.GetCurrentProcess().MainModule?.FileName ?? "";
location = Process.GetCurrentProcess().MainModule?.FileName ?? throw new Exception("Assembly location is unavailable!");
_logger.LogInformation("App Location: {Location}", assembly.Location);
var fvi = FileVersionInfo.GetVersionInfo(location);
Consts.CurrentMinimumWabbajackVersion = Version.Parse(fvi.FileVersion);
VersionDisplay = $"v{fvi.FileVersion}";
AppName = "WABBAJACK " + VersionDisplay;
_logger.LogInformation("Wabbajack Version: {FileVersion}", fvi.FileVersion);
Task.Run(() => _wjClient.SendMetric("started_wabbajack", fvi.FileVersion)).FireAndForget();
Task.Run(() => _wjClient.SendMetric("started_sha", ThisAssembly.Git.Sha));
// setup file association
try
{
var applicationRegistrationService =
_serviceProvider.GetRequiredService<IApplicationRegistrationService>();
var applicationRegistrationService = _serviceProvider.GetRequiredService<IApplicationRegistrationService>();
var applicationInfo = new ApplicationInfo(assembly);
var applicationInfo = new ApplicationInfo("Wabbajack", "Wabbajack", "Wabbajack", location);
applicationInfo.SupportedExtensions.Add("wabbajack");
applicationRegistrationService.RegisterApplication(applicationInfo);
}
@ -181,27 +180,27 @@ namespace Wabbajack
ActivePane = objViewModel;
}
private void HandleNavigateBack(NavigateBack navigateBack)
{
ActivePane = PreviousPanes.Last();
PreviousPanes.RemoveAt(PreviousPanes.Count - 1);
}
private void HandleManualDownload(ManualDownload manualDownload)
{
var handler = _serviceProvider.GetRequiredService<ManualDownloadHandler>();
handler.Intervention = manualDownload;
//MessageBus.Current.SendMessage(new OpenBrowserTab(handler));
}
private void HandleManualBlobDownload(ManualBlobDownload manualDownload)
{
var handler = _serviceProvider.GetRequiredService<ManualBlobDownloadHandler>();
handler.Intervention = manualDownload;
//MessageBus.Current.SendMessage(new OpenBrowserTab(handler));
}
private void HandleSpawnBrowserWindow(SpawnBrowserWindow msg)
{
var window = _serviceProvider.GetRequiredService<BrowserWindow>();
@ -213,7 +212,7 @@ namespace Wabbajack
{
if (s is NavigateToGlobal.ScreenType.Settings)
PreviousPanes.Add(ActivePane);
ActivePane = s switch
{
NavigateToGlobal.ScreenType.ModeSelectionView => ModeSelectionVM,

View File

@ -54,61 +54,58 @@ public class VerifyModlistInstall
_logger.LogInformation("Scanning files");
var errors = await list.Directives.PMapAllBatchedAsync(_limiter, async directive =>
{
if (directive is ArchiveMeta)
return null;
if (directive is RemappedInlineFile)
return null;
if (directive.To.InFolder(Consts.BSACreationDir))
return null;
var dest = directive.To.RelativeTo(installFolder);
if (!dest.FileExists())
var errors = await list.Directives.PMapAllBatchedAsync(_limiter, async directive =>
{
return new Result
if (!(directive is CreateBSA || directive.IsDeterministic))
return null;
if (directive.To.InFolder(Consts.BSACreationDir))
return null;
var dest = directive.To.RelativeTo(installFolder);
if (!dest.FileExists())
{
Path = directive.To,
Message = $"File does not exist directive {directive.GetType()}"
};
}
return new Result
{
Path = directive.To,
Message = $"File does not exist directive {directive.GetType()}"
};
}
if (Consts.KnownModifiedFiles.Contains(directive.To.FileName))
return null;
if (directive is CreateBSA bsa)
{
return await VerifyBSA(dest, bsa, byTo, token);
}
if (dest.Size() != directive.Size)
{
return new Result
{
Path = directive.To,
Message = $"Sizes do not match got {dest.Size()} expected {directive.Size}"
};
}
if (directive.Size > (1024 * 1024 * 128))
{
_logger.LogInformation("Hashing {Size} file at {Path}", directive.Size.ToFileSizeString(),
directive.To);
}
var hash = await AbsolutePathExtensions.Hash(dest, token);
if (hash != directive.Hash)
{
return new Result
{
Path = directive.To,
Message = $"Hashes do not match, got {hash} expected {directive.Hash}"
};
}
if (Consts.KnownModifiedFiles.Contains(directive.To.FileName))
return null;
if (directive is CreateBSA bsa)
{
return await VerifyBSA(dest, bsa, byTo, token);
}
if (dest.Size() != directive.Size)
{
return new Result
{
Path = directive.To,
Message = $"Sizes do not match got {dest.Size()} expected {directive.Size}"
};
}
if (directive.Size > (1024 * 1024 * 128))
{
_logger.LogInformation("Hashing {Size} file at {Path}", directive.Size.ToFileSizeString(),
directive.To);
}
var hash = await AbsolutePathExtensions.Hash(dest, token);
if (hash != directive.Hash)
{
return new Result
{
Path = directive.To,
Message = $"Hashes do not match, got {hash} expected {directive.Hash}"
};
}
return null;
}).Where(r => r != null)
.ToList();
@ -150,6 +147,9 @@ public class VerifyModlistInstall
var astate = bsa.FileStates.First(f => f.Path == state.Path);
var srcDirective = byTo[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)];
if (!srcDirective.IsDeterministic)
continue;
if (srcDirective.Hash != hash)
{
return new Result

View File

@ -1,3 +1,4 @@
using System.Text.Json.Serialization;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
@ -12,4 +13,6 @@ public abstract class Directive
/// location the file will be copied to, relative to the install path.
/// </summary>
public RelativePath To { get; set; }
[JsonIgnore] public virtual bool IsDeterministic => true;
}

View File

@ -8,4 +8,6 @@ namespace Wabbajack.DTOs.Directives;
public class ArchiveMeta : Directive
{
public RelativePath SourceDataID { get; set; }
public override bool IsDeterministic => false;
}

View File

@ -13,4 +13,6 @@ public class CreateBSA : Directive
public RelativePath TempID { get; set; }
public IArchive State { get; set; }
public AFile[] FileStates { get; set; } = Array.Empty<AFile>();
public override bool IsDeterministic => false;
}

View File

@ -9,4 +9,5 @@ namespace Wabbajack.DTOs.Directives;
[JsonAlias("RemappedInlineFile, Wabbajack.Lib")]
public class RemappedInlineFile : InlineFile
{
public override bool IsDeterministic => false;
}

View File

@ -11,4 +11,5 @@ public class TransformedTexture : FromArchive
/// The file to apply to the source file to patch it
/// </summary>
public ImageState ImageState { get; set; } = new();
public override bool IsDeterministic => false;
}

View File

@ -9,7 +9,6 @@ namespace Wabbajack.Downloaders.Dispatcher.Test;
public class VerificationCacheTests
{
private readonly TemporaryFileManager _temp;
private readonly ILogger<VerificationCache.VerificationCache> _logger;
public VerificationCacheTests(ILogger<VerificationCache.VerificationCache> logger)

View File

@ -1,3 +1,6 @@
using System.Runtime.InteropServices;
using GameFinder.Common;
using GameFinder.RegistryUtils;
using GameFinder.StoreHandlers.EGS;
using GameFinder.StoreHandlers.GOG;
using GameFinder.StoreHandlers.Origin;
@ -5,67 +8,120 @@ using GameFinder.StoreHandlers.Steam;
using Microsoft.Extensions.Logging;
using Wabbajack.DTOs;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
namespace Wabbajack.Downloaders.GameFile;
public class GameLocator : IGameLocator
{
private readonly EGSHandler? _egs;
private readonly SteamHandler _steam;
private readonly GOGHandler? _gog;
private readonly EGSHandler? _egs;
private readonly OriginHandler? _origin;
private readonly Dictionary<int, AbsolutePath> _steamGames = new();
private readonly Dictionary<long, AbsolutePath> _gogGames = new();
private readonly Dictionary<string, AbsolutePath> _egsGames = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, AbsolutePath> _originGames = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<Game, AbsolutePath> _locationCache;
private readonly ILogger<GameLocator> _logger;
private readonly OriginHandler? _origin;
private readonly SteamHandler _steam;
public GameLocator(ILogger<GameLocator> logger)
{
_logger = logger;
_steam = new SteamHandler(logger);
if (OperatingSystem.IsWindows())
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
_origin = new OriginHandler(true, false, logger);
_gog = new GOGHandler(logger);
var windowsRegistry = new WindowsRegistry();
_egs = new EGSHandler(logger);
_steam = new SteamHandler(windowsRegistry);
_gog = new GOGHandler(windowsRegistry);
_egs = new EGSHandler(windowsRegistry);
_origin = new OriginHandler();
}
else
{
_steam = new SteamHandler(null);
}
_locationCache = new Dictionary<Game, AbsolutePath>();
FindAllGames();
}
private void FindAllGames()
{
try
{
_steam.FindAllGames();
FindStoreGames(_steam, _steamGames, game => game.Path);
}
catch (Exception ex)
catch (Exception e)
{
_logger.LogError(ex, "While finding Steam games");
_logger.LogError(e, "While finding games installed with Steam");
}
try
{
_origin?.FindAllGames();
FindStoreGames(_gog, _gogGames, game => game.Path);
}
catch (Exception ex)
catch (Exception e)
{
_logger.LogInformation(ex, "While finding Origin games");
_logger.LogError(e, "While finding games installed with GOG Galaxy");
}
try
{
_gog?.FindAllGames();
FindStoreGames(_egs, _egsGames, game => game.InstallLocation);
}
catch (Exception ex)
catch (Exception e)
{
_logger.LogInformation(ex, "While finding GoG games");
_logger.LogError(e, "While finding games installed with the Epic Games Store");
}
try
{
_egs?.FindAllGames();
FindStoreGames(_origin, _originGames, game => game.InstallPath);
}
catch (Exception ex)
catch (Exception e)
{
_logger.LogInformation(ex, "While finding Epic Games");
_logger.LogError(e, "While finding games installed with Origin");
}
}
private void FindStoreGames<TGame, TId>(
AHandler<TGame, TId>? handler,
Dictionary<TId, AbsolutePath> paths,
Func<TGame, string> getPath)
where TGame : class
{
if (handler is null) return;
var games = handler.FindAllGamesById(out var errors);
foreach (var (id, game) in games)
{
try
{
var path = getPath(game).ToAbsolutePath();
if (!path.DirectoryExists())
{
_logger.LogError("Game does not exist: {Game}", game);
continue;
}
paths[id] = path;
_logger.LogDebug("Found {Game}", game);
}
catch (Exception e)
{
_logger.LogError(e, "While locating {Game}", game);
}
}
foreach (var error in errors)
{
_logger.LogError("{Error}", error);
}
}
@ -102,71 +158,35 @@ public class GameLocator : IGameLocator
{
var metaData = game.MetaData();
try
foreach (var id in metaData.SteamIDs)
{
foreach (var steamGame in _steam.Games.Where(steamGame => metaData.SteamIDs.Contains(steamGame.ID)))
{
path = steamGame!.Path.ToAbsolutePath();
return true;
}
}
catch (Exception ex)
{
_logger.LogInformation(ex, "While finding {Game} from Steam", game);
if (!_steamGames.TryGetValue(id, out var found)) continue;
path = found;
return true;
}
try
foreach (var id in metaData.GOGIDs)
{
if (_gog != null)
{
foreach (var gogGame in _gog.Games.Where(gogGame => metaData.GOGIDs.Contains(gogGame.GameID)))
{
path = gogGame!.Path.ToAbsolutePath();
return true;
}
}
}
catch (Exception ex)
{
_logger.LogInformation(ex, "While finding {Game} from GoG", game);
if (!_gogGames.TryGetValue(id, out var found)) continue;
path = found;
return true;
}
try
foreach (var id in metaData.EpicGameStoreIDs)
{
if (_egs != null)
{
foreach (var egsGame in _egs.Games.Where(egsGame =>
metaData.EpicGameStoreIDs.Contains(egsGame.CatalogItemId)))
{
path = egsGame!.Path.ToAbsolutePath();
return true;
}
}
}
catch (Exception ex)
{
_logger.LogInformation(ex, "While finding {Game} from Epic", game);
if (!_egsGames.TryGetValue(id, out var found)) continue;
path = found;
return true;
}
try
foreach (var id in metaData.OriginIDs)
{
if (_origin != null)
{
foreach (var originGame in _origin.Games.Where(originGame =>
metaData.EpicGameStoreIDs.Contains(originGame.Id)))
{
path = originGame.Path.ToAbsolutePath();
return true;
}
}
}
catch (Exception ex)
{
_logger.LogInformation(ex, "While finding {Game} from Origin", game);
if (!_originGames.TryGetValue(id, out var found)) continue;
path = found;
return true;
}
path = default;
return false;
}
}
}

View File

@ -18,10 +18,10 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="GameFinder.StoreHandlers.EGS" Version="1.8.0" />
<PackageReference Include="GameFinder.StoreHandlers.GOG" Version="1.8.0" />
<PackageReference Include="GameFinder.StoreHandlers.Origin" Version="1.8.0" />
<PackageReference Include="GameFinder.StoreHandlers.Steam" Version="1.8.0" />
<PackageReference Include="GameFinder.StoreHandlers.EGS" Version="2.2.1" />
<PackageReference Include="GameFinder.StoreHandlers.GOG" Version="2.2.1" />
<PackageReference Include="GameFinder.StoreHandlers.Origin" Version="2.2.1" />
<PackageReference Include="GameFinder.StoreHandlers.Steam" Version="2.2.1" />
</ItemGroup>
</Project>

View File

@ -40,7 +40,8 @@ public class MediaFireDownloader : ADownloader<DTOs.DownloadStates.MediaFire>, I
public override bool IsAllowed(ServerAllowList allowList, IDownloadState state)
{
return true;
var mediaFireState = (DTOs.DownloadStates.MediaFire) state;
return allowList.AllowedPrefixes.Any(p => mediaFireState.Url.ToString().StartsWith(p, StringComparison.OrdinalIgnoreCase));
}
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
@ -105,26 +106,26 @@ public class MediaFireDownloader : ADownloader<DTOs.DownloadStates.MediaFire>, I
if (!result.IsSuccessStatusCode)
return null;
if (job != null)
if (job != null)
job.Size = result.Content.Headers.ContentLength ?? 0;
if (result.Content.Headers.ContentType!.MediaType!.StartsWith("text/html",
StringComparison.OrdinalIgnoreCase))
{
var bodyData = await result.Content.ReadAsStringAsync((CancellationToken) token);
if (job != null)
if (job != null)
await job.Report((int) (job.Size ?? 0), (CancellationToken) token);
var body = new HtmlDocument();
body.LoadHtml(bodyData);
var node = body.DocumentNode.DescendantsAndSelf().FirstOrDefault(d => d.HasClass("input") && d.HasClass("popsok") &&
d.GetAttributeValue("aria-label", "") ==
"Download file");
if (node != null)
if (node != null)
return new Uri(node.GetAttributeValue("href", "not-found"));
var startText = "window.location.href = '";
var start = body.DocumentNode.InnerHtml.IndexOf(startText, StringComparison.CurrentCultureIgnoreCase);
if (start != -1)
{
var end = body.DocumentNode.InnerHtml.IndexOf("\'", start + startText.Length,
@ -141,4 +142,4 @@ public class MediaFireDownloader : ADownloader<DTOs.DownloadStates.MediaFire>, I
{
return new[] {$"directURL={state.Url}"};
}
}
}

View File

@ -316,7 +316,7 @@ public class StandardInstaller : AInstaller<StandardInstaller>
var astate = bsa.FileStates.First(f => f.Path == state.Path);
var srcDirective = indexedByDestination[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)];
//DX10Files are lossy
if (astate is not BA2DX10File)
if (astate is not BA2DX10File && srcDirective.IsDeterministic)
ThrowOnNonMatchingHash(bsa, srcDirective, astate, hash);
return (srcDirective, hash);
}).ToHashSet();

View File

@ -33,8 +33,7 @@ public class NexusApi
private (ValidateInfo info, ResponseMetadata header) _lastValidatedInfo;
public NexusApi(ITokenProvider<NexusApiState> apiKey, ILogger<NexusApi> logger, HttpClient client,
IResource<HttpClient> limiter,
ApplicationInfo appInfo, JsonSerializerOptions jsonOptions)
IResource<HttpClient> limiter, ApplicationInfo appInfo, JsonSerializerOptions jsonOptions)
{
ApiKey = apiKey;
_logger = logger;

View File

@ -55,6 +55,11 @@ public static class AbsolutePathExtensions
{
return new FileInfo(file.ToNativePath()).LastWriteTimeUtc;
}
public static DateTime CreatedUtc(this AbsolutePath file)
{
return new FileInfo(file.ToNativePath()).CreationTimeUtc;
}
public static DateTime LastModified(this AbsolutePath file)
{

View File

@ -91,6 +91,10 @@ public class FileHashCache
var result = await Get(file);
if (result == default || result.Hash == default)
return default;
// Fix for strange issue where dates are messed up on some systems
if (file.LastModifiedUtc() < file.CreatedUtc())
file.Touch();
if (result.LastModified == file.LastModifiedUtc().ToFileTimeUtc())
{

View File

@ -111,6 +111,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".solutionItems", ".solution
ProjectSection(SolutionItems) = preProject
.gitignore = .gitignore
nuget.config = nuget.config
CHANGELOG.md = CHANGELOG.md
README.md = README.md
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Downloaders.GameFile", "Wabbajack.Downloaders.GameFile\Wabbajack.Downloaders.GameFile.csproj", "{4F252332-CA77-41DE-95A8-9DF38A81D675}"