diff --git a/CHANGELOG.md b/CHANGELOG.md index 670c554f..702f5035 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ ### Changelog +#### Version - 2.4.1.2 - 1/29/2020 +* Don't install .meta files for files sourced from the game folder +* Fix bug MO2 archive name detection in .meta files (rare bug with FO4VR and other like games) +* Catch exceptions when ECS downloads manifest data +* Don't double-index game files in some situations (duplicate game names in config files) +* Update all deps + #### Version - 2.4.1.1 - 1/13/2020 * HOTFIX: Fix game file sources that don't have MO2 specific names diff --git a/Compression.BSA.Test/BSATests.cs b/Compression.BSA.Test/BSATests.cs index 765c48a5..4b4b56d0 100644 --- a/Compression.BSA.Test/BSATests.cs +++ b/Compression.BSA.Test/BSATests.cs @@ -70,6 +70,7 @@ namespace Compression.BSA.Test } [Theory] + //[InlineData(Game.SkyrimSpecialEdition, 29194)] // 3D NPCS This fails not sure why [InlineData(Game.SkyrimSpecialEdition, 12604)] // SkyUI [InlineData(Game.Skyrim, 3863)] // SkyUI [InlineData(Game.Skyrim, 51473)] // INeed @@ -150,6 +151,7 @@ namespace Compression.BSA.Test Assert.Equal(pair.ai.Path, pair.bi.Path); //Equal(pair.ai.Compressed, pair.bi.Compressed); Assert.Equal(pair.ai.Size, pair.bi.Size); + Utils.Log($"Comparing {pair.ai.Path} to {pair.bi.Path}"); Assert.Equal(await GetData(pair.ai), await GetData(pair.bi)); }); } diff --git a/Compression.BSA/Utils.cs b/Compression.BSA/Utils.cs index 1a3e9b85..3fb43afc 100644 --- a/Compression.BSA/Utils.cs +++ b/Compression.BSA/Utils.cs @@ -6,6 +6,9 @@ using System.Threading.Tasks; using Wabbajack.Common; using Path = Alphaleonis.Win32.Filesystem.Path; +// Yeah, we know, but BSAs use UTF7, that's how old they are +#pragma warning disable 618 + namespace Compression.BSA { public static class BSAUtils @@ -20,11 +23,12 @@ namespace Compression.BSA private static Encoding GetEncoding(VersionType version) { - if (version == VersionType.TES3) - return Encoding.ASCII; - if (version == VersionType.SSE) - return Windows1252; - return Encoding.UTF7; + return version switch + { + VersionType.TES3 => Encoding.ASCII, + VersionType.SSE => Windows1252, + _ => Encoding.UTF7 + }; } public static string ReadStringLen(this BinaryReader rdr, VersionType version) @@ -72,9 +76,10 @@ namespace Compression.BSA } /// - /// Returns bytes for a \0 terminated string + /// Returns \0 terminated bytes for a string encoded with a given BSA version's encoding format /// /// + /// /// public static byte[] ToBZString(this RelativePath val, VersionType version) { @@ -101,9 +106,10 @@ namespace Compression.BSA } /// - /// Returns bytes for a \0 terminated string prefixed by a length + /// Returns bytes for a string with a length prefix, version is the BSA version /// /// + /// /// public static byte[] ToTermString(this string val, VersionType version) { diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 518eae21..b45f22b0 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -19,7 +19,7 @@ - + diff --git a/Wabbajack.Common/Paths/AbsolutePath.cs b/Wabbajack.Common/Paths/AbsolutePath.cs index 26d8a27e..a3cb416a 100644 --- a/Wabbajack.Common/Paths/AbsolutePath.cs +++ b/Wabbajack.Common/Paths/AbsolutePath.cs @@ -89,13 +89,13 @@ namespace Wabbajack.Common public ValueTask Create() { var path = _path; - return CircuitBreaker.WithAutoRetryAsync(async () => File.Open(path, FileMode.Create, FileAccess.ReadWrite)); + return CircuitBreaker.WithAutoRetryAsync(async () => File.Open(path, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 1024 * 32)); } public ValueTask OpenWrite() { var path = _path; - return CircuitBreaker.WithAutoRetryAsync(async () => File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite)); + return CircuitBreaker.WithAutoRetryAsync(async () => File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read, 1024 * 32)); } public async Task WriteAllTextAsync(string text) diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 86b25d5e..5350df06 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -49,7 +49,7 @@ - + @@ -61,7 +61,7 @@ - + diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index 27302b91..a1625aa8 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -151,7 +151,7 @@ TOP: { read = await webs.ReadAsync(buffer, 0, bufferSize); } - catch (Exception ex) + catch (Exception) { if (readThisCycle == 0) throw; diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index 668d89d8..7f1a9aa4 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -262,7 +262,7 @@ namespace Wabbajack.Lib if (HashedArchives.TryGetValue(archive.Hash, out var paths)) { var metaPath = paths.WithExtension(Consts.MetaFileExtension); - if (!metaPath.Exists) + if (!metaPath.Exists && !(archive.State is GameFileSourceDownloader.State)) { Status($"Writing {metaPath.FileName}"); var meta = AddInstalled(archive.State.GetMetaIni()).ToArray(); diff --git a/Wabbajack.Lib/NexusApi/HtmlInterface.cs b/Wabbajack.Lib/NexusApi/HtmlInterface.cs index 0cbdcd69..fc07dfec 100644 --- a/Wabbajack.Lib/NexusApi/HtmlInterface.cs +++ b/Wabbajack.Lib/NexusApi/HtmlInterface.cs @@ -1,8 +1,12 @@ -using System.Linq; +using System; +using System.Linq; using System.Net; +using System.Threading; using System.Threading.Tasks; +using HtmlAgilityPack; using Wabbajack.Common; using Wabbajack.Lib.LibCefHelpers; +using Wabbajack.Lib.WebAutomation; namespace Wabbajack.Lib.NexusApi { @@ -10,15 +14,21 @@ namespace Wabbajack.Lib.NexusApi { public static async Task GetUploadPermissions(Game game, long modId) { - var client = new Lib.Http.Client(); - if (Utils.HaveEncryptedJson("nexus-cookies")) + HtmlDocument response; + using (var driver = await Driver.Create()) { - var cookies = await Utils.FromEncryptedJson("nexus-cookies"); - client.AddCookies(cookies); + await driver.NavigateTo(new Uri($"https://nexusmods.com/{game.MetaData().NexusName}/mods/{modId}")); + TOP: + response = await driver.GetHtmlAsync(); + + if (response!.Text!.Contains("This process is automatic. Your browser will redirect to your requested content shortly.")) + { + await Task.Delay(5000); + goto TOP; + } + } - var response = await client.GetHtmlAsync($"https://nexusmods.com/{game.MetaData().NexusName}/mods/{modId}"); - var hidden = response.DocumentNode .Descendants() .Any(n => n.Id == $"{modId}-title" && n.InnerText == "Hidden mod"); diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 107e0da6..441ba5ce 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -25,7 +25,7 @@ 2.2.2.1 - 1.11.29 + 1.11.30 1.8.2 @@ -37,13 +37,13 @@ 2.1.1 - 13.0.27 + 13.1.1 - 13.0.27 + 13.1.1 - 0.26.0 + 0.27.1 5.0.0 diff --git a/Wabbajack.Lib/WebAutomation/WebAutomation.cs b/Wabbajack.Lib/WebAutomation/WebAutomation.cs index b1c7d22f..c259d327 100644 --- a/Wabbajack.Lib/WebAutomation/WebAutomation.cs +++ b/Wabbajack.Lib/WebAutomation/WebAutomation.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using System.Windows; using CefSharp; using CefSharp.OffScreen; +using HtmlAgilityPack; using Wabbajack.Common; using Wabbajack.Lib.LibCefHelpers; @@ -83,6 +84,14 @@ namespace Wabbajack.Lib.WebAutomation return await _browser.GetSourceAsync(); } + public async ValueTask GetHtmlAsync() + { + var body = await GetSourceAsync(); + var doc = new HtmlDocument(); + doc.LoadHtml(body); + return doc; + } + public Action DownloadHandler { set => _driver.DownloadHandler = value; } diff --git a/Wabbajack.Server.Test/Wabbajack.Server.Test.csproj b/Wabbajack.Server.Test/Wabbajack.Server.Test.csproj index 4f3bf2ac..48b7f721 100644 --- a/Wabbajack.Server.Test/Wabbajack.Server.Test.csproj +++ b/Wabbajack.Server.Test/Wabbajack.Server.Test.csproj @@ -10,7 +10,10 @@ - + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index 191d190b..74a62210 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -18,7 +18,7 @@ - + diff --git a/Wabbajack.Test/ContentRightsManagementTests.cs b/Wabbajack.Test/ContentRightsManagementTests.cs index 7526097a..a739813d 100644 --- a/Wabbajack.Test/ContentRightsManagementTests.cs +++ b/Wabbajack.Test/ContentRightsManagementTests.cs @@ -9,10 +9,11 @@ using Wabbajack.Common; using System.Threading.Tasks; using Wabbajack.Lib.NexusApi; using Xunit; +using Xunit.Abstractions; namespace Wabbajack.Test { - public class ContentRightsManagementTests : IDisposable + public class ContentRightsManagementTests : ATestBase { private ValidateModlist validate; private WorkQueue queue; @@ -27,17 +28,12 @@ namespace Wabbajack.Test "; - public ContentRightsManagementTests() - { - queue = new WorkQueue(); - validate = new ValidateModlist(); - validate.LoadServerWhitelist(server_whitelist); - } - public void Dispose() + + public override void Dispose() { queue?.Dispose(); - + base.Dispose(); } @@ -127,5 +123,12 @@ namespace Wabbajack.Test Assert.Equal(HTMLInterface.PermissionValue.NotFound, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 24287)); } + + public ContentRightsManagementTests(ITestOutputHelper output) : base(output) + { + queue = new WorkQueue(); + validate = new ValidateModlist(); + validate.LoadServerWhitelist(server_whitelist); + } } } diff --git a/Wabbajack.Test/Wabbajack.Test.csproj b/Wabbajack.Test/Wabbajack.Test.csproj index 97dd2e09..5ca5f9b9 100644 --- a/Wabbajack.Test/Wabbajack.Test.csproj +++ b/Wabbajack.Test/Wabbajack.Test.csproj @@ -33,7 +33,10 @@ - + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index a9fb03cc..2d24b797 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -201,14 +201,17 @@ namespace Wabbajack.VirtualFileSystem } } + /// - /// Extract the set of files and call the callback for each, handing it a stream factory and the virtual file, - /// top level archives (native archives) will be processed in parallel. Duplicate files will not be + /// Extracts a file /// - /// < - /// - /// + /// Work queue to use when required by some formats + /// Predefined list of files to extract, all others will be skipped + /// Func called for each file extracted + /// Optional: folder to use for temporary storage + /// Optional: Status update tracker /// + /// public async Task Extract(WorkQueue queue, HashSet files, Func callback, AbsolutePath? tempFolder = null, StatusUpdateTracker updateTracker = null) { var top = new VirtualFile(); diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs index 1809e4d8..22eeb1dd 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs @@ -17,7 +17,7 @@ namespace Wabbajack.VirtualFileSystem { public static class FileExtractor2 { - public static readonly SignatureChecker ArchiveSigs = new SignatureChecker(Definitions.FileType.TES3, + public static readonly SignatureChecker ArchiveSigs = new(Definitions.FileType.TES3, Definitions.FileType.BSA, Definitions.FileType.BA2, Definitions.FileType.ZIP, @@ -26,19 +26,19 @@ namespace Wabbajack.VirtualFileSystem Definitions.FileType.RAR_NEW, Definitions.FileType._7Z); - private static Extension OMODExtension = new Extension(".omod"); - private static Extension FOMODExtension = new Extension(".fomod"); + private static Extension OMODExtension = new(".omod"); + private static Extension FOMODExtension = new(".fomod"); - private static Extension BSAExtension = new Extension(".bsa"); + private static Extension BSAExtension = new(".bsa"); public static readonly HashSet ExtractableExtensions = new HashSet { - new Extension(".bsa"), - new Extension(".ba2"), - new Extension(".7z"), - new Extension(".7zip"), - new Extension(".rar"), - new Extension(".zip"), + new(".bsa"), + new(".ba2"), + new(".7z"), + new(".7zip"), + new(".rar"), + new(".zip"), OMODExtension, FOMODExtension }; @@ -196,7 +196,7 @@ namespace Wabbajack.VirtualFileSystem else { spoolFile = new TempFile(tempPath.Combine(Guid.NewGuid().ToString()) - .WithExtension(source.Extension)); + .WithExtension(sf.Name.FileName.Extension)); await using var s = await sf.GetStream(); await spoolFile.Path.WriteAllAsync(s); source = spoolFile.Path; diff --git a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj index cdedcc3a..b18f807f 100644 --- a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj +++ b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj @@ -17,7 +17,7 @@ - + diff --git a/Wabbajack/View Models/BackNavigatingVM.cs b/Wabbajack/View Models/BackNavigatingVM.cs index b4cef5e5..120f1941 100644 --- a/Wabbajack/View Models/BackNavigatingVM.cs +++ b/Wabbajack/View Models/BackNavigatingVM.cs @@ -29,7 +29,7 @@ namespace Wabbajack public ViewModel NavigateBackTarget { get; set; } public ReactiveCommand BackCommand { get; protected set; } - private readonly ObservableAsPropertyHelper _IsActive; + protected ObservableAsPropertyHelper _IsActive; public bool IsActive => _IsActive.Value; public Subject IsBackEnabledSubject { get; } = new Subject(); diff --git a/Wabbajack/View Models/Compilers/CompilerVM.cs b/Wabbajack/View Models/Compilers/CompilerVM.cs index 38f78933..9ab5ac1d 100644 --- a/Wabbajack/View Models/Compilers/CompilerVM.cs +++ b/Wabbajack/View Models/Compilers/CompilerVM.cs @@ -45,7 +45,6 @@ namespace Wabbajack public ObservableCollectionExtended Log => MWVM.Log; - public ReactiveCommand BackCommand { get; } public ReactiveCommand GoToCommand { get; } public ReactiveCommand CloseWhenCompleteCommand { get; } public ReactiveCommand BeginCommand { get; } diff --git a/Wabbajack/View Models/Installers/InstallerVM.cs b/Wabbajack/View Models/Installers/InstallerVM.cs index ab8de50d..d92c0521 100644 --- a/Wabbajack/View Models/Installers/InstallerVM.cs +++ b/Wabbajack/View Models/Installers/InstallerVM.cs @@ -85,15 +85,10 @@ namespace Wabbajack private readonly ObservableAsPropertyHelper _LoadingModlist; public bool LoadingModlist => _LoadingModlist.Value; - private readonly ObservableAsPropertyHelper _IsActive; - public bool IsActive => _IsActive.Value; - // Command properties public ReactiveCommand ShowManifestCommand { get; } public ReactiveCommand OpenReadmeCommand { get; } public ReactiveCommand VisitModListWebsiteCommand { get; } - public ReactiveCommand BackCommand { get; } - public ReactiveCommand CloseWhenCompleteCommand { get; } public ReactiveCommand GoToInstallCommand { get; } diff --git a/Wabbajack/View Models/Installers/MO2InstallerVM.cs b/Wabbajack/View Models/Installers/MO2InstallerVM.cs index ff463bc7..a49bb832 100644 --- a/Wabbajack/View Models/Installers/MO2InstallerVM.cs +++ b/Wabbajack/View Models/Installers/MO2InstallerVM.cs @@ -90,7 +90,7 @@ namespace Wabbajack // Load settings _CurrentSettings = installerVM.WhenAny(x => x.ModListLocation.TargetPath) - .Select(path => path == null ? null : installerVM.MWVM.Settings.Installer.Mo2ModlistSettings.TryCreate(path)) + .Select(path => path == default ? null : installerVM.MWVM.Settings.Installer.Mo2ModlistSettings.TryCreate(path)) .ToGuiProperty(this, nameof(CurrentSettings)); this.WhenAny(x => x.CurrentSettings) .Pairwise() diff --git a/Wabbajack/View Models/Settings/AuthorFilesVM.cs b/Wabbajack/View Models/Settings/AuthorFilesVM.cs index 09869587..2ef85bca 100644 --- a/Wabbajack/View Models/Settings/AuthorFilesVM.cs +++ b/Wabbajack/View Models/Settings/AuthorFilesVM.cs @@ -16,9 +16,6 @@ namespace Wabbajack private readonly ObservableAsPropertyHelper _isVisible; public Visibility IsVisible => _isVisible.Value; - - private readonly ObservableAsPropertyHelper _selectedFile; - public ICommand SelectFile { get; } public ICommand HyperlinkCommand { get; } public IReactiveCommand Upload { get; } @@ -69,7 +66,7 @@ namespace Wabbajack _isUploading.OnNext(false); } }, IsUploading.StartWith(false).Select(u => !u) - .CombineLatest(Picker.WhenAnyValue(t => t.TargetPath).Select(f => f != null), + .CombineLatest(Picker.WhenAnyValue(t => t.TargetPath).Select(f => f != default), (a, b) => a && b)); } } diff --git a/Wabbajack/View Models/WebBrowserVM.cs b/Wabbajack/View Models/WebBrowserVM.cs index 2885c8df..d5876b77 100644 --- a/Wabbajack/View Models/WebBrowserVM.cs +++ b/Wabbajack/View Models/WebBrowserVM.cs @@ -45,9 +45,10 @@ namespace Wabbajack return new WebBrowserVM(url); } - public void Dispose() + public override void Dispose() { Browser.Dispose(); + base.Dispose(); } } } diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 3ee1accc..fd5cc509 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -71,9 +71,9 @@ - - - + + +