From 4aaa46090ce0d775691044e4db2f7fb9e1978ba7 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 13 May 2021 14:16:29 -0600 Subject: [PATCH 1/3] Few small fixes and optimizations --- CHANGELOG.md | 3 +++ Wabbajack/View Models/Compilers/MO2CompilerVM.cs | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5ca9720..41fcbe7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ### Changelog +#### Version - 2.4.4.2 - ??? +* Modlists are now exported as X.wabbajack where X is the name chosen in the Compiler UI + #### Version - 2.4.4.1 - 5/1/2021 * HOTFIX: downgrade cefsharp to fix the in-app browser (fixes Nexus login issue) diff --git a/Wabbajack/View Models/Compilers/MO2CompilerVM.cs b/Wabbajack/View Models/Compilers/MO2CompilerVM.cs index 63b140e4..12e92bf7 100644 --- a/Wabbajack/View Models/Compilers/MO2CompilerVM.cs +++ b/Wabbajack/View Models/Compilers/MO2CompilerVM.cs @@ -184,13 +184,18 @@ namespace Wabbajack public async Task> Compile() { AbsolutePath outputFile; + + var profileName = string.IsNullOrWhiteSpace(ModlistSettings.ModListName) + ? MOProfile + : ModlistSettings.ModListName; + if (Parent.OutputLocation.TargetPath == default) { - outputFile = (MOProfile + Consts.ModListExtension).RelativeTo(AbsolutePath.EntryPoint); + outputFile = (profileName + Consts.ModListExtension).RelativeTo(AbsolutePath.EntryPoint); } else { - outputFile = Parent.OutputLocation.TargetPath.Combine(MOProfile + Consts.ModListExtension); + outputFile = Parent.OutputLocation.TargetPath.Combine(profileName + Consts.ModListExtension); } try From 04bbbfed7f63ba48e0ee5a22c5d5a9fd1ea1a5c6 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 17 May 2021 16:37:01 -0600 Subject: [PATCH 2/3] Network workaround mode --- Wabbajack.Common/Consts.cs | 5 ++++ Wabbajack.Lib/Http/Client.cs | 25 +++++++++++++++++++ Wabbajack.Server/DataLayer/Patches.cs | 15 +++++++++-- Wabbajack.Server/Services/ListValidator.cs | 20 ++++++++------- Wabbajack.Test/MetricsTests.cs | 17 ++++++++----- Wabbajack/Settings.cs | 11 ++++++++ .../Settings/PerformanceSettingsView.xaml | 14 ++++++++++- .../Settings/PerformanceSettingsView.xaml.cs | 2 ++ 8 files changed, 91 insertions(+), 18 deletions(-) diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 49819adf..1c07b236 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -118,6 +118,11 @@ namespace Wabbajack.Common public static string WabbajackCacheLocation = "http://build.wabbajack.org/nexus_api_cache/"; public static string WabbajackCacheHostname = "build.wabbajack.org"; + + // Direct IP to the CDN + public static string NetworkWorkaroundHost = "storage.wabbajack.org"; //"51.81.80.6"; + public static bool UseNetworkWorkaroundMode = false; + public static Uri WabbajackBuildServerUri = new Uri("https://build.wabbajack.org"); public static int WabbajackCachePort = 80; public static int MaxHTTPRetries = 4; diff --git a/Wabbajack.Lib/Http/Client.cs b/Wabbajack.Lib/Http/Client.cs index 66e74d43..200ec85c 100644 --- a/Wabbajack.Lib/Http/Client.cs +++ b/Wabbajack.Lib/Http/Client.cs @@ -75,6 +75,7 @@ namespace Wabbajack.Lib.Http public async Task SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = true, CancellationToken? token = null) { + msg = FixupMessage(msg); foreach (var (k, v) in Headers) msg.Headers.Add(k, v); if (Cookies.Count > 0) @@ -123,6 +124,30 @@ namespace Wabbajack.Lib.Http } + private Dictionary> _workaroundMappings = new() + { + {"patches.wabbajack.org", () => (Consts.NetworkWorkaroundHost, "patches.wabbajack.org")}, + {"authored-files.wabbajack.org", () => (Consts.NetworkWorkaroundHost, "authored-files.wabbajack.org")}, + {"mirror.wabbajack.org", () => (Consts.NetworkWorkaroundHost, "mirror.wabbajack.org")}, + {"build.wabbajack.org", () => (Consts.NetworkWorkaroundHost, "proxy-build.wabbajack.org")}, + {"test-files.wabbajack.org", () => (Consts.NetworkWorkaroundHost, "test-files.wabbajack.org")}, + }; + private HttpRequestMessage FixupMessage(HttpRequestMessage msg) + { + if (!Consts.UseNetworkWorkaroundMode) return msg; + var uri = new UriBuilder(msg.RequestUri!); + + if (!_workaroundMappings.TryGetValue(uri.Host, out var f)) + return msg; + + var (ip, host) = f(); + uri.Host = ip; + msg.Headers.Host = host; + msg.RequestUri = uri.Uri; + + return msg; + } + private HttpRequestMessage CloneMessage(HttpRequestMessage msg) { var new_message = new HttpRequestMessage(msg.Method, msg.RequestUri); diff --git a/Wabbajack.Server/DataLayer/Patches.cs b/Wabbajack.Server/DataLayer/Patches.cs index f46597f3..412c0430 100644 --- a/Wabbajack.Server/DataLayer/Patches.cs +++ b/Wabbajack.Server/DataLayer/Patches.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; using Dapper; @@ -16,11 +17,21 @@ namespace Wabbajack.Server.DataLayer /// /// /// - public async Task AddPatch(Patch patch) + public async Task AddPatch(Patch patch) { await using var conn = await Open(); + await using var trans = conn.BeginTransaction(); + + if (await conn.QuerySingleOrDefaultAsync<(Guid, Guid)>( + "Select SrcID, DestID FROM dbo.Patches where SrcID = @SrcId and DestID = @DestId", + new {SrcId = patch.Src.Id, DestId = patch.Dest.Id}, trans) != default) + return false; + await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)", - new {SrcId = patch.Src.Id, DestId = patch.Dest.Id}); + new {SrcId = patch.Src.Id, DestId = patch.Dest.Id}, trans); + await trans.CommitAsync(); + return true; + } /// diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 42d5623c..46f9fd80 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -322,16 +322,18 @@ namespace Wabbajack.Server.Services var existing = await _sql.FindPatch(srcDownload.Id, destDownload.Id); if (existing == null) { - await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload}); + if (await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload})) + { - _logger.Log(LogLevel.Information, - $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}"); - await _discord.Send(Channel.Ham, - new DiscordMessage - { - Content = - $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`" - }); + _logger.Log(LogLevel.Information, + $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}"); + await _discord.Send(Channel.Ham, + new DiscordMessage + { + Content = + $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`" + }); + } } await upgrade.NewFile.DisposeAsync(); diff --git a/Wabbajack.Test/MetricsTests.cs b/Wabbajack.Test/MetricsTests.cs index 8d8b0a68..f3d1e64c 100644 --- a/Wabbajack.Test/MetricsTests.cs +++ b/Wabbajack.Test/MetricsTests.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using Wabbajack.Common; using Wabbajack.Lib; using Xunit; @@ -10,13 +11,17 @@ namespace Wabbajack.Test [Fact] public async Task CanSendExceptions() { - try + foreach (var mode in new[] {true, false}) { - throw new Exception("Test Exception"); - } - catch (Exception ex) - { - await Metrics.Error(this.GetType(), ex); + Consts.UseNetworkWorkaroundMode = mode; + try + { + throw new Exception("Test Exception"); + } + catch (Exception ex) + { + await Metrics.Error(this.GetType(), ex); + } } } diff --git a/Wabbajack/Settings.cs b/Wabbajack/Settings.cs index 963ce5ac..28ca821a 100644 --- a/Wabbajack/Settings.cs +++ b/Wabbajack/Settings.cs @@ -133,6 +133,17 @@ namespace Wabbajack private bool _favorPerfOverRam; public bool FavorPerfOverRam { get => _favorPerfOverRam; set => RaiseAndSetIfChanged(ref _favorPerfOverRam, value); } + + private bool _networkWorkaroundMode; + public bool NetworkWorkaroundMode + { + get => _networkWorkaroundMode; + set + { + Consts.UseNetworkWorkaroundMode = value; + RaiseAndSetIfChanged(ref _networkWorkaroundMode, value); + } + } public void SetProcessorSettings(ABatchProcessor processor) diff --git a/Wabbajack/Views/Settings/PerformanceSettingsView.xaml b/Wabbajack/Views/Settings/PerformanceSettingsView.xaml index 33ec8449..478f6b3e 100644 --- a/Wabbajack/Views/Settings/PerformanceSettingsView.xaml +++ b/Wabbajack/Views/Settings/PerformanceSettingsView.xaml @@ -27,6 +27,7 @@ + @@ -86,7 +87,18 @@ HorizontalAlignment="Right" VerticalAlignment="Center" Foreground="White" - > + > + + + diff --git a/Wabbajack/Views/Settings/PerformanceSettingsView.xaml.cs b/Wabbajack/Views/Settings/PerformanceSettingsView.xaml.cs index 7950b928..be145ff5 100644 --- a/Wabbajack/Views/Settings/PerformanceSettingsView.xaml.cs +++ b/Wabbajack/Views/Settings/PerformanceSettingsView.xaml.cs @@ -43,6 +43,8 @@ namespace Wabbajack .DisposeWith(disposable); this.BindStrict(this.ViewModel, x => x.FavorPerfOverRam, x => x.FavorPerfOverRam.IsChecked) .DisposeWith(disposable); + this.BindStrict(this.ViewModel, x => x.NetworkWorkaroundMode, x => x.UseNetworkWorkAround.IsChecked) + .DisposeWith(disposable); }); } } From bf0d098b4461aa5ac917b91c177a4ee434527465 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 17 May 2021 17:20:03 -0600 Subject: [PATCH 3/3] Use a pinned IP --- Wabbajack.Common/Consts.cs | 4 ++-- Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs | 2 +- Wabbajack.Lib/Http/Client.cs | 10 ++++++++-- Wabbajack.Lib/Http/ClientFactory.cs | 12 ++++++++++-- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 1c07b236..9c40ad32 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -92,7 +92,7 @@ namespace Wabbajack.Common public static string ModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/modlists.json"; public static string UtilityModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/utility_modlists.json"; public static string UnlistedModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/unlisted_modlists.json"; - public static string ModlistSummaryURL = "http://build.wabbajack.org/lists/status.json"; + public static string ModlistSummaryURL = "https://build.wabbajack.org/lists/status.json"; public static string UserAgent { get @@ -120,7 +120,7 @@ namespace Wabbajack.Common public static string WabbajackCacheHostname = "build.wabbajack.org"; // Direct IP to the CDN - public static string NetworkWorkaroundHost = "storage.wabbajack.org"; //"51.81.80.6"; + public static string NetworkWorkaroundHost = "51.81.80.6"; public static bool UseNetworkWorkaroundMode = false; public static Uri WabbajackBuildServerUri = new Uri("https://build.wabbajack.org"); diff --git a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs index 822c8ec6..1c86b514 100644 --- a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs +++ b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs @@ -83,7 +83,7 @@ namespace Wabbajack.Lib.Downloaders using var mmfile = MemoryMappedFile.CreateFromFile(fs, null, definition.Size, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, false); var client = new Wabbajack.Lib.Http.Client(); - if (!DomainRemaps.ContainsKey(Url.Host)) + if (!DomainRemaps.ContainsKey(Url.Host)) client.Headers.Add(("Host", Url.Host)); using var queue = new WorkQueue(); diff --git a/Wabbajack.Lib/Http/Client.cs b/Wabbajack.Lib/Http/Client.cs index 200ec85c..82f5dd93 100644 --- a/Wabbajack.Lib/Http/Client.cs +++ b/Wabbajack.Lib/Http/Client.cs @@ -76,8 +76,14 @@ namespace Wabbajack.Lib.Http public async Task SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = true, CancellationToken? token = null) { msg = FixupMessage(msg); - foreach (var (k, v) in Headers) - msg.Headers.Add(k, v); + foreach (var (k, v) in Headers) + { + if (k == "Host") + msg.Headers.Host = v; + else + msg.Headers.Add(k, v); + } + if (Cookies.Count > 0) Cookies.ForEach(c => ClientFactory.Cookies.Add(c)); int retries = 0; diff --git a/Wabbajack.Lib/Http/ClientFactory.cs b/Wabbajack.Lib/Http/ClientFactory.cs index 98ef1837..4f120efd 100644 --- a/Wabbajack.Lib/Http/ClientFactory.cs +++ b/Wabbajack.Lib/Http/ClientFactory.cs @@ -22,10 +22,18 @@ namespace Wabbajack.Lib.Http MaxConnectionsPerServer = 20, PooledConnectionLifetime = TimeSpan.FromMilliseconds(100), PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(100), - AutomaticDecompression = DecompressionMethods.All - + AutomaticDecompression = DecompressionMethods.All, + }; Utils.Log($"Configuring with SSL {_socketsHandler.SslOptions.EnabledSslProtocols}"); + + ServicePointManager.ServerCertificateValidationCallback += + (sender, certificate, chain, errors) => + { + if (Consts.UseNetworkWorkaroundMode) + return true; + return errors == SslPolicyErrors.None; + }; Client = new SysHttp.HttpClient(_socketsHandler); Client.DefaultRequestHeaders.Add("User-Agent", Consts.UserAgent); }