From ae8569bb38226ebf300f909f9735296dc40e27d0 Mon Sep 17 00:00:00 2001 From: LostDragonist Date: Thu, 13 Feb 2020 08:36:03 -0700 Subject: [PATCH 1/7] Fix checking the file association for needed updates Previously, this would fail to notice that the file association needed to be updated. This would lead to Windows trying to open non-existent copies of Wabbajack or to Wabbajack selecting "-i" as the chosen mod list instead of the file selected. --- Wabbajack.Common/ModListAssociationManager1.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Wabbajack.Common/ModListAssociationManager1.cs b/Wabbajack.Common/ModListAssociationManager1.cs index 69ddb6a8..361ae05e 100644 --- a/Wabbajack.Common/ModListAssociationManager1.cs +++ b/Wabbajack.Common/ModListAssociationManager1.cs @@ -32,7 +32,7 @@ namespace Wabbajack.Common var tempKey = progIDKey?.OpenSubKey("shell\\open\\command"); if (progIDKey == null || tempKey == null) return true; var value = tempKey.GetValue(""); - return value == null || value.ToString().Equals($"\"{appPath}\" -i=\"%1\""); + return value == null || !value.ToString().Equals($"\"{appPath}\" -i=\"%1\""); } public static bool IsAssociated() From b77db18bfaf5c2b5d907d50278084eee8c791423 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 14 Feb 2020 06:30:58 -0700 Subject: [PATCH 2/7] Fix a bunch of issues with compilation, indexing and VFS loading performance --- .../Controllers/IndexedFiles.cs | 3 ++ Wabbajack.BuildServer/Models/Jobs/IndexJob.cs | 3 ++ Wabbajack.Common/Consts.cs | 1 + Wabbajack.Lib/ACompiler.cs | 2 +- Wabbajack.Lib/FileUploader/AuthorAPI.cs | 12 ++++---- Wabbajack.Lib/MO2Compiler.cs | 25 +++++++++++++---- Wabbajack.Lib/NexusApi/NexusApi.cs | 28 +++++++++++++------ Wabbajack.VirtualFileSystem/Context.cs | 7 +++-- Wabbajack.VirtualFileSystem/VirtualFile.cs | 16 +++++++++++ 9 files changed, 74 insertions(+), 23 deletions(-) diff --git a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs index 2a3e8865..1c751add 100644 --- a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs @@ -112,6 +112,9 @@ namespace Wabbajack.BuildServer.Controllers Utils.Log("No valid INI parser for: \n" + iniString); continue; } + + if (data is ManualDownloader.State) + continue; var key = data.PrimaryKeyString; var found = await Db.DownloadStates.AsQueryable().Where(f => f.Key == key).Take(1).ToListAsync(); diff --git a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs index dd7c48e3..20039e5f 100644 --- a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs @@ -24,6 +24,9 @@ namespace Wabbajack.BuildServer.Models.Jobs public override bool UsesNexus { get => Archive.State is NexusDownloader.State; } public override async Task Execute(DBContext db, SqlService sql, AppSettings settings) { + if (Archive.State is ManualDownloader.State) + return JobResult.Success(); + var pk = new List(); pk.Add(AbstractDownloadState.TypeToName[Archive.State.GetType()]); pk.AddRange(Archive.State.PrimaryKey); diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 5367e73b..9db5019d 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -102,5 +102,6 @@ namespace Wabbajack.Common public const string MO2ModFolderName = "mods"; public static string PatchCacheFolder => Path.Combine(LocalAppDataPath, "patch_cache"); + public static int MaxConnectionsPerServer = 4; } } diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 49283eac..ded670f6 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -22,7 +22,7 @@ namespace Wabbajack.Lib public bool ReadmeIsWebsite; public string WabbajackVersion; - protected static string _vfsCacheName => Path.Combine(Consts.LocalAppDataPath, "vfs_compile_cache.bin"); + protected string _vfsCacheName => Path.Combine(Consts.LocalAppDataPath, $"vfs_compile_cache_{ModListName.StringSHA256Hex()}.bin"); /// /// A stream of tuples of ("Update Title", 0.25) which represent the name of the current task /// and the current progress. diff --git a/Wabbajack.Lib/FileUploader/AuthorAPI.cs b/Wabbajack.Lib/FileUploader/AuthorAPI.cs index ad2098cc..73905f52 100644 --- a/Wabbajack.Lib/FileUploader/AuthorAPI.cs +++ b/Wabbajack.Lib/FileUploader/AuthorAPI.cs @@ -121,7 +121,7 @@ namespace Wabbajack.Lib.FileUploader public static HttpClient GetAuthorizedClient() { - var handler = new HttpClientHandler {MaxConnectionsPerServer = MAX_CONNECTIONS}; + var handler = new HttpClientHandler {MaxConnectionsPerServer = Consts.MaxConnectionsPerServer}; var client = new HttpClient(handler); client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey()); return client; @@ -144,7 +144,7 @@ namespace Wabbajack.Lib.FileUploader return await RunJob("UpdateModLists"); } - public static async Task UploadPackagedInis(IEnumerable archives) + public static async Task UploadPackagedInis(WorkQueue queue, IEnumerable archives) { archives = archives.ToArray(); // defensive copy Utils.Log($"Packaging {archives.Count()} inis"); @@ -153,12 +153,12 @@ namespace Wabbajack.Lib.FileUploader await using var ms = new MemoryStream(); using (var z = new ZipArchive(ms, ZipArchiveMode.Create, true)) { - foreach (var archive in archives) + foreach (var e in archives) { - var state = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(archive.IniData)); - var entry = z.CreateEntry(Path.GetFileName(archive.Name)); + if (e.State == null) continue; + var entry = z.CreateEntry(Path.GetFileName(e.Name)); await using var os = entry.Open(); - await os.WriteAsync(Encoding.UTF8.GetBytes(string.Join("\n", state.GetMetaIni()))); + await os.WriteAsync(Encoding.UTF8.GetBytes(string.Join("\n", e.State.GetMetaIni()))); } } diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index f6725caa..36c1ee25 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -123,12 +123,13 @@ namespace Wabbajack.Lib .Where(p => p.FileExists()) .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath)))); } - + if (cancel.IsCancellationRequested) return false; UpdateTracker.NextStep("Cleaning output folder"); if (Directory.Exists(ModListOutputFolder)) Utils.DeleteDirectory(ModListOutputFolder); + /* if (cancel.IsCancellationRequested) return false; UpdateTracker.NextStep("Inferring metas for game file downloads"); await InferMetas(); @@ -137,9 +138,11 @@ namespace Wabbajack.Lib UpdateTracker.NextStep("Reindexing downloads after meta inferring"); await VFS.AddRoot(MO2DownloadsFolder); await VFS.WriteToFile(_vfsCacheName); + */ if (cancel.IsCancellationRequested) return false; UpdateTracker.NextStep("Pre-validating Archives"); + IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder) .Where(f => File.Exists(f + Consts.MetaFileExtension)) @@ -152,8 +155,7 @@ namespace Wabbajack.Lib }) .ToList(); - // Don't await this because we don't care if it fails. - var _ = AuthorAPI.UploadPackagedInis(IndexedArchives); + await CleanInvalidArchives(); @@ -261,6 +263,11 @@ namespace Wabbajack.Lib UpdateTracker.NextStep("Gathering Archives"); await GatherArchives(); + + // Don't await this because we don't care if it fails. + Utils.Log("Finding States to package"); + await AuthorAPI.UploadPackagedInis(Queue, SelectedArchives.ToArray()); + UpdateTracker.NextStep("Including Archive Metadata"); await IncludeArchiveMetadata(); UpdateTracker.NextStep("Building Patches"); @@ -348,8 +355,16 @@ namespace Wabbajack.Lib var response = await client.GetAsync( $"http://build.wabbajack.org/indexed_files/{vf.Hash.FromBase64().ToHex()}/meta.ini"); - - if (!response.IsSuccessStatusCode) return; + + if (!response.IsSuccessStatusCode) + { + File.WriteAllLines(vf.FullPath + Consts.MetaFileExtension, new [] + { + "[General]", + "unknownArchive=true" + }); + return; + } var ini_data = await response.Content.ReadAsStringAsync(); Utils.Log($"Inferred .meta for {Path.GetFileName(vf.FullPath)}, writing to disk"); diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index ca95f072..6dcf63b2 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -25,8 +25,8 @@ namespace Wabbajack.Lib.NexusApi private static string _additionalEntropy = "vtP2HF6ezg"; private static object _diskLock = new object(); - - public HttpClient HttpClient { get; } = new HttpClient(); + + public HttpClient HttpClient { get; } = new HttpClient(new HttpClientHandler {MaxConnectionsPerServer = Consts.MaxConnectionsPerServer}) {Timeout = TimeSpan.FromMinutes(1), DefaultRequestHeaders = {ConnectionClose = true}}; #region Authentication @@ -227,16 +227,17 @@ namespace Wabbajack.Lib.NexusApi TOP: try { - var response = await HttpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); + using var response = await HttpClient.GetAsync(url, HttpCompletionOption.ResponseContentRead); UpdateRemaining(response); if (!response.IsSuccessStatusCode) - throw new HttpRequestException($"{response.StatusCode} - {response.ReasonPhrase}"); - - - using (var stream = await response.Content.ReadAsStreamAsync()) { - return stream.FromJSON(); + Utils.Log($"Nexus call failed: {response.RequestMessage}"); + throw new HttpRequestException($"{response.StatusCode} - {response.ReasonPhrase}"); } + + + await using var stream = await response.Content.ReadAsStreamAsync(); + return stream.FromJSON(); } catch (TimeoutException) { @@ -246,13 +247,18 @@ namespace Wabbajack.Lib.NexusApi retries++; goto TOP; } + catch (Exception e) + { + Utils.Log(e.ToString()); + throw; + } } private async Task GetCached(string url) { try { - var builder = new UriBuilder(url) { Host = Consts.WabbajackCacheHostname, Port = Consts.WabbajackCachePort, Scheme = "http" }; + var builder = new UriBuilder(url) { Host = Consts.WabbajackCacheHostname, Scheme = "https" }; return await Get(builder.ToString()); } catch (Exception) @@ -273,6 +279,10 @@ namespace Wabbajack.Lib.NexusApi } catch (HttpRequestException) { + if (await IsPremium()) + { + throw; + } } try diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index 8164ecaa..df0eb818 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -24,7 +24,7 @@ namespace Wabbajack.VirtualFileSystem Utils.Log("Cleaning VFS, this may take a bit of time"); Utils.DeleteDirectory(_stagingFolder); } - public const ulong FileVersion = 0x02; + public const ulong FileVersion = 0x03; public const string Magic = "WABBAJACK VFS FILE"; private static readonly string _stagingFolder = "vfs_staging"; @@ -420,13 +420,16 @@ namespace Wabbajack.VirtualFileSystem public TemporaryDirectory(string name) { FullName = name; + if (!Directory.Exists(FullName)) + Directory.CreateDirectory(FullName); } public string FullName { get; } public void Dispose() { - Utils.DeleteDirectory(FullName); + if (Directory.Exists(FullName)) + Utils.DeleteDirectory(FullName); } } } diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 29f4c06e..db0239ea 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -112,6 +112,20 @@ namespace Wabbajack.VirtualFileSystem return _thisAndAllChildren; } } + + + public T ThisAndAllChildrenReduced(T acc, Func fn) + { + acc = fn(acc, this); + return this.Children.Aggregate(acc, (current, itm) => itm.ThisAndAllChildrenReduced(current, fn)); + } + + public void ThisAndAllChildrenReduced(Action fn) + { + fn(this); + foreach (var itm in Children) + itm.ThisAndAllChildrenReduced(fn); + } /// @@ -233,6 +247,7 @@ namespace Wabbajack.VirtualFileSystem private void Write(BinaryWriter bw) { bw.Write(Name); + bw.Write(FullPath); bw.Write(Hash); bw.Write(Size); bw.Write(LastModified); @@ -258,6 +273,7 @@ namespace Wabbajack.VirtualFileSystem Context = context, Parent = parent, Name = br.ReadString(), + _fullPath = br.ReadString(), Hash = br.ReadString(), Size = br.ReadInt64(), LastModified = br.ReadInt64(), From f53e05d741c77e3c4f2c098f6d591b0a2b8bd050 Mon Sep 17 00:00:00 2001 From: erri120 Date: Fri, 14 Feb 2020 19:10:04 +0100 Subject: [PATCH 3/7] Log files will now be stored in a separate logs folder --- Wabbajack.Common/Utils.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 45dd77ad..e0b26ae9 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -53,15 +53,18 @@ namespace Wabbajack.Common if (!Directory.Exists(Consts.LocalAppDataPath)) Directory.CreateDirectory(Consts.LocalAppDataPath); + if (!Directory.Exists("logs")) + Directory.CreateDirectory("logs"); + var programName = Assembly.GetEntryAssembly()?.Location ?? "Wabbajack"; - LogFile = Path.GetFileNameWithoutExtension(programName) + ".current.log"; + LogFile = Path.Combine("logs", Path.GetFileNameWithoutExtension(programName) + ".current.log"); _startTime = DateTime.Now; if (LogFile.FileExists()) { - var new_path = Path.GetFileNameWithoutExtension(programName) + (new FileInfo(LogFile)).LastWriteTime.ToString(" yyyy-MM-dd HH_mm_ss") + ".log"; - File.Move(LogFile, new_path, MoveOptions.ReplaceExisting); + var newPath = Path.Combine("logs", Path.GetFileNameWithoutExtension(programName) + (new FileInfo(LogFile)).LastWriteTime.ToString(" yyyy-MM-dd HH_mm_ss") + ".log"); + File.Move(LogFile, newPath, MoveOptions.ReplaceExisting); } var watcher = new FileSystemWatcher(Consts.LocalAppDataPath); From 8d13a4c455457a62d0c14084c5662f1c0ecb8d0b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 14 Feb 2020 15:23:27 -0700 Subject: [PATCH 4/7] Swap HttpClient over to SocketsHttpHandler --- Compression.BSA/Compression.BSA.csproj | 2 +- Wabbajack.Common/Http/Client.cs | 62 +++++++++++++++++++ Wabbajack.Common/Http/ClientFactory.cs | 25 ++++++++ Wabbajack.Common/Wabbajack.Common.csproj | 3 +- .../AbstractNeedsLoginDownloader.cs | 4 +- .../Downloaders/GoogleDriveDownloader.cs | 2 +- Wabbajack.Lib/Downloaders/HTTPDownloader.cs | 8 +-- .../Downloaders/MediaFireDownloader.cs | 2 +- Wabbajack.Lib/LibCefHelpers/Init.cs | 6 +- Wabbajack.Lib/NexusApi/NexusApi.cs | 32 +++------- .../StatusMessages/ManuallyDownloadFile.cs | 6 +- Wabbajack.Lib/Wabbajack.Lib.csproj | 2 +- .../Wabbajack.VirtualFileSystem.csproj | 2 +- Wabbajack.sln | 16 ++--- 14 files changed, 124 insertions(+), 48 deletions(-) create mode 100644 Wabbajack.Common/Http/Client.cs create mode 100644 Wabbajack.Common/Http/ClientFactory.cs diff --git a/Compression.BSA/Compression.BSA.csproj b/Compression.BSA/Compression.BSA.csproj index 1e860870..1836fee8 100644 --- a/Compression.BSA/Compression.BSA.csproj +++ b/Compression.BSA/Compression.BSA.csproj @@ -1,7 +1,7 @@  - netstandard2.1 + netcoreapp3.1 true x64 win10-x64 diff --git a/Wabbajack.Common/Http/Client.cs b/Wabbajack.Common/Http/Client.cs new file mode 100644 index 00000000..19f678dd --- /dev/null +++ b/Wabbajack.Common/Http/Client.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; + +namespace Wabbajack.Common.Http +{ + public class Client + { + public List<(string, string)> Headers => new List<(string, string)>(); + public List Cookies = new List(); + public async Task GetAsync(string url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseContentRead) + { + var request = new HttpRequestMessage(HttpMethod.Get, url); + foreach (var (k, v) in Headers) + request.Headers.Add(k, v); + return await SendAsync(request, responseHeadersRead); + } + + public async Task GetStringAsync(string url) + { + var request = new HttpRequestMessage(HttpMethod.Get, url); + foreach (var (k, v) in Headers) + request.Headers.Add(k, v); + if (Cookies.Count > 0) + Cookies.ForEach(c => ClientFactory.Cookies.Add(c)); + + return await SendStringAsync(request); + } + + private async Task SendStringAsync(HttpRequestMessage request) + { + var result = await SendAsync(request); + return await result.Content.ReadAsStringAsync(); + } + + + public async Task SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseContentRead) + { + int retries = 0; + TOP: + try + { + var response = await ClientFactory.Client.SendAsync(msg, responseHeadersRead); + return response; + } + catch (Exception) + { + if (retries > Consts.MaxHTTPRetries) throw; + + retries++; + Utils.Log($"Http Connect error to {msg.RequestUri} retry {retries}"); + await Task.Delay(100 * retries); + goto TOP; + + } + + } + } +} diff --git a/Wabbajack.Common/Http/ClientFactory.cs b/Wabbajack.Common/Http/ClientFactory.cs new file mode 100644 index 00000000..6e2aa00a --- /dev/null +++ b/Wabbajack.Common/Http/ClientFactory.cs @@ -0,0 +1,25 @@ +using System; +using System.Net; +using SysHttp = System.Net.Http; +namespace Wabbajack.Common.Http +{ + public static class ClientFactory + { + private static SysHttp.SocketsHttpHandler _socketsHandler { get; } + internal static SysHttp.HttpClient Client { get; } + internal static CookieContainer Cookies { get; } + + static ClientFactory() + { + Cookies = new CookieContainer(); + _socketsHandler = new SysHttp.SocketsHttpHandler + { + CookieContainer = Cookies, + MaxConnectionsPerServer = 8, + PooledConnectionLifetime = TimeSpan.FromSeconds(2), + PooledConnectionIdleTimeout = TimeSpan.FromSeconds(2) + }; + Client = new SysHttp.HttpClient(_socketsHandler); + } + } +} diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 8498ea3a..45633134 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -1,7 +1,7 @@  - netstandard2.1 + netcoreapp3.1 x64 win10-x64 @@ -39,6 +39,7 @@ + diff --git a/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs b/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs index 0ac94906..601b653f 100644 --- a/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs @@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders private readonly string _encryptedKeyName; private readonly string _cookieDomain; private readonly string _cookieName; - internal HttpClient AuthedClient; + internal Common.Http.Client AuthedClient; /// /// Sets up all the login facilites needed for a INeedsLogin downloader based on having the user log @@ -81,7 +81,7 @@ namespace Wabbajack.Lib.Downloaders return cookies; } - public async Task GetAuthedClient() + public async Task GetAuthedClient() { Helpers.Cookie[] cookies; try diff --git a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs index 06afd874..bf8086e3 100644 --- a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs @@ -53,7 +53,7 @@ namespace Wabbajack.Lib.Downloaders private async Task ToHttpState() { var initialURL = $"https://drive.google.com/uc?id={Id}&export=download"; - var client = new HttpClient(); + var client = new Common.Http.Client(); var response = await client.GetAsync(initialURL); if (!response.IsSuccessStatusCode) throw new HttpException((int)response.StatusCode, response.ReasonPhrase); diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index 860a6848..63e92e0c 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -61,7 +61,7 @@ namespace Wabbajack.Lib.Downloaders public List Headers { get; set; } [Exclude] - public HttpClient Client { get; set; } + public Common.Http.Client Client { get; set; } public override object[] PrimaryKey { get => new object[] {Url};} @@ -86,8 +86,8 @@ namespace Wabbajack.Lib.Downloaders using (var fs = download ? File.Open(destination, FileMode.Create) : null) { - var client = Client ?? new HttpClient(); - client.DefaultRequestHeaders.Add("User-Agent", Consts.UserAgent); + var client = Client ?? new Common.Http.Client(); + client.Headers.Add(("User-Agent", Consts.UserAgent)); if (Headers != null) foreach (var header in Headers) @@ -95,7 +95,7 @@ namespace Wabbajack.Lib.Downloaders var idx = header.IndexOf(':'); var k = header.Substring(0, idx); var v = header.Substring(idx + 1); - client.DefaultRequestHeaders.Add(k, v); + client.Headers.Add((k, v)); } long totalRead = 0; diff --git a/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs b/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs index 65b341d8..08d3f898 100644 --- a/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs +++ b/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs @@ -53,7 +53,7 @@ namespace Wabbajack.Lib.Downloaders if (newURL == null || !newURL.StartsWith("http")) return null; return new HTTPDownloader.State() { - Client = new HttpClient(), + Client = new Common.Http.Client(), Url = newURL }; } diff --git a/Wabbajack.Lib/LibCefHelpers/Init.cs b/Wabbajack.Lib/LibCefHelpers/Init.cs index f411d74a..e3b0729a 100644 --- a/Wabbajack.Lib/LibCefHelpers/Init.cs +++ b/Wabbajack.Lib/LibCefHelpers/Init.cs @@ -16,13 +16,15 @@ namespace Wabbajack.Lib.LibCefHelpers { public static class Helpers { - public static HttpClient GetClient(IEnumerable cookies, string referer) + public static Common.Http.Client GetClient(IEnumerable cookies, string referer) { + throw new NotImplementedException("tt"); + /* var container = ToCookieContainer(cookies); var handler = new HttpClientHandler { CookieContainer = container }; var client = new HttpClient(handler); client.DefaultRequestHeaders.Referrer = new Uri(referer); - return client; + return client;*/ } private static CookieContainer ToCookieContainer(IEnumerable cookies) diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index 6dcf63b2..6357b36e 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -23,10 +23,8 @@ namespace Wabbajack.Lib.NexusApi { private static readonly string API_KEY_CACHE_FILE = "nexus.key_cache"; private static string _additionalEntropy = "vtP2HF6ezg"; - - private static object _diskLock = new object(); - - public HttpClient HttpClient { get; } = new HttpClient(new HttpClientHandler {MaxConnectionsPerServer = Consts.MaxConnectionsPerServer}) {Timeout = TimeSpan.FromMinutes(1), DefaultRequestHeaders = {ConnectionClose = true}}; + + public Common.Http.Client HttpClient { get; } = new Common.Http.Client(); #region Authentication @@ -202,14 +200,13 @@ namespace Wabbajack.Lib.NexusApi { ApiKey = apiKey; - HttpClient.BaseAddress = new Uri("https://api.nexusmods.com"); // set default headers for all requests to the Nexus API - var headers = HttpClient.DefaultRequestHeaders; - headers.Add("User-Agent", Consts.UserAgent); - headers.Add("apikey", ApiKey); - headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - headers.Add("Application-Name", Consts.AppName); - headers.Add("Application-Version", $"{Assembly.GetEntryAssembly()?.GetName()?.Version ?? new Version(0, 1)}"); + var headers = HttpClient.Headers; + headers.Add(("User-Agent", Consts.UserAgent)); + headers.Add(("apikey", ApiKey)); + headers.Add(("Accept", "application/json")); + headers.Add(("Application-Name", Consts.AppName)); + headers.Add(("Application-Version", $"{Assembly.GetEntryAssembly()?.GetName()?.Version ?? new Version(0, 1)}")); if (!Directory.Exists(Consts.NexusCacheDirectory)) Directory.CreateDirectory(Consts.NexusCacheDirectory); @@ -329,19 +326,6 @@ namespace Wabbajack.Lib.NexusApi return await GetCached(url); } - public async Task EndorseMod(NexusDownloader.State mod) - { - Utils.Status($"Endorsing ${mod.GameName} - ${mod.ModID}"); - var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(mod.GameName)}/mods/{mod.ModID}/endorse.json"; - - var content = new FormUrlEncodedContent(new Dictionary { { "version", mod.Version } }); - - using (var stream = await HttpClient.PostStream(url, content)) - { - return stream.FromJSON(); - } - } - private class DownloadLink { public string URI { get; set; } diff --git a/Wabbajack.Lib/StatusMessages/ManuallyDownloadFile.cs b/Wabbajack.Lib/StatusMessages/ManuallyDownloadFile.cs index 0bef7492..a782e089 100644 --- a/Wabbajack.Lib/StatusMessages/ManuallyDownloadFile.cs +++ b/Wabbajack.Lib/StatusMessages/ManuallyDownloadFile.cs @@ -12,8 +12,8 @@ namespace Wabbajack.Lib public override string ShortDescription { get; } public override string ExtendedDescription { get; } - private TaskCompletionSource<(Uri, HttpClient)> _tcs = new TaskCompletionSource<(Uri, HttpClient)>(); - public Task<(Uri, HttpClient)> Task => _tcs.Task; + private TaskCompletionSource<(Uri, Common.Http.Client)> _tcs = new TaskCompletionSource<(Uri, Common.Http.Client)>(); + public Task<(Uri, Common.Http.Client)> Task => _tcs.Task; private ManuallyDownloadFile(ManualDownloader.State state) { @@ -30,7 +30,7 @@ namespace Wabbajack.Lib _tcs.SetCanceled(); } - public void Resume(Uri s, HttpClient client) + public void Resume(Uri s, Common.Http.Client client) { _tcs.SetResult((s, client)); } diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 06e5a53a..62b329b5 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -1,7 +1,7 @@  - netstandard2.1 + netcoreapp3.1 x64 win10-x64 diff --git a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj index 47e68e3f..a2ec35cc 100644 --- a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj +++ b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj @@ -1,7 +1,7 @@  - netstandard2.1 + netcoreapp3.1 x64 win10-x64 diff --git a/Wabbajack.sln b/Wabbajack.sln index d0b9081c..e4766ed6 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -38,7 +38,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Test", "Wabbajack EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.CLI", "Wabbajack.CLI\Wabbajack.CLI.csproj", "{685D8BB1-D178-4D2C-85C7-C54A36FB7454}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Launcher", "Wabbajack.Launcher\Wabbajack.Launcher.csproj", "{D6856DBF-C959-4867-A8A8-343DA2D2715E}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Launcher", "Wabbajack.Launcher\Wabbajack.Launcher.csproj", "{D6856DBF-C959-4867-A8A8-343DA2D2715E}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -84,12 +84,14 @@ Global {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|Any CPU.ActiveCfg = Release|x64 {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.ActiveCfg = Release|x64 {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.Build.0 = Release|x64 - {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|Any CPU.ActiveCfg = Debug|x64 - {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.ActiveCfg = Debug|x64 - {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.Build.0 = Debug|x64 - {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.ActiveCfg = Release|x64 - {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.ActiveCfg = Release|x64 - {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.Build.0 = Release|x64 + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.ActiveCfg = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.Build.0 = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.Build.0 = Release|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.ActiveCfg = Release|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.Build.0 = Release|Any CPU {6ED08CFB-B879-4B55-8741-663A4A3491CE}.Debug|Any CPU.ActiveCfg = Debug|x64 {6ED08CFB-B879-4B55-8741-663A4A3491CE}.Debug|x64.ActiveCfg = Debug|x64 {6ED08CFB-B879-4B55-8741-663A4A3491CE}.Debug|x64.Build.0 = Debug|x64 From f9459c2d853d1f90ff3059db26c3c756570c639f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 14 Feb 2020 15:27:48 -0700 Subject: [PATCH 5/7] Fix disabled test --- Wabbajack.Lib/LibCefHelpers/Init.cs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Wabbajack.Lib/LibCefHelpers/Init.cs b/Wabbajack.Lib/LibCefHelpers/Init.cs index e3b0729a..1f7d8a95 100644 --- a/Wabbajack.Lib/LibCefHelpers/Init.cs +++ b/Wabbajack.Lib/LibCefHelpers/Init.cs @@ -18,13 +18,10 @@ namespace Wabbajack.Lib.LibCefHelpers { public static Common.Http.Client GetClient(IEnumerable cookies, string referer) { - throw new NotImplementedException("tt"); - /* - var container = ToCookieContainer(cookies); - var handler = new HttpClientHandler { CookieContainer = container }; - var client = new HttpClient(handler); - client.DefaultRequestHeaders.Referrer = new Uri(referer); - return client;*/ + var client = new Common.Http.Client(); + client.Headers.Add(("Referrer", referer)); + client.Cookies.AddRange(cookies.Select(cookie => new System.Net.Cookie(cookie.Name, cookie.Value, cookie.Path, cookie.Domain))); + return client; } private static CookieContainer ToCookieContainer(IEnumerable cookies) From 6b4abb8f4098de9d03b75dbfb392ce06741aca72 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 14 Feb 2020 15:50:14 -0700 Subject: [PATCH 6/7] Fix bad header info --- Wabbajack.Common/Http/Client.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Wabbajack.Common/Http/Client.cs b/Wabbajack.Common/Http/Client.cs index 19f678dd..d6393820 100644 --- a/Wabbajack.Common/Http/Client.cs +++ b/Wabbajack.Common/Http/Client.cs @@ -9,7 +9,7 @@ namespace Wabbajack.Common.Http { public class Client { - public List<(string, string)> Headers => new List<(string, string)>(); + public List<(string, string)> Headers = new List<(string, string)>(); public List Cookies = new List(); public async Task GetAsync(string url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseContentRead) { From 7d69893901c676ee22dc18afadc151dacc4e59e8 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 14 Feb 2020 15:58:51 -0700 Subject: [PATCH 7/7] Null string fix in VFS saving --- Wabbajack.Lib/ACompiler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index ded670f6..08b34c62 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -22,7 +22,7 @@ namespace Wabbajack.Lib public bool ReadmeIsWebsite; public string WabbajackVersion; - protected string _vfsCacheName => Path.Combine(Consts.LocalAppDataPath, $"vfs_compile_cache_{ModListName.StringSHA256Hex()}.bin"); + protected string _vfsCacheName => Path.Combine(Consts.LocalAppDataPath, $"vfs_compile_cache_{ModListName?.StringSHA256Hex() ?? "_Unknown_"}.bin"); /// /// A stream of tuples of ("Update Title", 0.25) which represent the name of the current task /// and the current progress.