From 161d8114164463491125ffb41e3c6af14dde8645 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 29 Aug 2019 22:24:31 -0600 Subject: [PATCH] fixes #21 also add log info when parsing modlist --- CHANGELOG.md | 13 ++++++ Wabbajack/Compiler.cs | 17 ++++---- Wabbajack/Installer.cs | 91 ++++++++++++++++++++++++------------------ Wabbajack/NexusAPI.cs | 2 +- 4 files changed, 73 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bafd5fe..a109b235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ ### Changelog +#### Version 0.9 - ???? +* Added log information for when modlists start parsing during installation +* Check all links during mod list creation + +#### Version 0.8.1 - 8/29/2019 +* Fixed a bug that was causing VFS temp folders not to be cleaned +* 7zip Extraction code now shows a progress bar +* Told 7zip not to ask for permission before overwriting a file (should fix the hanging installer problem) +* Fixed several places where we were using long-path incompatible file routines +* Changed the work queue from FIFO to LIFO which results in depth-first work instead of breadth-first +TLDR: We now fully analyze a single archive before moving on to the next. + + #### Version 0.8 - 8/26/2019 * Mod folders that contain ESMs with names matching the Skyrim core ESMs are assumed to be cleaned versions of the core game mods, and will be patched from the ESMs found in the game folder. **Note:** if you have also cleaned the files in the Skyrim diff --git a/Wabbajack/Compiler.cs b/Wabbajack/Compiler.cs index 50cc8fda..e2e148c2 100644 --- a/Wabbajack/Compiler.cs +++ b/Wabbajack/Compiler.cs @@ -443,16 +443,6 @@ namespace Wabbajack ModID = general.modID, Version = general.version ?? "0.0.0.0" }; - Status($"Getting Nexus info for {found.Name}"); - try - { - var link = NexusAPI.GetNexusDownloadLink((NexusMod)result, NexusKey, true); - } - catch (Exception ex) - { - Error($"Unable to resolve {found.Name} on the Nexus was the file removed?"); - } - } else { @@ -464,6 +454,13 @@ namespace Wabbajack result.Hash = found.File.Hash; result.Meta = found.Meta; + Info($"Checking link for {found.Name}"); + + var installer = new Installer(null, "", s=>Utils.Log(s)); + installer.NexusAPIKey = NexusKey; + if (!installer.DownloadArchive(result, false)) + Error($"Unable to resolve link for {found.Name}. If this is hosted on the nexus the file may have been removed."); + return result; } Error("No match found for Archive sha: {0} this shouldn't happen", sha); diff --git a/Wabbajack/Installer.cs b/Wabbajack/Installer.cs index 8cf7c25a..984f78b8 100644 --- a/Wabbajack/Installer.cs +++ b/Wabbajack/Installer.cs @@ -45,7 +45,7 @@ namespace Wabbajack public Action Log_Fn { get; } public Dictionary HashedArchives { get; private set; } - public string NexusAPIKey { get; private set; } + public string NexusAPIKey { get; set; } public bool IgnoreMissingFiles { get; internal set; } public string GameFolder { get; private set; } @@ -413,52 +413,62 @@ namespace Wabbajack return; } - private void DownloadMissingArchives(List missing) + private void DownloadMissingArchives(List missing, bool download=true) { missing.PMap(archive => { Info($"Downloading {archive.Name}"); var output_path = Path.Combine(DownloadFolder, archive.Name); - if (output_path.FileExists()) - File.Delete(output_path); + if (download) + if (output_path.FileExists()) + File.Delete(output_path); + return DownloadArchive(archive, download); + }); + } - switch (archive) { + public bool DownloadArchive(Archive archive, bool download) + { + try + { + switch (archive) + { case NexusMod a: - Info($"Downloading Nexus Archive - {archive.Name} - {a.GameName} - {a.ModID} - {a.FileID}"); string url; try { - url = NexusAPI.GetNexusDownloadLink(a as NexusMod, NexusAPIKey); + url = NexusAPI.GetNexusDownloadLink(a as NexusMod, NexusAPIKey, !download); + if (!download) return true; } catch (Exception ex) { Info($"{a.Name} - Error Getting Nexus Download URL - {ex.Message}"); - return; + return false; } + Info($"Downloading Nexus Archive - {archive.Name} - {a.GameName} - {a.ModID} - {a.FileID}"); DownloadURLDirect(archive, url); - break; + return true; case MEGAArchive a: - DownloadMegaArchive(a); - break; + return DownloadMegaArchive(a, download); case GoogleDriveMod a: - DownloadGoogleDriveArchive(a); - break; + return DownloadGoogleDriveArchive(a, download); case MODDBArchive a: - DownloadModDBArchive(archive, (archive as MODDBArchive).URL); - break; + return DownloadModDBArchive(archive, (archive as MODDBArchive).URL, download); case MediaFireArchive a: - DownloadMediaFireArchive(archive, a.URL); - break; + return false; + //return DownloadMediaFireArchive(archive, a.URL, download); case DirectURLArchive a: - DownloadURLDirect(archive, a.URL, headers:a.Headers); - break; - default: - break; - + return DownloadURLDirect(archive, a.URL, headers: a.Headers, download: download); + } } - }); + catch (Exception ex) + { + Utils.Log($"Download error for file {archive.Name}"); + Utils.Log(ex.ToString()); + return false; + } + return false; } private void DownloadMediaFireArchive(Archive a, string url) @@ -470,39 +480,41 @@ namespace Wabbajack DownloadURLDirect(a, confirm.ToString(), client); } - private void DownloadMegaArchive(MEGAArchive m) + private bool DownloadMegaArchive(MEGAArchive m, bool download) { var client = new MegaApiClient(); Status("Logging into MEGA (as anonymous)"); client.LoginAnonymous(); var file_link = new Uri(m.URL); var node = client.GetNodeFromLink(file_link); + if (!download) return true; Status("Downloading MEGA file: {0}", m.Name); var output_path = Path.Combine(DownloadFolder, m.Name); client.DownloadFile(file_link, output_path); + return true; } - private void DownloadGoogleDriveArchive(GoogleDriveMod a) + private bool DownloadGoogleDriveArchive(GoogleDriveMod a, bool download) { var initial_url = $"https://drive.google.com/uc?id={a.Id}&export=download"; var client = new HttpClient(); var result = client.GetStringSync(initial_url); var regex = new Regex("(?<=/uc\\?export=download&confirm=).*(?=;id=)"); var confirm = regex.Match(result); - DownloadURLDirect(a, $"https://drive.google.com/uc?export=download&confirm={confirm}&id={a.Id}", client); + return DownloadURLDirect(a, $"https://drive.google.com/uc?export=download&confirm={confirm}&id={a.Id}", client, download: download); } - private void DownloadModDBArchive(Archive archive, string url) + private bool DownloadModDBArchive(Archive archive, string url, bool download) { var client = new HttpClient(); var result = client.GetStringSync(url); var regex = new Regex("https:\\/\\/www\\.moddb\\.com\\/downloads\\/mirror\\/.*(?=\\\")"); var match = regex.Match(result); - DownloadURLDirect(archive, match.Value); + return DownloadURLDirect(archive, match.Value, download: download); } - private void DownloadURLDirect(Archive archive, string url, HttpClient client = null, List headers = null) + private bool DownloadURLDirect(Archive archive, string url, HttpClient client = null, bool download = true, List headers = null) { try { @@ -533,14 +545,16 @@ namespace Wabbajack } catch (Exception ex) { - }; if (stream.IsFaulted) { Info($"While downloading {url} - {Utils.ExceptionToString(stream.Exception)}"); - return; + return false; } + if (!download) + return true; + string header_var = "1"; if (response.Content.Headers.Contains("Content-Length")) header_var = response.Content.Headers.GetValues("Content-Length").FirstOrDefault(); @@ -567,25 +581,21 @@ namespace Wabbajack } Status("Hashing {0}", archive.Name); HashArchive(output_path); + return true; } catch (Exception ex) { Info($"{archive.Name} - Error downloading from: {url}"); + return false; } } - private object GetNexusAPIKey() - { - throw new NotImplementedException(); - } - private void HashArchives() { HashedArchives = Directory.EnumerateFiles(DownloadFolder) .Where(e => Consts.SupportedArchives.Contains(Path.GetExtension(e))) .PMap(e => (HashArchive(e), e)) .ToDictionary(e => e.Item1, e => e.Item2); - } private string HashArchive(string e) @@ -597,11 +607,11 @@ namespace Wabbajack Status("Hashing {0}", Path.GetFileName(e)); File.WriteAllText(cache, Utils.FileSHA256(e)); return HashArchive(e); - } public static string CheckForModPack() { + Utils.Log("Looking for attached modlist"); using (var s = File.OpenRead(Assembly.GetExecutingAssembly().Location)) { var magic_bytes = Encoding.ASCII.GetBytes(Consts.ModPackMagic); @@ -621,7 +631,10 @@ namespace Wabbajack s.Position = start_pos; long length = br.ReadInt64(); - return br.ReadBytes((int)length).BZip2String(); + Utils.Log("Modlist found, loading..."); + var list = br.ReadBytes((int)length).BZip2String(); + Utils.Log("Modlist loaded."); + return list; } } diff --git a/Wabbajack/NexusAPI.cs b/Wabbajack/NexusAPI.cs index 84368aa6..916df916 100644 --- a/Wabbajack/NexusAPI.cs +++ b/Wabbajack/NexusAPI.cs @@ -65,7 +65,7 @@ namespace Wabbajack return _baseHttpClient; } - public static string GetNexusDownloadLink(NexusMod archive, string apikey, bool cache=true) + public static string GetNexusDownloadLink(NexusMod archive, string apikey, bool cache=false) { if (cache && TryGetCachedLink(archive, apikey, out string result)) return result;