From efd61c4b39dedde70ddb190c69f48219a27e35c8 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 22 Jul 2019 22:17:52 -0600 Subject: [PATCH] Can install files from archives in modpacks --- Wabbajack/Installer.cs | 266 +++++++++++++++++++++++++++++++++++++++++ Wabbajack/NexusAPI.cs | 88 ++++++++++++++ 2 files changed, 354 insertions(+) create mode 100644 Wabbajack/Installer.cs create mode 100644 Wabbajack/NexusAPI.cs diff --git a/Wabbajack/Installer.cs b/Wabbajack/Installer.cs new file mode 100644 index 00000000..f3c60d8c --- /dev/null +++ b/Wabbajack/Installer.cs @@ -0,0 +1,266 @@ +using SevenZipExtractor; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack +{ + public class Installer + { + public Installer(ModList mod_list, string output_folder, Action log_fn) + { + Outputfolder = output_folder; + ModList = mod_list; + Log_Fn = log_fn; + } + + public string Outputfolder { get; } + public string DownloadFolder + { + get + { + return Path.Combine(Outputfolder, "downloads"); + } + } + public ModList ModList { get; } + public Action Log_Fn { get; } + public Dictionary HashedArchives { get; private set; } + public string NexusAPIKey { get; private set; } + + public void Info(string msg, params object[] args) + { + if (args.Length > 0) + msg = String.Format(msg, args); + Log_Fn(msg); + } + + public void Status(string msg, params object[] args) + { + if (args.Length > 0) + msg = String.Format(msg, args); + WorkQueue.Report(msg, 0); + } + + public void Status(int progress, string msg, params object[] args) + { + if (args.Length > 0) + msg = String.Format(msg, args); + WorkQueue.Report(msg, progress); + } + private void Error(string msg, params object[] args) + { + if (args.Length > 0) + msg = String.Format(msg, args); + Log_Fn(msg); + throw new Exception(msg); + } + + public void Install() + { + Directory.CreateDirectory(Outputfolder); + Directory.CreateDirectory(DownloadFolder); + + HashArchives(); + DownloadArchives(); + HashArchives(); + + var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList(); + if (missing.Count > 0) + { + foreach (var a in missing) + Info("Unable to download {0}", a.Name); + Error("Cannot continue, was unable to download one or more archives"); + } + BuildFolderStructure(); + InstallArchives(); + + Info("Installation complete! You may exit the program."); + } + + private void BuildFolderStructure() + { + Info("Building Folder Structure"); + ModList.Directives + .Select(d => Path.Combine(Outputfolder, Path.GetDirectoryName(d.To))) + .ToHashSet() + .Do(f => { + if (Directory.Exists(f)) return; + Directory.CreateDirectory(f); + }); + } + + private void InstallArchives() + { + Info("Installing Archives"); + var grouped = ModList.Directives + .OfType() + .GroupBy(e => e.ArchiveHash) + .ToDictionary(k => k.Key); + var archives = ModList.Archives + .Select(a => new { Archive = a, AbsolutePath = HashedArchives[a.Hash] }) + .ToList(); + + archives.PMap(a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash])); + + } + + private void InstallArchive(Archive archive, string absolutePath, IGrouping grouping) + { + Status("Extracting {0}", archive.Name); + var files = grouping.GroupBy(e => e.From) + .ToDictionary(e => e.Key); + + using (var a = new ArchiveFile(absolutePath)) + { + a.Extract(entry => + { + if (files.TryGetValue(entry.FileName, out var directives)) + { + var directive = directives.First(); + var absolute = Path.Combine(Outputfolder, directive.To); + if (absolute.FileExists()) File.Delete(absolute); + return File.OpenWrite(absolute); + } + return null; + }); + } + + Status("Copying duplicated files for {0}", archive.Name); + + foreach (var dups in files.Where(e => e.Value.Count() > 1).Select(v => v.Value)) + { + var ffrom = dups.First(); + var from_path = Path.Combine(Outputfolder, ffrom.To); + foreach (var to in dups.Skip(1)) + { + var to_path = Path.Combine(Outputfolder, to.To); + if (to_path.FileExists()) File.Delete(to_path); + File.Copy(from_path, to_path); + } + }; + + // Now patch all the files from this archive + foreach (var to_patch in grouping.OfType()) + { + using (var patch_stream = new MemoryStream()) + { + Status("Patching {0}", Path.GetFileName(to_patch.To)); + // Read in the patch data + + var patch_data = to_patch.Patch.FromBase64(); + + var to_file = Path.Combine(Outputfolder, to_patch.To); + MemoryStream old_data = new MemoryStream(File.ReadAllBytes(to_file)); + + // Patch it + using (var out_stream = File.OpenWrite(to_file)) + { + BSDiff.Apply(old_data, () => new MemoryStream(patch_data), out_stream); + } + } + + } + } + + private void DownloadArchives() + { + var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList(); + Info("Missing {0} archives", missing.Count); + + Info("Getting Nexus API Key, if a browser appears, please accept"); + NexusAPIKey = NexusAPI.GetNexusAPIKey(); + + DownloadMissingArchives(missing); + } + + private void DownloadMissingArchives(List missing) + { + missing.PMap(archive => + { + if (archive is NexusMod) + { + var url = NexusAPI.GetNexusDownloadLink(archive as NexusMod, NexusAPIKey); + DownloadURLDirect(archive, url); + } + else if (archive is DirectURLArchive) + { + DownloadURLDirect(archive, (archive as DirectURLArchive).URL); + } + else + { + + } + }); + } + + private void DownloadURLDirect(Archive archive, string url) + { + HttpClient client = new HttpClient(); + client.DefaultRequestHeaders.Add("User-Agent", Consts.UserAgent); + long total_read = 0; + int buffer_size = 1024 * 32; + + var response = client.GetSync(url); + var stream = response.Content.ReadAsStreamAsync(); + stream.Wait(); + + var header = response.Content.Headers.GetValues("Content-Length").FirstOrDefault(); + long content_size = header != null ? long.Parse(header) : 1; + + var output_path = Path.Combine(DownloadFolder, archive.Name); + + if (output_path.FileExists()) + File.Delete(output_path); + + using (var webs = stream.Result) + using (var fs = File.OpenWrite(output_path)) + { + var buffer = new byte[buffer_size]; + while (true) + { + var read = webs.Read(buffer, 0, buffer_size); + if (read == 0) break; + Status((int)(total_read * 100 / content_size), "Downloading {0}", archive.Name); + + fs.Write(buffer, 0, read); + total_read += read; + + } + } + Status("Hashing {0}", archive.Name); + HashArchive(output_path); + } + + private object GetNexusAPIKey() + { + throw new NotImplementedException(); + } + + private void HashArchives() + { + HashedArchives = Directory.EnumerateFiles(DownloadFolder) + .Where(e => Consts.SupportedArchives.Contains(Path.GetExtension(e))) + .PMap(e => (HashArchive(e), e)) + .ToDictionary(e => e.Item1, e => e.Item2); + + } + + private string HashArchive(string e) + { + var cache = e + ".sha"; + if (cache.FileExists() && new FileInfo(cache).LastWriteTime >= new FileInfo(e).LastWriteTime) + return File.ReadAllText(cache); + + Status("Hashing {0}", Path.GetFileName(e)); + File.WriteAllText(cache, Utils.FileSHA256(e)); + return HashArchive(e); + + } + + } +} diff --git a/Wabbajack/NexusAPI.cs b/Wabbajack/NexusAPI.cs new file mode 100644 index 00000000..c0ccd1fe --- /dev/null +++ b/Wabbajack/NexusAPI.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common; +using WebSocketSharp; + +namespace Wabbajack +{ + class NexusAPI + { + public static string GetNexusAPIKey() + { + FileInfo fi = new FileInfo("nexus.key_cache"); + if (fi.Exists && fi.LastWriteTime > DateTime.Now.AddHours(-12)) + { + return File.ReadAllText("nexus.key_cache"); + } + + var guid = Guid.NewGuid(); + var _websocket = new WebSocket("wss://sso.nexusmods.com") + { + SslConfiguration = { + EnabledSslProtocols = System.Security.Authentication.SslProtocols.Tls12 + } + }; + + TaskCompletionSource api_key = new TaskCompletionSource(); + _websocket.OnMessage += (sender, msg) => + { + api_key.SetResult(msg.Data); + return; + }; + + _websocket.Connect(); + _websocket.Send("{\"id\": \"" + guid + "\", \"appid\": \"Wabbajack\"}"); + + Process.Start($"https://www.nexusmods.com/sso?id={guid}&application=Wabbajack"); + + api_key.Task.Wait(); + var result = api_key.Task.Result; + File.WriteAllText("nexus.key_cache", result); + return result; + } + class DownloadLink + { + public string URI { get; set; } + } + private static HttpClient BaseNexusClient(string apikey) + { + var _baseHttpClient = new HttpClient(); + + _baseHttpClient.DefaultRequestHeaders.Add("User-Agent", Consts.UserAgent); + _baseHttpClient.DefaultRequestHeaders.Add("apikey", apikey); + _baseHttpClient.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/json")); + _baseHttpClient.DefaultRequestHeaders.Add("Application-Name", "Wabbajack"); + _baseHttpClient.DefaultRequestHeaders.Add("Application-Version", $"{Assembly.GetEntryAssembly().GetName().Version}"); + return _baseHttpClient; + } + + public static string GetNexusDownloadLink(NexusMod archive, string apikey) + { + var client = BaseNexusClient(apikey); + string url; + string get_url_link = String.Format("https://api.nexusmods.com/v1/games/{0}/mods/{1}/files/{2}/download_link.json", + ConvertGameName(archive.GameName), archive.ModID, archive.FileID); + using (var s = client.GetStreamSync(get_url_link)) + { + url = s.FromJSON>().First().URI; + return url; + } + } + + private static string ConvertGameName(string gameName) + { + if (gameName == "SkyrimSE") return "skyrimspecialedition"; + return gameName; + + } + + } +}