diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs new file mode 100644 index 00000000..8c566eab --- /dev/null +++ b/Wabbajack.Lib/AInstaller.cs @@ -0,0 +1,293 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using Wabbajack.Common; +using Wabbajack.Lib.Downloaders; +using Wabbajack.VirtualFileSystem; +using Context = Wabbajack.VirtualFileSystem.Context; +using Directory = Alphaleonis.Win32.Filesystem.Directory; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.Lib +{ + public abstract class AInstaller + { + public bool IgnoreMissingFiles { get; internal set; } = false; + + public Context VFS { get; internal set; } = new Context(); + + public string OutputFolder { get; set; } + public string DownloadFolder { get; set; } + + public ModManager ModManager; + + public string ModListArchive { get; internal set; } + public ModList ModList { get; internal set; } + public Dictionary HashedArchives { get; set; } + + public abstract void Install(); + + public void Info(string msg) + { + Utils.Log(msg); + } + + public void Status(string msg) + { + WorkQueue.Report(msg, 0); + } + + public void Error(string msg) + { + Utils.Log(msg); + throw new Exception(msg); + } + + public byte[] LoadBytesFromPath(string path) + { + using (var fs = new FileStream(ModListArchive, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) + using (var ms = new MemoryStream()) + { + var entry = ar.GetEntry(path); + using (var e = entry.Open()) + e.CopyTo(ms); + return ms.ToArray(); + } + } + + public static ModList LoadFromFile(string path) + { + using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) + { + var entry = ar.GetEntry("modlist"); + if (entry == null) + { + entry = ar.GetEntry("modlist.json"); + using (var e = entry.Open()) + return e.FromJSON(); + } + using (var e = entry.Open()) + return e.FromCERAS(ref CerasConfig.Config); + } + } + + /// + /// We don't want to make the installer index all the archives, that's just a waste of time, so instead + /// we'll pass just enough information to VFS to let it know about the files we have. + /// + public void PrimeVFS() + { + VFS.AddKnown(HashedArchives.Select(a => new KnownFile + { + Paths = new[] { a.Value }, + Hash = a.Key + })); + + + VFS.AddKnown( + ModList.Directives + .OfType() + .Select(f => new KnownFile { Paths = f.ArchiveHashPath})); + + VFS.BackfillMissing(); + } + + public void BuildFolderStructure() + { + Info("Building Folder Structure"); + ModList.Directives + .Select(d => Path.Combine(OutputFolder, Path.GetDirectoryName(d.To))) + .ToHashSet() + .Do(f => + { + if (Directory.Exists(f)) return; + Directory.CreateDirectory(f); + }); + } + + public void InstallArchives() + { + Info("Installing Archives"); + Info("Grouping Install Files"); + var grouped = ModList.Directives + .OfType() + .GroupBy(e => e.ArchiveHashPath[0]) + .ToDictionary(k => k.Key); + var archives = ModList.Archives + .Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) }) + .Where(a => a.AbsolutePath != null) + .ToList(); + + Info("Installing Archives"); + archives.PMap(a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash])); + } + + private void InstallArchive(Archive archive, string absolutePath, IGrouping grouping) + { + Status($"Extracting {archive.Name}"); + + List vFiles = grouping.Select(g => + { + var file = VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath); + g.FromFile = file; + return g; + }).ToList(); + + var onFinish = VFS.Stage(vFiles.Select(f => f.FromFile).Distinct()); + + + Status($"Copying files for {archive.Name}"); + + void CopyFile(string from, string to, bool useMove) + { + if (File.Exists(to)) + { + var fi = new FileInfo(to); + if (fi.IsReadOnly) + fi.IsReadOnly = false; + File.Delete(to); + } + + if (File.Exists(from)) + { + var fi = new FileInfo(from); + if (fi.IsReadOnly) + fi.IsReadOnly = false; + } + + + if (useMove) + File.Move(from, to); + else + File.Copy(from, to); + } + + vFiles.GroupBy(f => f.FromFile) + .DoIndexed((idx, group) => + { + Utils.Status("Installing files", idx * 100 / vFiles.Count); + var firstDest = Path.Combine(OutputFolder, group.First().To); + CopyFile(group.Key.StagedPath, firstDest, true); + + foreach (var copy in group.Skip(1)) + { + var nextDest = Path.Combine(OutputFolder, copy.To); + CopyFile(firstDest, nextDest, false); + } + + }); + + Status("Unstaging files"); + onFinish(); + + // Now patch all the files from this archive + foreach (var toPatch in grouping.OfType()) + using (var patchStream = new MemoryStream()) + { + Status($"Patching {Path.GetFileName(toPatch.To)}"); + // Read in the patch data + + byte[] patchData = LoadBytesFromPath(toPatch.PatchID); + + var toFile = Path.Combine(OutputFolder, toPatch.To); + var oldData = new MemoryStream(File.ReadAllBytes(toFile)); + + // Remove the file we're about to patch + File.Delete(toFile); + + // Patch it + using (var outStream = File.OpenWrite(toFile)) + { + BSDiff.Apply(oldData, () => new MemoryStream(patchData), outStream); + } + + Status($"Verifying Patch {Path.GetFileName(toPatch.To)}"); + var resultSha = toFile.FileHash(); + if (resultSha != toPatch.Hash) + throw new InvalidDataException($"Invalid Hash for {toPatch.To} after patching"); + } + } + + public void DownloadArchives() + { + var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList(); + Info($"Missing {missing.Count} archives"); + + Info("Getting Nexus API Key, if a browser appears, please accept"); + + var dispatchers = missing.Select(m => m.State.GetDownloader()).Distinct(); + + foreach (var dispatcher in dispatchers) + dispatcher.Prepare(); + + DownloadMissingArchives(missing); + } + + private void DownloadMissingArchives(List missing, bool download = true) + { + if (download) + { + foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State))) + { + var outputPath = Path.Combine(DownloadFolder, a.Name); + a.State.Download(a, outputPath); + } + } + + missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State)) + .PMap(archive => + { + Info($"Downloading {archive.Name}"); + var outputPath = Path.Combine(DownloadFolder, archive.Name); + + if (download) + if (outputPath.FileExists()) + File.Delete(outputPath); + + return DownloadArchive(archive, download); + }); + } + + public bool DownloadArchive(Archive archive, bool download) + { + try + { + archive.State.Download(archive, Path.Combine(DownloadFolder, archive.Name)); + } + catch (Exception ex) + { + Utils.Log($"Download error for file {archive.Name}"); + Utils.Log(ex.ToString()); + return false; + } + + return false; + } + + public void HashArchives() + { + HashedArchives = Directory.EnumerateFiles(DownloadFolder) + .Where(e => !e.EndsWith(".sha")) + .PMap(e => (HashArchive(e), e)) + .OrderByDescending(e => File.GetLastWriteTime(e.Item2)) + .GroupBy(e => e.Item1) + .Select(e => e.First()) + .ToDictionary(e => e.Item1, e => e.Item2); + } + + public string HashArchive(string e) + { + var cache = e + ".sha"; + if (cache.FileExists() && new FileInfo(cache).LastWriteTime >= new FileInfo(e).LastWriteTime) + return File.ReadAllText(cache); + + Status($"Hashing {Path.GetFileName(e)}"); + File.WriteAllText(cache, e.FileHash()); + return HashArchive(e); + } + } +} diff --git a/Wabbajack.Lib/Installer.cs b/Wabbajack.Lib/Installer.cs index 4b38d7d5..f3d86826 100644 --- a/Wabbajack.Lib/Installer.cs +++ b/Wabbajack.Lib/Installer.cs @@ -1,16 +1,12 @@ using System; -using System.Collections.Generic; using System.IO; -using System.IO.Compression; using System.Linq; using System.Text; -using System.Threading.Tasks; using System.Windows; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; -using Wabbajack.VirtualFileSystem; using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; @@ -18,90 +14,24 @@ using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib { - public class Installer + public class Installer : AInstaller { - private string _downloadsFolder; - private WorkQueue Queue { get; set; } public Installer(string archive, ModList mod_list, string output_folder) { Queue = new WorkQueue(); VFS = new Context(Queue); + ModManager = ModManager.MO2; ModListArchive = archive; - Outputfolder = output_folder; + OutputFolder = output_folder; + DownloadFolder = Path.Combine(OutputFolder, "downloads"); ModList = mod_list; } - public Context VFS { get; } - - public string Outputfolder { get; } - - public string DownloadFolder - { - get => _downloadsFolder ?? Path.Combine(Outputfolder, "downloads"); - set => _downloadsFolder = value; - } - - public string ModListArchive { get; } - public ModList ModList { get; } - public Dictionary HashedArchives { get; private set; } - - public bool IgnoreMissingFiles { get; internal set; } public string GameFolder { get; set; } - public void Info(string msg) - { - Utils.Log(msg); - } - - public void Status(string msg) - { - Queue.Report(msg, 0); - } - - public void Status(string msg, int progress) - { - Queue.Report(msg, progress); - } - - private void Error(string msg) - { - Utils.Log(msg); - throw new Exception(msg); - } - - public byte[] LoadBytesFromPath(string path) - { - using (var fs = new FileStream(ModListArchive, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) - using (var ms = new MemoryStream()) - { - var entry = ar.GetEntry(path); - using (var e = entry.Open()) - e.CopyTo(ms); - return ms.ToArray(); - } - } - - public static ModList LoadFromFile(string path) - { - using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) - { - var entry = ar.GetEntry("modlist"); - if (entry == null) - { - entry = ar.GetEntry("modlist.json"); - using (var e = entry.Open()) - return e.FromJSON(); - } - using (var e = entry.Open()) - return e.FromCERAS(ref CerasConfig.Config); - } - } - - public void Install() + public override void Install() { var game = GameRegistry.Games[ModList.GameType]; @@ -121,10 +51,10 @@ namespace Wabbajack.Lib ValidateGameESMs(); ValidateModlist.RunValidation(ModList); - Directory.CreateDirectory(Outputfolder); + Directory.CreateDirectory(OutputFolder); Directory.CreateDirectory(DownloadFolder); - if (Directory.Exists(Path.Combine(Outputfolder, "mods"))) + if (Directory.Exists(Path.Combine(OutputFolder, "mods"))) { if (MessageBox.Show( "There already appears to be a Mod Organizer 2 install in this folder, are you sure you wish to continue" + @@ -159,7 +89,7 @@ namespace Wabbajack.Lib BuildFolderStructure(); InstallArchives(); InstallIncludedFiles(); - InctallIncludedDownloadMetas(); + InstallIncludedDownloadMetas(); BuildBSAs(); zEditIntegration.GenerateMerges(this); @@ -170,7 +100,7 @@ namespace Wabbajack.Lib //AskToEndorse(); } - private void InctallIncludedDownloadMetas() + private void InstallIncludedDownloadMetas() { ModList.Directives .OfType() @@ -233,27 +163,6 @@ namespace Wabbajack.Lib Info("Done! You may now exit the application!"); } - /// - /// We don't want to make the installer index all the archives, that's just a waste of time, so instead - /// we'll pass just enough information to VFS to let it know about the files we have. - /// - private void PrimeVFS() - { - VFS.AddKnown(HashedArchives.Select(a => new KnownFile - { - Paths = new[] { a.Value }, - Hash = a.Key - })); - - - VFS.AddKnown( - ModList.Directives - .OfType() - .Select(f => new KnownFile { Paths = f.ArchiveHashPath})); - - VFS.BackfillMissing(); - } - private void BuildBSAs() { var bsas = ModList.Directives.OfType().ToList(); @@ -262,7 +171,7 @@ namespace Wabbajack.Lib bsas.Do(bsa => { Status($"Building {bsa.To}"); - var source_dir = Path.Combine(Outputfolder, Consts.BSACreationDir, bsa.TempID); + var source_dir = Path.Combine(OutputFolder, Consts.BSACreationDir, bsa.TempID); using (var a = bsa.State.MakeBuilder()) { @@ -276,12 +185,12 @@ namespace Wabbajack.Lib }); Info($"Writing {bsa.To}"); - a.Build(Path.Combine(Outputfolder, bsa.To)); + a.Build(Path.Combine(OutputFolder, bsa.To)); } }); - var bsa_dir = Path.Combine(Outputfolder, Consts.BSACreationDir); + var bsa_dir = Path.Combine(OutputFolder, Consts.BSACreationDir); if (Directory.Exists(bsa_dir)) { Info($"Removing temp folder {Consts.BSACreationDir}"); @@ -297,7 +206,7 @@ namespace Wabbajack.Lib .PMap(Queue, directive => { Status($"Writing included file {directive.To}"); - var out_path = Path.Combine(Outputfolder, directive.To); + var out_path = Path.Combine(OutputFolder, directive.To); if (File.Exists(out_path)) File.Delete(out_path); if (directive is RemappedInlineFile) WriteRemappedFile((RemappedInlineFile)directive); @@ -321,7 +230,7 @@ namespace Wabbajack.Lib $"Cannot patch {filename} from the game folder hashes don't match have you already cleaned the file?"); var patch_data = LoadBytesFromPath(directive.SourceDataID); - var to_file = Path.Combine(Outputfolder, directive.To); + var to_file = Path.Combine(OutputFolder, directive.To); Status($"Patching {filename}"); using (var output = File.OpenWrite(to_file)) using (var input = File.OpenRead(game_file)) @@ -338,209 +247,15 @@ namespace Wabbajack.Lib data = data.Replace(Consts.GAME_PATH_MAGIC_DOUBLE_BACK, GameFolder.Replace("\\", "\\\\")); data = data.Replace(Consts.GAME_PATH_MAGIC_FORWARD, GameFolder.Replace("\\", "/")); - data = data.Replace(Consts.MO2_PATH_MAGIC_BACK, Outputfolder); - data = data.Replace(Consts.MO2_PATH_MAGIC_DOUBLE_BACK, Outputfolder.Replace("\\", "\\\\")); - data = data.Replace(Consts.MO2_PATH_MAGIC_FORWARD, Outputfolder.Replace("\\", "/")); + data = data.Replace(Consts.MO2_PATH_MAGIC_BACK, OutputFolder); + data = data.Replace(Consts.MO2_PATH_MAGIC_DOUBLE_BACK, OutputFolder.Replace("\\", "\\\\")); + data = data.Replace(Consts.MO2_PATH_MAGIC_FORWARD, OutputFolder.Replace("\\", "/")); data = data.Replace(Consts.DOWNLOAD_PATH_MAGIC_BACK, DownloadFolder); data = data.Replace(Consts.DOWNLOAD_PATH_MAGIC_DOUBLE_BACK, DownloadFolder.Replace("\\", "\\\\")); data = data.Replace(Consts.DOWNLOAD_PATH_MAGIC_FORWARD, DownloadFolder.Replace("\\", "/")); - File.WriteAllText(Path.Combine(Outputfolder, directive.To), data); - } - - private void BuildFolderStructure() - { - Info("Building Folder Structure"); - ModList.Directives - .Select(d => Path.Combine(Outputfolder, Path.GetDirectoryName(d.To))) - .ToHashSet() - .Do(f => - { - if (Directory.Exists(f)) return; - Directory.CreateDirectory(f); - }); - } - - private void InstallArchives() - { - Info("Installing Archives"); - Info("Grouping Install Files"); - var grouped = ModList.Directives - .OfType() - .GroupBy(e => e.ArchiveHashPath[0]) - .ToDictionary(k => k.Key); - var archives = ModList.Archives - .Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) }) - .Where(a => a.AbsolutePath != null) - .ToList(); - - Info("Installing Archives"); - archives.PMap(Queue, a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash])); - } - - private void InstallArchive(Archive archive, string absolutePath, IGrouping grouping) - { - Status($"Extracting {archive.Name}"); - - var vfiles = grouping.Select(g => - { - var file = VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath); - g.FromFile = file; - return g; - }).ToList(); - - var on_finish = VFS.Stage(vfiles.Select(f => f.FromFile).Distinct()); - - - Status($"Copying files for {archive.Name}"); - - void CopyFile(string from, string to, bool use_move) - { - if (File.Exists(to)) - { - var fi = new FileInfo(to); - if (fi.IsReadOnly) - fi.IsReadOnly = false; - File.Delete(to); - } - - if (File.Exists(from)) - { - var fi = new FileInfo(from); - if (fi.IsReadOnly) - fi.IsReadOnly = false; - } - - - if (use_move) - File.Move(from, to); - else - File.Copy(from, to); - } - - vfiles.GroupBy(f => f.FromFile) - .DoIndexed((idx, group) => - { - Utils.Status("Installing files", idx * 100 / vfiles.Count); - var first_dest = Path.Combine(Outputfolder, group.First().To); - CopyFile(group.Key.StagedPath, first_dest, true); - - foreach (var copy in group.Skip(1)) - { - var next_dest = Path.Combine(Outputfolder, copy.To); - CopyFile(first_dest, next_dest, false); - } - - }); - - Status("Unstaging files"); - on_finish(); - - // Now patch all the files from this archive - foreach (var to_patch in grouping.OfType()) - using (var patch_stream = new MemoryStream()) - { - Status($"Patching {Path.GetFileName(to_patch.To)}"); - // Read in the patch data - - var patch_data = LoadBytesFromPath(to_patch.PatchID); - - var to_file = Path.Combine(Outputfolder, to_patch.To); - var old_data = new MemoryStream(File.ReadAllBytes(to_file)); - - // Remove the file we're about to patch - File.Delete(to_file); - - // Patch it - using (var out_stream = File.OpenWrite(to_file)) - { - BSDiff.Apply(old_data, () => new MemoryStream(patch_data), out_stream); - } - - Status($"Verifying Patch {Path.GetFileName(to_patch.To)}"); - var result_sha = to_file.FileHash(); - if (result_sha != to_patch.Hash) - throw new InvalidDataException($"Invalid Hash for {to_patch.To} after patching"); - } - } - - private void DownloadArchives() - { - var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList(); - Info($"Missing {missing.Count} archives"); - - Info("Getting Nexus API Key, if a browser appears, please accept"); - - var dispatchers = missing.Select(m => m.State.GetDownloader()).Distinct(); - - foreach (var dispatcher in dispatchers) - dispatcher.Prepare(); - - DownloadMissingArchives(missing); - } - - private void DownloadMissingArchives(List missing, bool download = true) - { - if (download) - { - foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State))) - { - var output_path = Path.Combine(DownloadFolder, a.Name); - a.State.Download(a, output_path); - } - } - - missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State)) - .PMap(Queue, archive => - { - Info($"Downloading {archive.Name}"); - var output_path = Path.Combine(DownloadFolder, archive.Name); - - if (download) - if (output_path.FileExists()) - File.Delete(output_path); - - return DownloadArchive(archive, download); - }); - } - - public bool DownloadArchive(Archive archive, bool download) - { - try - { - archive.State.Download(archive, Path.Combine(DownloadFolder, archive.Name)); - } - catch (Exception ex) - { - Utils.Log($"Download error for file {archive.Name}"); - Utils.Log(ex.ToString()); - return false; - } - - return false; - } - - private void HashArchives() - { - HashedArchives = Directory.EnumerateFiles(DownloadFolder) - .Where(e => !e.EndsWith(".sha")) - .PMap(Queue, e => (HashArchive(e), e)) - .OrderByDescending(e => File.GetLastWriteTime(e.Item2)) - .GroupBy(e => e.Item1) - .Select(e => e.First()) - .ToDictionary(e => e.Item1, e => e.Item2); - } - - private string HashArchive(string e) - { - var cache = e + ".sha"; - if (cache.FileExists() && new FileInfo(cache).LastWriteTime >= new FileInfo(e).LastWriteTime) - return File.ReadAllText(cache); - - Status($"Hashing {Path.GetFileName(e)}"); - File.WriteAllText(cache, e.FileHash()); - return HashArchive(e); + File.WriteAllText(Path.Combine(OutputFolder, directive.To), data); } } } diff --git a/Wabbajack.Lib/VortexInstaller.cs b/Wabbajack.Lib/VortexInstaller.cs index 6fc8cf69..8f1b5f78 100644 --- a/Wabbajack.Lib/VortexInstaller.cs +++ b/Wabbajack.Lib/VortexInstaller.cs @@ -13,27 +13,18 @@ using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib { - public class VortexInstaller + public class VortexInstaller : AInstaller { - public string ModListArchive { get; } - public ModList ModList { get; } - public Dictionary HashedArchives { get; private set; } - public GameMetaData GameInfo { get; internal set; } - public string StagingFolder { get; set; } - public string DownloadFolder { get; set; } - public WorkQueue Queue { get; } - public Context VFS { get; } public bool IgnoreMissingFiles { get; internal set; } public VortexInstaller(string archive, ModList modList) { Queue = new WorkQueue(); - VFS = new Context(Queue); - + ModManager = ModManager.Vortex; ModListArchive = archive; ModList = modList; @@ -43,53 +34,7 @@ namespace Wabbajack.Lib GameInfo = GameRegistry.Games[ModList.GameType]; } - public void Info(string msg) - { - Utils.Log(msg); - } - - public void Status(string msg) - { - Queue.Report(msg, 0); - } - - private void Error(string msg) - { - Utils.Log(msg); - throw new Exception(msg); - } - - public byte[] LoadBytesFromPath(string path) - { - using (var fs = new FileStream(ModListArchive, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) - using (var ms = new MemoryStream()) - { - var entry = ar.GetEntry(path); - using (var e = entry.Open()) - e.CopyTo(ms); - return ms.ToArray(); - } - } - - public static ModList LoadFromFile(string path) - { - using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) - { - var entry = ar.GetEntry("modlist"); - if (entry == null) - { - entry = ar.GetEntry("modlist.json"); - using (var e = entry.Open()) - return e.FromJSON(); - } - using (var e = entry.Open()) - return e.FromCERAS(ref CerasConfig.Config); - } - } - - public void Install() + public override void Install() { Directory.CreateDirectory(DownloadFolder); @@ -113,85 +58,11 @@ namespace Wabbajack.Lib BuildFolderStructure(); InstallArchives(); InstallIncludedFiles(); - //InctallIncludedDownloadMetas(); + //InstallIncludedDownloadMetas(); Info("Installation complete! You may exit the program."); } - private void BuildFolderStructure() - { - Info("Building Folder Structure"); - ModList.Directives - .OfType() - .Select(d => Path.Combine(StagingFolder, Path.GetDirectoryName(d.To))) - .ToHashSet() - .Do(f => - { - if (Directory.Exists(f)) return; - Directory.CreateDirectory(f); - }); - } - - private void InstallArchives() - { - Info("Installing Archives"); - Info("Grouping Install Files"); - var grouped = ModList.Directives - .OfType() - .GroupBy(e => e.ArchiveHashPath[0]) - .ToDictionary(k => k.Key); - var archives = ModList.Archives - .Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) }) - .Where(a => a.AbsolutePath != null) - .ToList(); - - Info("Installing Archives"); - archives.PMap(Queue,a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash])); - } - - private void InstallArchive(Archive archive, string absolutePath, IGrouping grouping) - { - Status($"Extracting {archive.Name}"); - - var vFiles = grouping.Select(g => - { - var file = VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath); - g.FromFile = file; - return g; - }).ToList(); - - var onFinish = VFS.Stage(vFiles.Select(f => f.FromFile).Distinct()); - - Status($"Copying files for {archive.Name}"); - - void CopyFile(string from, string to, bool useMove) - { - if(File.Exists(to)) - File.Delete(to); - if (useMove) - File.Move(from, to); - else - File.Copy(from, to); - } - - vFiles.GroupBy(f => f.FromFile) - .DoIndexed((idx, group) => - { - Utils.Status("Installing files", idx * 100 / vFiles.Count); - var firstDest = Path.Combine(StagingFolder, group.First().To); - CopyFile(group.Key.StagedPath, firstDest, true); - - foreach (var copy in group.Skip(1)) - { - var nextDest = Path.Combine(StagingFolder, copy.To); - CopyFile(firstDest, nextDest, false); - } - }); - - Status("Unstaging files"); - onFinish(); - } - private void InstallIncludedFiles() { Info("Writing inline files"); @@ -199,114 +70,10 @@ namespace Wabbajack.Lib .PMap(Queue,directive => { Status($"Writing included file {directive.To}"); - var outPath = Path.Combine(StagingFolder, directive.To); + var outPath = Path.Combine(OutputFolder, directive.To); if(File.Exists(outPath)) File.Delete(outPath); File.WriteAllBytes(outPath, LoadBytesFromPath(directive.SourceDataID)); }); } - - /// - /// We don't want to make the installer index all the archives, that's just a waste of time, so instead - /// we'll pass just enough information to VFS to let it know about the files we have. - /// - private void PrimeVFS() - { - VFS.AddKnown(HashedArchives.Select(a => new KnownFile - { - Paths = new[] { a.Value }, - Hash = a.Key - })); - - - ModList.Directives - .OfType() - .Select(f => - { - var updated_path = new string[f.ArchiveHashPath.Length]; - f.ArchiveHashPath.CopyTo(updated_path, 0); - updated_path[0] = VFS.Index.ByHash[updated_path[0]].First(e => e.IsNative).FullPath; - return new KnownFile { Paths = updated_path }; - }); - - VFS.BackfillMissing(); - } - - private void DownloadArchives() - { - var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList(); - Info($"Missing {missing.Count} archives"); - - Info("Getting Nexus API Key, if a browser appears, please accept"); - - var dispatchers = missing.Select(m => m.State.GetDownloader()).Distinct(); - - foreach (var dispatcher in dispatchers) - dispatcher.Prepare(); - - DownloadMissingArchives(missing); - } - - private void DownloadMissingArchives(List missing, bool download = true) - { - if (download) - { - foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State))) - { - var output_path = Path.Combine(DownloadFolder, a.Name); - a.State.Download(a, output_path); - } - } - - missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State)) - .PMap(Queue, archive => - { - Info($"Downloading {archive.Name}"); - var output_path = Path.Combine(DownloadFolder, archive.Name); - - if (!download) return DownloadArchive(archive, download); - if (output_path.FileExists()) - File.Delete(output_path); - - return DownloadArchive(archive, download); - }); - } - - public bool DownloadArchive(Archive archive, bool download) - { - try - { - archive.State.Download(archive, Path.Combine(DownloadFolder, archive.Name)); - } - catch (Exception ex) - { - Utils.Log($"Download error for file {archive.Name}"); - Utils.Log(ex.ToString()); - return false; - } - - return false; - } - - private void HashArchives() - { - HashedArchives = Directory.EnumerateFiles(DownloadFolder) - .Where(e => !e.EndsWith(".sha")) - .PMap(Queue,e => (HashArchive(e), e)) - .OrderByDescending(e => File.GetLastWriteTime(e.Item2)) - .GroupBy(e => e.Item1) - .Select(e => e.First()) - .ToDictionary(e => e.Item1, e => e.Item2); - } - - private string HashArchive(string e) - { - var cache = e + ".sha"; - if (cache.FileExists() && new FileInfo(cache).LastWriteTime >= new FileInfo(e).LastWriteTime) - return File.ReadAllText(cache); - - Status($"Hashing {Path.GetFileName(e)}"); - File.WriteAllText(cache, e.FileHash()); - return HashArchive(e); - } } } diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 6c10d581..b17545b0 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -79,6 +79,7 @@ +