From d52a09094b732faee14091be7e46834307bbfedb Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 12 Jul 2020 21:50:12 -0600 Subject: [PATCH] Update ChangeDownload.cs to use Path code. Update MEGADownloader to not try and heal archives --- Wabbajack.CLI/CLIUtils.cs | 1 + Wabbajack.CLI/Verbs/ChangeDownload.cs | 216 ++++++++++---------- Wabbajack.Lib/Downloaders/HTTPDownloader.cs | 2 +- Wabbajack.Lib/Downloaders/MEGADownloader.cs | 5 + 4 files changed, 113 insertions(+), 111 deletions(-) diff --git a/Wabbajack.CLI/CLIUtils.cs b/Wabbajack.CLI/CLIUtils.cs index e5304c1b..d227b2bb 100644 --- a/Wabbajack.CLI/CLIUtils.cs +++ b/Wabbajack.CLI/CLIUtils.cs @@ -180,6 +180,7 @@ namespace Wabbajack.CLI Console.WriteLine(msg); else Console.Write(msg); + Console.Out.Flush(); } internal static ExitCode Exit(string msg, ExitCode code) diff --git a/Wabbajack.CLI/Verbs/ChangeDownload.cs b/Wabbajack.CLI/Verbs/ChangeDownload.cs index 31da2f97..9e1e5fc1 100644 --- a/Wabbajack.CLI/Verbs/ChangeDownload.cs +++ b/Wabbajack.CLI/Verbs/ChangeDownload.cs @@ -19,19 +19,27 @@ namespace Wabbajack.CLI.Verbs { [IsDirectory(CustomMessage = "Downloads folder %1 does not exist!")] [Option("input", Required = true, HelpText = "Input folder containing the downloads you want to move")] - public string Input { get; set; } = ""; + public string _input { get; set; } = ""; + public AbsolutePath Input => (AbsolutePath)_input; + [IsDirectory(Create = true)] [Option("output", Required = true, HelpText = "Output folder the downloads should be transferred to")] - public string Output { get; set; } = ""; + public string _output { get; set; } = ""; + + public AbsolutePath Output => (AbsolutePath)_output; [IsFile(CustomMessage = "Modlist file %1 does not exist!")] [Option("modlist", Required = true, HelpText = "The Modlist, can either be a .wabbajack or a modlist.txt file")] - public string Modlist { get; set; } = ""; + public string _modlist { get; set; } = ""; + + public AbsolutePath ModList => (AbsolutePath)_modlist; [Option("mods", Required = false, HelpText = "Mods folder location if the provided modlist file is an MO2 modlist.txt")] - public string Mods { get; set; } = ""; + public string _mods { get; set; } = ""; + + public AbsolutePath Mods => (AbsolutePath)_mods; [Option("copy", Default = true, HelpText = "Whether to copy the files", SetName = "copy")] public bool Copy { get; set; } @@ -44,33 +52,64 @@ namespace Wabbajack.CLI.Verbs [Option("meta", Default = true, HelpText = "Whether to also transfer the meta file for the archive")] public bool IncludeMeta { get; set; } - - private struct TransferFile + + private interface ITransferFile { - public readonly string Input; - public readonly string Output; - public readonly bool IsMeta; + public Task PerformCopy(); + public AbsolutePath Output { get; } + } - public TransferFile(string input, string output, bool isMeta = false) + private struct FileCopy : ITransferFile + { + private AbsolutePath _src; + private AbsolutePath _dest; + + public FileCopy(AbsolutePath src, AbsolutePath dest) { - Input = input; - Output = output; - IsMeta = isMeta; + _src = src; + _dest = dest; } + + public async Task PerformCopy() + { + CLIUtils.Log($"Copying {_src} to {_dest}"); + await _src.CopyToAsync(_dest); + } + + public AbsolutePath Output => _dest; + } + + private struct StringCopy : ITransferFile + { + private string _src; + private AbsolutePath _dest; + + public StringCopy(string src, AbsolutePath dest) + { + _src = src; + _dest = dest; + } + + public async Task PerformCopy() + { + CLIUtils.Log($"Writing data to {_src}"); + await _dest.WriteAllTextAsync(_src); + } + public AbsolutePath Output => _dest; + } protected override async Task Run() { - var modListPath = (AbsolutePath)Modlist; - if (modListPath.Extension != Consts.ModListExtension && modListPath.FileName != (RelativePath)"modlist.txt") - return CLIUtils.Exit($"The file {Modlist} is not a valid modlist file!", ExitCode.BadArguments); + if (ModList.Extension != Consts.ModListExtension && ModList.FileName != (RelativePath)"modlist.txt") + return CLIUtils.Exit($"The file {ModList} is not a valid modlist file!", ExitCode.BadArguments); if (Copy && Move) return CLIUtils.Exit("You can't set both copy and move flags!", ExitCode.BadArguments); - var isModlist = modListPath.Extension == Consts.ModListExtension; + var isModlist = ModList.Extension == Consts.ModListExtension; - var list = new List(); + var list = new List(); if (isModlist) { @@ -78,7 +117,7 @@ namespace Wabbajack.CLI.Verbs try { - modlist = AInstaller.LoadFromFile(modListPath); + modlist = AInstaller.LoadFromFile(ModList); } catch (Exception e) { @@ -91,31 +130,41 @@ namespace Wabbajack.CLI.Verbs } CLIUtils.Log($"Modlist contains {modlist.Archives.Count} archives."); + + using var queue = new WorkQueue(); + + CLIUtils.Log($"Hashing Downloads (this may take some time)"); + var hashes = (await Input.EnumerateFiles().PMap(queue, async f => + { + CLIUtils.Log($"Hashing {f}"); + return (f, await f.FileHashCachedAsync()); + })) + .GroupBy(d => d.Item2) + .ToDictionary(d => d.Key, d => d.First().f); modlist.Archives.Do(a => { - var inputPath = Path.Combine(Input, a.Name); - var outputPath = Path.Combine(Output, a.Name); - - if (!File.Exists(inputPath)) + if (!hashes.TryGetValue(a.Hash, out var inputPath)) { - CLIUtils.Log($"File {inputPath} does not exist, skipping."); + CLIUtils.Log($"Archive not found for hash {a.Hash}"); return; } + var outputPath = Output.Combine(a.Name); + CLIUtils.Log($"Adding {inputPath} to the transfer list."); - list.Add(new TransferFile(inputPath, outputPath)); + list.Add(new FileCopy(inputPath, outputPath)); - var metaInputPath = Path.Combine(inputPath, ".meta"); - var metaOutputPath = Path.Combine(outputPath, ".meta"); + var metaInputPath = inputPath.WithExtension(Consts.MetaFileExtension); + var metaOutputPath = outputPath.WithExtension(Consts.MetaFileExtension); - if (File.Exists(metaInputPath)) + if (metaInputPath.Exists) { CLIUtils.Log($"Found meta file {metaInputPath}"); if (IncludeMeta) { CLIUtils.Log($"Adding {metaInputPath} to the transfer list."); - list.Add(new TransferFile(metaInputPath, metaOutputPath)); + list.Add(new FileCopy(metaInputPath, metaOutputPath)); } else { @@ -134,7 +183,7 @@ namespace Wabbajack.CLI.Verbs } CLIUtils.Log("Adding meta from archive info the transfer list"); - list.Add(new TransferFile(a.Meta, metaOutputPath, true)); + list.Add(new StringCopy(a.Meta, metaOutputPath)); } else { @@ -145,16 +194,16 @@ namespace Wabbajack.CLI.Verbs } else { - if (!Directory.Exists(Mods)) + if (!Mods.Exists) return CLIUtils.Exit($"Mods directory {Mods} does not exist!", ExitCode.BadArguments); - CLIUtils.Log($"Reading modlist.txt from {Modlist}"); - string[] modlist = File.ReadAllLines(Modlist); + CLIUtils.Log($"Reading modlist.txt from {ModList}"); + var modlist = await ModList.ReadAllLinesAsync(); - if (modlist == null || modlist.Length == 0) - return CLIUtils.Exit($"Provided modlist.txt file at {Modlist} is empty or could not be read!", ExitCode.BadArguments); + if (modlist == null || !modlist.Any()) + return CLIUtils.Exit($"Provided modlist.txt file at {ModList} is empty or could not be read!", ExitCode.BadArguments); - var mods = modlist.Where(s => s.StartsWith("+")).Select(s => s.Substring(1)).ToHashSet(); + var mods = modlist.Where(s => s.StartsWith("+")).Select(s => s.Substring(1)).Select(f => (RelativePath)f).ToHashSet(); if (mods.Count == 0) return CLIUtils.Exit("Counted mods from modlist.txt are 0!", ExitCode.BadArguments); @@ -163,19 +212,19 @@ namespace Wabbajack.CLI.Verbs var downloads = new HashSet(); - Directory.EnumerateDirectories(Mods, "*", SearchOption.TopDirectoryOnly) - .Where(d => mods.Contains(Path.GetRelativePath(Path.GetDirectoryName(d), d))) - .Do(d => + Mods.EnumerateDirectories(recursive:false) + .Where(d => mods.Contains(d.Parent.FileName)) + .Do(async d => { - var meta = Path.Combine(d, "meta.ini"); - if (!File.Exists(meta)) + var meta = d.Combine("meta.ini"); + if (!meta.Exists) { CLIUtils.Log($"Mod meta file {meta} does not exist, skipping"); return; } - string[] ini = File.ReadAllLines(meta); - if (ini == null || ini.Length == 0) + var ini = await meta.ReadAllLinesAsync(); + if (ini == null || !ini.Any()) { CLIUtils.Log($"Mod meta file {meta} could not be read or is empty!"); return; @@ -192,26 +241,26 @@ namespace Wabbajack.CLI.Verbs CLIUtils.Log($"Found {downloads.Count} installationFiles from mod metas."); - Directory.EnumerateFiles(Input, "*", SearchOption.TopDirectoryOnly) - .Where(f => downloads.Contains(Path.GetFileNameWithoutExtension(f))) + Input.EnumerateFiles() + .Where(f => downloads.Contains(f.FileNameWithoutExtension.ToString())) .Do(f => { CLIUtils.Log($"Found archive {f}"); - var outputPath = Path.Combine(Output, Path.GetFileName(f)); + var outputPath = Output.Combine(f.FileName); CLIUtils.Log($"Adding {f} to the transfer list"); - list.Add(new TransferFile(f, outputPath)); + list.Add(new FileCopy(f, outputPath)); - var metaInputPath = Path.Combine(f, ".meta"); - if (File.Exists(metaInputPath)) + var metaInputPath = f.WithExtension(Consts.MetaFileExtension); + if (metaInputPath.Exists) { CLIUtils.Log($"Found meta file for {f} at {metaInputPath}"); if (IncludeMeta) { - var metaOutputPath = Path.Combine(outputPath, ".meta"); + var metaOutputPath = outputPath.WithExtension(Consts.MetaFileExtension); CLIUtils.Log($"Adding {metaInputPath} to the transfer list."); - list.Add(new TransferFile(metaInputPath, metaOutputPath)); + list.Add(new FileCopy(metaInputPath, metaOutputPath)); } else { @@ -229,19 +278,20 @@ namespace Wabbajack.CLI.Verbs var success = 0; var failed = 0; var skipped = 0; - list.Do(f => + list.Do(async f => { - if (File.Exists(f.Output)) + if (f.Output.Exists) { if (Overwrite) { CLIUtils.Log($"Output file {f.Output} already exists, it will be overwritten"); - if (f.IsMeta || Move) + if (f is StringCopy || Move) { CLIUtils.Log($"Deleting file at {f.Output}"); try { - File.Delete(f.Output); + if (f.Output.Exists) + f.Output.Delete(); } catch (Exception e) { @@ -258,51 +308,7 @@ namespace Wabbajack.CLI.Verbs } } - if (f.IsMeta) - { - CLIUtils.Log($"Writing meta data to {f.Output}"); - try - { - File.WriteAllText(f.Output, f.Input, Encoding.UTF8); - success++; - } - catch (Exception e) - { - CLIUtils.Log($"Error while writing meta data to {f.Output}!\n{e}"); - failed++; - } - } - else - { - if (Copy) - { - CLIUtils.Log($"Copying file {f.Input} to {f.Output}"); - try - { - File.Copy(f.Input, f.Output, Overwrite ? CopyOptions.None : CopyOptions.FailIfExists, CopyMoveProgressHandler, null); - success++; - } - catch (Exception e) - { - CLIUtils.Log($"Error while copying file {f.Input} to {f.Output}!\n{e}"); - failed++; - } - } - else if(Move) - { - CLIUtils.Log($"Moving file {f.Input} to {f.Output}"); - try - { - File.Move(f.Input, f.Output, Overwrite ? MoveOptions.ReplaceExisting : MoveOptions.None, CopyMoveProgressHandler, null); - success++; - } - catch (Exception e) - { - CLIUtils.Log($"Error while moving file {f.Input} to {f.Output}!\n{e}"); - failed++; - } - } - } + await f.PerformCopy(); }); CLIUtils.Log($"Skipped transfers: {skipped}"); @@ -311,15 +317,5 @@ namespace Wabbajack.CLI.Verbs return 0; } - - private static CopyMoveProgressResult CopyMoveProgressHandler(long totalfilesize, long totalbytestransferred, long streamsize, long streambytestransferred, int streamnumber, CopyMoveProgressCallbackReason callbackreason, object userdata) - { - Console.Write($"\r{' ', 100}"); - - Console.Write(totalfilesize == totalbytestransferred - ? "\rTransfer complete!\n" - : $"\rTotal Size: {totalfilesize}, Transferred: {totalbytestransferred}"); - return CopyMoveProgressResult.Continue; - } } } diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index 92e33807..4cfb3cbc 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -212,7 +212,7 @@ TOP: } - public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a) + public virtual async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a) { var tmpFile = new TempFile(); diff --git a/Wabbajack.Lib/Downloaders/MEGADownloader.cs b/Wabbajack.Lib/Downloaders/MEGADownloader.cs index 4c928d74..5a741849 100644 --- a/Wabbajack.Lib/Downloaders/MEGADownloader.cs +++ b/Wabbajack.Lib/Downloaders/MEGADownloader.cs @@ -193,6 +193,11 @@ namespace Wabbajack.Lib.Downloaders return false; } } + + public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a) + { + return default; + } } public ReactiveCommand TriggerLogin { get; }