diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d74b69f..debee507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ install thread. * Fixed Extraction so that zip files no longer cause WJ to CTD * Better path logging during install and compilation * Fix the "this was created with a newer version of Wabbajack" issue +* If a downloaded file doesn't match the expected hash, try alternative download locations, if allowed #### Version - 2.2.2.0 - 8/31/2020 diff --git a/Wabbajack.Common/Logging.cs b/Wabbajack.Common/Logging.cs new file mode 100644 index 00000000..383a0fe7 --- /dev/null +++ b/Wabbajack.Common/Logging.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Subjects; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common.StatusFeed; +using Wabbajack.Common.StatusFeed.Errors; +using File = Alphaleonis.Win32.Filesystem.File; +using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.Common +{ + public static class LoggingSettings + { + // False by default, so that library users do not have to swap it + public static bool LogToFile = false; + } + + public static partial class Utils + { + public static AbsolutePath LogFile { get; private set; } + public static AbsolutePath LogFolder { get; private set; } + + private static object _logLock = new object(); + + private static DateTime _startTime; + + private static readonly Subject LoggerSubj = new Subject(); + public static IObservable LogMessages => LoggerSubj; + + private static async Task InitalizeLogging() + { + _startTime = DateTime.Now; + + if (LoggingSettings.LogToFile) + { + LogFolder = Consts.LogsFolder; + LogFile = Consts.LogFile; + Consts.LocalAppDataPath.CreateDirectory(); + Consts.LogsFolder.CreateDirectory(); + + if (LogFile.Exists) + { + var newPath = Consts.LogsFolder.Combine(Consts.EntryPoint.FileNameWithoutExtension + LogFile.LastModified.ToString(" yyyy-MM-dd HH_mm_ss") + ".log"); + await LogFile.MoveToAsync(newPath, true); + } + + var logFiles = LogFolder.EnumerateFiles(false).ToList(); + if (logFiles.Count >= Consts.MaxOldLogs) + { + Log($"Maximum amount of old logs reached ({logFiles.Count} >= {Consts.MaxOldLogs})"); + var filesToDelete = logFiles + .Where(f => f.IsFile) + .OrderBy(f => f.LastModified) + .Take(logFiles.Count - Consts.MaxOldLogs) + .ToList(); + + Log($"Found {filesToDelete.Count} old log files to delete"); + + var success = 0; + var failed = 0; + filesToDelete.Do(f => + { + try + { + f.Delete(); + success++; + } + catch (Exception e) + { + failed++; + Log($"Could not delete log at {f}!\n{e}"); + } + }); + + Log($"Deleted {success} log files, failed to delete {failed} logs"); + } + } + } + + + public static void Log(string msg) + { + Log(new GenericInfo(msg)); + } + + public static T Log(T msg) where T : IStatusMessage + { + LogStraightToFile(string.IsNullOrWhiteSpace(msg.ExtendedDescription) ? msg.ShortDescription : msg.ExtendedDescription); + LoggerSubj.OnNext(msg); + return msg; + } + + public static void Error(string errMessage) + { + Log(errMessage); + } + + public static void Error(Exception ex, string? extraMessage = null) + { + Log(new GenericException(ex, extraMessage)); + } + + public static void ErrorThrow(Exception ex, string? extraMessage = null) + { + Error(ex, extraMessage); + throw ex; + } + + public static void Error(IException err) + { + LogStraightToFile($"{err.ShortDescription}\n{err.Exception.StackTrace}"); + LoggerSubj.OnNext(err); + } + + public static void ErrorThrow(IException err) + { + Error(err); + throw err.Exception; + } + + public static void LogStraightToFile(string msg) + { + if (!LoggingSettings.LogToFile || LogFile == default) return; + lock (_logLock) + { + File.AppendAllText(LogFile.ToString(), $"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n"); + } + } + + public static void Status(string msg, Percent progress, bool alsoLog = false) + { + WorkQueue.AsyncLocalCurrentQueue.Value?.Report(msg, progress); + if (alsoLog) + { + Utils.Log(msg); + } + } + + public static void Status(string msg, bool alsoLog = false) + { + Status(msg, Percent.Zero, alsoLog: alsoLog); + } + + public static void CatchAndLog(Action a) + { + try + { + a(); + } + catch (Exception ex) + { + Utils.Error(ex); + } + } + + public static async Task CatchAndLog(Func f) + { + try + { + await f(); + } + catch (Exception ex) + { + Utils.Error(ex); + } + } + } +} diff --git a/Wabbajack.Common/Paths/FileCompaction.cs b/Wabbajack.Common/Paths/FileCompaction.cs index cd5bbaf3..be6a16be 100644 --- a/Wabbajack.Common/Paths/FileCompaction.cs +++ b/Wabbajack.Common/Paths/FileCompaction.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +using System.Linq; +using System.Threading.Tasks; using Wabbajack.Common.IO; namespace Wabbajack.Common @@ -60,7 +61,12 @@ namespace Wabbajack.Common public static async Task CompactFolder(this AbsolutePath folder, WorkQueue queue, Algorithm algorithm) { - await folder.EnumerateFiles(true) + var driveInfo = folder.DriveInfo().DiskSpaceInfo; + var clusterSize = driveInfo.SectorsPerCluster * driveInfo.BytesPerSector; + + await folder + .EnumerateFiles(true) + .Where(f => f.Size > clusterSize) .PMap(queue, async path => { Utils.Status($"Compacting {path.FileName}"); diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 3da4e717..6e88a707 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -1,16 +1,12 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Data.HashFunction.xxHash; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Net.Http; -using System.Reactive.Concurrency; using System.Reactive.Linq; -using System.Reactive.Subjects; -using System.Reflection; using System.Runtime.InteropServices; using System.Security; using System.Security.Cryptography; @@ -23,9 +19,6 @@ using IniParser; using IniParser.Model.Configuration; using IniParser.Parser; using Microsoft.Win32; -using Newtonsoft.Json; -using Wabbajack.Common.StatusFeed; -using Wabbajack.Common.StatusFeed.Errors; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; using Directory = System.IO.Directory; @@ -43,167 +36,13 @@ namespace Wabbajack.Common return processList.Where(process => process.ProcessName == "ModOrganizer").Any(process => Path.GetDirectoryName(process.MainModule?.FileName) == mo2Path); } - public static AbsolutePath LogFile { get; } - public static AbsolutePath LogFolder { get; } - - public enum FileEventType - { - Created, - Changed, - Deleted - } - static Utils() { - LogFolder = Consts.LogsFolder; - LogFile = Consts.LogFile; - Consts.LocalAppDataPath.CreateDirectory(); - Consts.LogsFolder.CreateDirectory(); - - _startTime = DateTime.Now; - - if (LogFile.Exists) - { - var newPath = Consts.LogsFolder.Combine(Consts.EntryPoint.FileNameWithoutExtension + LogFile.LastModified.ToString(" yyyy-MM-dd HH_mm_ss") + ".log"); - LogFile.MoveToAsync(newPath, true).Wait(); - } - - var logFiles = LogFolder.EnumerateFiles(false).ToList(); - if (logFiles.Count >= Consts.MaxOldLogs) - { - Log($"Maximum amount of old logs reached ({logFiles.Count} >= {Consts.MaxOldLogs})"); - var filesToDelete = logFiles - .Where(f => f.IsFile) - .OrderBy(f => f.LastModified) - .Take(logFiles.Count - Consts.MaxOldLogs) - .ToList(); - - Log($"Found {filesToDelete.Count} old log files to delete"); - - var success = 0; - var failed = 0; - filesToDelete.Do(f => - { - try - { - f.Delete(); - success++; - } - catch (Exception e) - { - failed++; - Log($"Could not delete log at {f}!\n{e}"); - } - }); - - Log($"Deleted {success} log files, failed to delete {failed} logs"); - } - - var watcher = new FileSystemWatcher((string)Consts.LocalAppDataPath); - AppLocalEvents = Observable.Merge(Observable.FromEventPattern(h => watcher.Changed += h, h => watcher.Changed -= h).Select(e => (FileEventType.Changed, e.EventArgs)), - Observable.FromEventPattern(h => watcher.Created += h, h => watcher.Created -= h).Select(e => (FileEventType.Created, e.EventArgs)), - Observable.FromEventPattern(h => watcher.Deleted += h, h => watcher.Deleted -= h).Select(e => (FileEventType.Deleted, e.EventArgs))) - .ObserveOn(Scheduler.Default); - watcher.EnableRaisingEvents = true; + InitalizeLogging().Wait(); } - private static readonly Subject LoggerSubj = new Subject(); - public static IObservable LogMessages => LoggerSubj; - private static readonly string[] Suffix = {"B", "KB", "MB", "GB", "TB", "PB", "EB"}; // Longs run out around EB - private static object _lock = new object(); - - private static DateTime _startTime; - - - public static void Log(string msg) - { - Log(new GenericInfo(msg)); - } - - public static T Log(T msg) where T : IStatusMessage - { - LogStraightToFile(string.IsNullOrWhiteSpace(msg.ExtendedDescription) ? msg.ShortDescription : msg.ExtendedDescription); - LoggerSubj.OnNext(msg); - return msg; - } - - public static void Error(string errMessage) - { - Log(errMessage); - } - - public static void Error(Exception ex, string? extraMessage = null) - { - Log(new GenericException(ex, extraMessage)); - } - - public static void ErrorThrow(Exception ex, string? extraMessage = null) - { - Error(ex, extraMessage); - throw ex; - } - - public static void Error(IException err) - { - LogStraightToFile($"{err.ShortDescription}\n{err.Exception.StackTrace}"); - LoggerSubj.OnNext(err); - } - - public static void ErrorThrow(IException err) - { - Error(err); - throw err.Exception; - } - - public static void LogStraightToFile(string msg) - { - if (LogFile == default) return; - lock (_lock) - { - File.AppendAllText(LogFile.ToString(), $"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n"); - } - } - - public static void Status(string msg, Percent progress, bool alsoLog = false) - { - WorkQueue.AsyncLocalCurrentQueue.Value?.Report(msg, progress); - if (alsoLog) - { - Utils.Log(msg); - } - } - - public static void Status(string msg, bool alsoLog = false) - { - Status(msg, Percent.Zero, alsoLog: alsoLog); - } - - public static void CatchAndLog(Action a) - { - try - { - a(); - } - catch (Exception ex) - { - Utils.Error(ex); - } - } - - public static async Task CatchAndLog(Func f) - { - try - { - await f(); - } - catch (Exception ex) - { - Utils.Error(ex); - } - } - public static void CopyToWithStatus(this Stream istream, long maxSize, Stream ostream, string status) { var buffer = new byte[1024 * 64]; @@ -993,16 +832,14 @@ namespace Wabbajack.Common return false; } - - public static IObservable<(FileEventType, FileSystemEventArgs)> AppLocalEvents { get; } - public static IObservable HaveEncryptedJsonObservable(string key) { var path = Consts.LocalAppDataPath.Combine(key); - return AppLocalEvents.Where(t => (AbsolutePath)t.Item2.FullPath.ToLower() == path) - .Select(_ => path.Exists) - .StartWith(path.Exists) - .DistinctUntilChanged(); + return WJFileWatcher.AppLocalEvents + .Where(t => (AbsolutePath)t.Item2.FullPath.ToLower() == path) + .Select(_ => path.Exists) + .StartWith(path.Exists) + .DistinctUntilChanged(); } public static async ValueTask DeleteEncryptedJson(string key) diff --git a/Wabbajack.Common/WJFileWatcher.cs b/Wabbajack.Common/WJFileWatcher.cs new file mode 100644 index 00000000..1b90963a --- /dev/null +++ b/Wabbajack.Common/WJFileWatcher.cs @@ -0,0 +1,32 @@ +using System; +using System.IO; +using System.Reactive.Concurrency; +using System.Reactive.Linq; +using System.Text; + +namespace Wabbajack.Common +{ + public static class WJFileWatcher + { + public enum FileEventType + { + Created, + Changed, + Deleted + } + + public static IObservable<(FileEventType, FileSystemEventArgs)> AppLocalEvents { get; } + + static WJFileWatcher() + { + var watcher = new FileSystemWatcher((string)Consts.LocalAppDataPath); + AppLocalEvents = Observable.Merge( + Observable.FromEventPattern(h => watcher.Changed += h, h => watcher.Changed -= h).Select(e => (FileEventType.Changed, e.EventArgs)), + Observable.FromEventPattern(h => watcher.Created += h, h => watcher.Created -= h).Select(e => (FileEventType.Created, e.EventArgs)), + Observable.FromEventPattern(h => watcher.Deleted += h, h => watcher.Deleted -= h).Select(e => (FileEventType.Deleted, e.EventArgs))) + .ObserveOn(Scheduler.Default); + + watcher.EnableRaisingEvents = true; + } + } +} diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs index 9ece4270..4ad33f64 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs @@ -137,15 +137,16 @@ namespace Wabbajack.Lib.Downloaders public override async Task Download(Archive a, AbsolutePath destination) { - using var stream = await ResolveDownloadStream(a); - if (stream == null) return false; + var (isValid, istream) = await ResolveDownloadStream(a, false); + if (!isValid) return false; + using var stream = istream!; await using var fromStream = await stream.Content.ReadAsStreamAsync(); await using var toStream = await destination.Create(); await fromStream.CopyToAsync(toStream); return true; } - private async Task ResolveDownloadStream(Archive a) + private async Task<(bool, HttpResponseMessage?)> ResolveDownloadStream(Archive a, bool quickMode) { TOP: string url; @@ -168,7 +169,7 @@ namespace Wabbajack.Lib.Downloaders if (csrfKey == null) { Utils.Log($"Returning null from IPS4 Downloader because no csrfKey was found"); - return null; + return (false, null); } var sep = Site.EndsWith("?") ? "&" : "?"; @@ -199,10 +200,10 @@ namespace Wabbajack.Lib.Downloaders if (a.Size != 0 && headerContentSize != 0 && a.Size != headerContentSize) { Utils.Log($"Bad Header Content sizes {a.Size} vs {headerContentSize}"); - return null; + return (false, null); } - return streamResult; + return (true, streamResult); } // Sometimes LL hands back a json object telling us to wait until a certain time @@ -210,6 +211,7 @@ namespace Wabbajack.Lib.Downloaders var secs = times.Download - times.CurrentTime; for (int x = 0; x < secs; x++) { + if (quickMode) return (true, default); Utils.Status($"Waiting for {secs} at the request of {Downloader.SiteName}", Percent.FactoryPutInRange(x, secs)); await Task.Delay(1000); } @@ -228,7 +230,8 @@ namespace Wabbajack.Lib.Downloaders public override async Task Verify(Archive a) { - var stream = await ResolveDownloadStream(a); + var (isValid, stream) = await ResolveDownloadStream(a, true); + if (!isValid) return false; if (stream == null) return false; diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index f01bcdf8..ce110f8a 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -104,8 +104,9 @@ namespace Wabbajack.Lib.Downloaders { if (await Download(archive, destination)) { - await destination.FileHashCachedAsync(); - return DownloadResult.Success; + var downloadedHash = await destination.FileHashCachedAsync(); + if (downloadedHash == archive.Hash || archive.Hash == default) + return DownloadResult.Success; } diff --git a/Wabbajack.Server/Program.cs b/Wabbajack.Server/Program.cs index b6f5e4be..6f247716 100644 --- a/Wabbajack.Server/Program.cs +++ b/Wabbajack.Server/Program.cs @@ -12,6 +12,7 @@ namespace Wabbajack.Server { public static void Main(string[] args) { + LoggingSettings.LogToFile = true; Consts.IsServer = true; bool testMode = args.Contains("TESTMODE"); CreateHostBuilder(args, testMode).Build().Run(); diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs index ca45bc17..5ea14e0a 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs @@ -24,8 +24,21 @@ namespace Wabbajack.VirtualFileSystem Definitions.FileType.RAR_OLD, Definitions.FileType.RAR_NEW, Definitions.FileType._7Z); - + private static Extension OMODExtension = new Extension(".omod"); + private static Extension BSAExtension = new Extension(".bsa"); + + public static readonly HashSet ExtractableExtensions = new HashSet + { + new Extension(".bsa"), + new Extension(".ba2"), + new Extension(".7z"), + new Extension(".7zip"), + new Extension(".rar"), + new Extension(".zip"), + OMODExtension + }; + /// /// When true, will allow 7z to use multiple threads and cache more data in memory, potentially @@ -64,11 +77,16 @@ namespace Wabbajack.VirtualFileSystem } } - case Definitions.FileType.TES3: case Definitions.FileType.BSA: case Definitions.FileType.BA2: return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn); - + + case Definitions.FileType.TES3: + if (sFn.Name.FileName.Extension == BSAExtension) + return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn); + else + throw new Exception($"Invalid file format {sFn.Name}"); + default: throw new Exception($"Invalid file format {sFn.Name}"); diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 32d0a6fc..ef852831 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -175,16 +175,27 @@ namespace Wabbajack.VirtualFileSystem public static async Task Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile, IPath relPath, int depth = 0) { - await using var stream = await extractedFile.GetStream(); - var hash = await stream.xxHashAsync(); - stream.Position = 0; + Hash hash = default; + if (extractedFile is NativeFileStreamFactory) + { + hash = await ((AbsolutePath)extractedFile.Name).FileHashCachedAsync(); + } + else + { + await using var hstream = await extractedFile.GetStream(); + hash = await hstream.xxHashAsync(); + } + if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself)) + { + return vself; + } + + + await using var stream = await extractedFile.GetStream(); var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream); stream.Position = 0; - if (sig.HasValue && TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself)) - return vself; - var self = new VirtualFile { Context = context, @@ -202,7 +213,7 @@ namespace Wabbajack.VirtualFileSystem self.ExtendedHashes = await ExtendedHashes.FromStream(stream); // Can't extract, so return - if (!sig.HasValue) return self; + if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension)) return self; try { diff --git a/Wabbajack/App.xaml.cs b/Wabbajack/App.xaml.cs index 7d8a7492..729690ce 100644 --- a/Wabbajack/App.xaml.cs +++ b/Wabbajack/App.xaml.cs @@ -11,6 +11,8 @@ namespace Wabbajack { public App() { + LoggingSettings.LogToFile = true; + CLIOld.ParseOptions(Environment.GetCommandLineArgs()); if (CLIArguments.Help) CLIOld.DisplayHelpText(); diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2ee42cd0..3550874e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -65,12 +65,12 @@ steps: - task: DownloadSecureFile@1 inputs: - secureFile: 'CertFile.p12' + secureFile: 'CodeCert2020.pfx' - task: codesigning@2 displayName: "Sign main app" condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') inputs: - secureFileId: 'CertFile.p12' + secureFileId: 'CodeCert2020.pfx' signCertPassword: '$(CertPassword)' files: '$(System.DefaultWorkingDirectory)/PublishApp/*abbajack*.exe' hashingAlgorithm: 'SHA256' @@ -80,7 +80,7 @@ steps: displayName: "Sign launcher" condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') inputs: - secureFileId: 'CertFile.p12' + secureFileId: 'CodeCert2020.pfx' signCertPassword: '$(CertPassword)' files: '$(System.DefaultWorkingDirectory)/PublishLauncher/*abbajack*.exe' hashingAlgorithm: 'SHA256'