Merge remote-tracking branch 'origin/extraction-fixes' into extraction-fixes

CHANGELOG.md

@@ -15,6 +15,7 @@ install thread.
 * Fixed Extraction so that zip files no longer cause WJ to CTD
 * Better path logging during install and compilation
 * Fix the "this was created with a newer version of Wabbajack" issue
+* If a downloaded file doesn't match the expected hash, try alternative download locations, if allowed
 
 
 #### Version - 2.2.2.0 - 8/31/2020

Wabbajack.Common/Logging.cs

@@ -0,0 +1,171 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Subjects;
+using System.Text;
+using System.Threading.Tasks;
+using Wabbajack.Common.StatusFeed;
+using Wabbajack.Common.StatusFeed.Errors;
+using File = Alphaleonis.Win32.Filesystem.File;
+using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
+using Path = Alphaleonis.Win32.Filesystem.Path;
+
+namespace Wabbajack.Common
+{
+    public static class LoggingSettings
+    {
+        // False by default, so that library users do not have to swap it
+        public static bool LogToFile = false;
+    }
+
+    public static partial class Utils
+    {
+        public static AbsolutePath LogFile { get; private set; }
+        public static AbsolutePath LogFolder { get; private set; }
+
+        private static object _logLock = new object();
+
+        private static DateTime _startTime;
+
+        private static readonly Subject<IStatusMessage> LoggerSubj = new Subject<IStatusMessage>();
+        public static IObservable<IStatusMessage> LogMessages => LoggerSubj;
+
+        private static async Task InitalizeLogging()
+        {
+            _startTime = DateTime.Now;
+
+            if (LoggingSettings.LogToFile)
+            {
+                LogFolder = Consts.LogsFolder;
+                LogFile = Consts.LogFile;
+                Consts.LocalAppDataPath.CreateDirectory();
+                Consts.LogsFolder.CreateDirectory();
+
+                if (LogFile.Exists)
+                {
+                    var newPath = Consts.LogsFolder.Combine(Consts.EntryPoint.FileNameWithoutExtension + LogFile.LastModified.ToString(" yyyy-MM-dd HH_mm_ss") + ".log");
+                    await LogFile.MoveToAsync(newPath, true);
+                }
+
+                var logFiles = LogFolder.EnumerateFiles(false).ToList();
+                if (logFiles.Count >= Consts.MaxOldLogs)
+                {
+                    Log($"Maximum amount of old logs reached ({logFiles.Count} >= {Consts.MaxOldLogs})");
+                    var filesToDelete = logFiles
+                        .Where(f => f.IsFile)
+                        .OrderBy(f => f.LastModified)
+                        .Take(logFiles.Count - Consts.MaxOldLogs)
+                        .ToList();
+
+                    Log($"Found {filesToDelete.Count} old log files to delete");
+
+                    var success = 0;
+                    var failed = 0;
+                    filesToDelete.Do(f =>
+                    {
+                        try
+                        {
+                            f.Delete();
+                            success++;
+                        }
+                        catch (Exception e)
+                        {
+                            failed++;
+                            Log($"Could not delete log at {f}!\n{e}");
+                        }
+                    });
+
+                    Log($"Deleted {success} log files, failed to delete {failed} logs");
+                }
+            }
+        }
+
+
+        public static void Log(string msg)
+        {
+            Log(new GenericInfo(msg));
+        }
+
+        public static T Log<T>(T msg) where T : IStatusMessage
+        {
+            LogStraightToFile(string.IsNullOrWhiteSpace(msg.ExtendedDescription) ? msg.ShortDescription : msg.ExtendedDescription);
+            LoggerSubj.OnNext(msg);
+            return msg;
+        }
+
+        public static void Error(string errMessage)
+        {
+            Log(errMessage);
+        }
+
+        public static void Error(Exception ex, string? extraMessage = null)
+        {
+            Log(new GenericException(ex, extraMessage));
+        }
+
+        public static void ErrorThrow(Exception ex, string? extraMessage = null)
+        {
+            Error(ex, extraMessage);
+            throw ex;
+        }
+
+        public static void Error(IException err)
+        {
+            LogStraightToFile($"{err.ShortDescription}\n{err.Exception.StackTrace}");
+            LoggerSubj.OnNext(err);
+        }
+
+        public static void ErrorThrow(IException err)
+        {
+            Error(err);
+            throw err.Exception;
+        }
+
+        public static void LogStraightToFile(string msg)
+        {
+            if (!LoggingSettings.LogToFile || LogFile == default) return;
+            lock (_logLock)
+            {
+                File.AppendAllText(LogFile.ToString(), $"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n");
+            }
+        }
+
+        public static void Status(string msg, Percent progress, bool alsoLog = false)
+        {
+            WorkQueue.AsyncLocalCurrentQueue.Value?.Report(msg, progress);
+            if (alsoLog)
+            {
+                Utils.Log(msg);
+            }
+        }
+
+        public static void Status(string msg, bool alsoLog = false)
+        {
+            Status(msg, Percent.Zero, alsoLog: alsoLog);
+        }
+
+        public static void CatchAndLog(Action a)
+        {
+            try
+            {
+                a();
+            }
+            catch (Exception ex)
+            {
+                Utils.Error(ex);
+            }
+        }
+
+        public static async Task CatchAndLog(Func<Task> f)
+        {
+            try
+            {
+                await f();
+            }
+            catch (Exception ex)
+            {
+                Utils.Error(ex);
+            }
+        }
+    }
+}

Wabbajack.Common/Paths/FileCompaction.cs

@@ -1,4 +1,5 @@
-using System.Threading.Tasks;
+using System.Linq;
+using System.Threading.Tasks;
 using Wabbajack.Common.IO;
 
 namespace Wabbajack.Common
@@ -60,7 +61,12 @@ namespace Wabbajack.Common
 
         public static async Task CompactFolder(this AbsolutePath folder, WorkQueue queue, Algorithm algorithm)
         {
-            await folder.EnumerateFiles(true)
+            var driveInfo = folder.DriveInfo().DiskSpaceInfo;
+            var clusterSize = driveInfo.SectorsPerCluster * driveInfo.BytesPerSector;
+
+            await folder
+                .EnumerateFiles(true)
+                .Where(f => f.Size > clusterSize)
                 .PMap(queue, async path =>
                 {
                     Utils.Status($"Compacting {path.FileName}");

Wabbajack.Common/Utils.cs

@@ -1,16 +1,12 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Data.HashFunction.xxHash;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
-using System.Reactive.Concurrency;
 using System.Reactive.Linq;
-using System.Reactive.Subjects;
-using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Security;
 using System.Security.Cryptography;
@@ -23,9 +19,6 @@ using IniParser;
 using IniParser.Model.Configuration;
 using IniParser.Parser;
 using Microsoft.Win32;
-using Newtonsoft.Json;
-using Wabbajack.Common.StatusFeed;
-using Wabbajack.Common.StatusFeed.Errors;
 using YamlDotNet.Serialization;
 using YamlDotNet.Serialization.NamingConventions;
 using Directory = System.IO.Directory;
@@ -43,167 +36,13 @@ namespace Wabbajack.Common
             return processList.Where(process => process.ProcessName == "ModOrganizer").Any(process => Path.GetDirectoryName(process.MainModule?.FileName) == mo2Path);
         }
 
-        public static AbsolutePath LogFile { get; }
-        public static AbsolutePath LogFolder { get; }
-
-        public enum FileEventType
-        {
-            Created,
-            Changed,
-            Deleted
-        }
-
         static Utils()
         {
-            LogFolder = Consts.LogsFolder;
+            InitalizeLogging().Wait();
-            LogFile = Consts.LogFile;
-            Consts.LocalAppDataPath.CreateDirectory();
-            Consts.LogsFolder.CreateDirectory();
-
-            _startTime = DateTime.Now;
-
-            if (LogFile.Exists)
-            {
-                var newPath = Consts.LogsFolder.Combine(Consts.EntryPoint.FileNameWithoutExtension + LogFile.LastModified.ToString(" yyyy-MM-dd HH_mm_ss") + ".log");
-                LogFile.MoveToAsync(newPath, true).Wait();
-            }
-
-            var logFiles = LogFolder.EnumerateFiles(false).ToList();
-            if (logFiles.Count >= Consts.MaxOldLogs)
-            {
-                Log($"Maximum amount of old logs reached ({logFiles.Count} >= {Consts.MaxOldLogs})");
-                var filesToDelete = logFiles
-                    .Where(f => f.IsFile)
-                    .OrderBy(f => f.LastModified)
-                    .Take(logFiles.Count - Consts.MaxOldLogs)
-                    .ToList();
-
-                Log($"Found {filesToDelete.Count} old log files to delete");
-
-                var success = 0;
-                var failed = 0;
-                filesToDelete.Do(f =>
-                {
-                    try
-                    {
-                        f.Delete();
-                        success++;
-                    }
-                    catch (Exception e)
-                    {
-                        failed++;
-                        Log($"Could not delete log at {f}!\n{e}");
-                    }
-                });
-
-                Log($"Deleted {success} log files, failed to delete {failed} logs");
-            }
-
-            var watcher = new FileSystemWatcher((string)Consts.LocalAppDataPath);
-            AppLocalEvents = Observable.Merge(Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Changed += h, h => watcher.Changed -= h).Select(e => (FileEventType.Changed, e.EventArgs)),
-                                                Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Created += h, h => watcher.Created -= h).Select(e => (FileEventType.Created, e.EventArgs)),
-                                                Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Deleted += h, h => watcher.Deleted -= h).Select(e => (FileEventType.Deleted, e.EventArgs)))
-                                       .ObserveOn(Scheduler.Default);
-            watcher.EnableRaisingEvents = true;
         }
 
-        private static readonly Subject<IStatusMessage> LoggerSubj = new Subject<IStatusMessage>();
-        public static IObservable<IStatusMessage> LogMessages => LoggerSubj;
-
         private static readonly string[] Suffix = {"B", "KB", "MB", "GB", "TB", "PB", "EB"}; // Longs run out around EB
 
-        private static object _lock = new object();
-
-        private static DateTime _startTime;
-
-        
-        public static void Log(string msg)
-        {
-            Log(new GenericInfo(msg));
-        }
-
-        public static T Log<T>(T msg) where T : IStatusMessage
-        {
-            LogStraightToFile(string.IsNullOrWhiteSpace(msg.ExtendedDescription) ? msg.ShortDescription : msg.ExtendedDescription);
-            LoggerSubj.OnNext(msg);
-            return msg;
-        }
-
-        public static void Error(string errMessage)
-        {
-            Log(errMessage);
-        }
-
-        public static void Error(Exception ex, string? extraMessage = null)
-        {
-            Log(new GenericException(ex, extraMessage));
-        }
-
-        public static void ErrorThrow(Exception ex, string? extraMessage = null)
-        {
-            Error(ex, extraMessage);
-            throw ex;
-        }
-
-        public static void Error(IException err)
-        {
-            LogStraightToFile($"{err.ShortDescription}\n{err.Exception.StackTrace}");
-            LoggerSubj.OnNext(err);
-        }
-
-        public static void ErrorThrow(IException err)
-        {
-            Error(err);
-            throw err.Exception;
-        }
-
-        public static void LogStraightToFile(string msg)
-        {
-            if (LogFile == default) return;
-            lock (_lock)
-            {
-                File.AppendAllText(LogFile.ToString(), $"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n");
-            }
-        }
-
-        public static void Status(string msg, Percent progress, bool alsoLog = false)
-        {
-            WorkQueue.AsyncLocalCurrentQueue.Value?.Report(msg, progress);
-            if (alsoLog)
-            {
-                Utils.Log(msg);
-            }
-        }
-
-        public static void Status(string msg, bool alsoLog = false)
-        {
-            Status(msg, Percent.Zero, alsoLog: alsoLog);
-        }
-
-        public static void CatchAndLog(Action a)
-        {
-            try
-            {
-                a();
-            }
-            catch (Exception ex)
-            {
-                Utils.Error(ex);
-            }
-        }
-
-        public static async Task CatchAndLog(Func<Task> f)
-        {
-            try
-            {
-                await f();
-            }
-            catch (Exception ex)
-            {
-                Utils.Error(ex);
-            }
-        }
-
         public static void CopyToWithStatus(this Stream istream, long maxSize, Stream ostream, string status)
         {
             var buffer = new byte[1024 * 64];
@@ -993,16 +832,14 @@ namespace Wabbajack.Common
             return false;
         }
 
-
-        public static IObservable<(FileEventType, FileSystemEventArgs)> AppLocalEvents { get; }
-
         public static IObservable<bool> HaveEncryptedJsonObservable(string key)
         {
             var path = Consts.LocalAppDataPath.Combine(key);
-            return AppLocalEvents.Where(t => (AbsolutePath)t.Item2.FullPath.ToLower() == path)
+            return WJFileWatcher.AppLocalEvents
-                                 .Select(_ => path.Exists)
+                .Where(t => (AbsolutePath)t.Item2.FullPath.ToLower() == path)
-                                 .StartWith(path.Exists)
+                .Select(_ => path.Exists)
-                                 .DistinctUntilChanged();
+                .StartWith(path.Exists)
+                .DistinctUntilChanged();
         }
 
         public static async ValueTask DeleteEncryptedJson(string key)

Wabbajack.Common/WJFileWatcher.cs

@@ -0,0 +1,32 @@
+using System;
+using System.IO;
+using System.Reactive.Concurrency;
+using System.Reactive.Linq;
+using System.Text;
+
+namespace Wabbajack.Common
+{
+    public static class WJFileWatcher
+    {
+        public enum FileEventType
+        {
+            Created,
+            Changed,
+            Deleted
+        }
+
+        public static IObservable<(FileEventType, FileSystemEventArgs)> AppLocalEvents { get; }
+
+        static WJFileWatcher()
+        {
+            var watcher = new FileSystemWatcher((string)Consts.LocalAppDataPath);
+            AppLocalEvents = Observable.Merge(
+                    Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Changed += h, h => watcher.Changed -= h).Select(e => (FileEventType.Changed, e.EventArgs)),
+                    Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Created += h, h => watcher.Created -= h).Select(e => (FileEventType.Created, e.EventArgs)),
+                    Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Deleted += h, h => watcher.Deleted -= h).Select(e => (FileEventType.Deleted, e.EventArgs)))
+                .ObserveOn(Scheduler.Default);
+
+            watcher.EnableRaisingEvents = true;
+        }
+    }
+}

Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs

@@ -137,15 +137,16 @@ namespace Wabbajack.Lib.Downloaders
 
             public override async Task<bool> Download(Archive a, AbsolutePath destination)
             {
-                using var stream = await ResolveDownloadStream(a);
+                var (isValid, istream) = await ResolveDownloadStream(a, false);
-                if (stream == null) return false;
+                if (!isValid) return false;
+                using var stream = istream!;
                 await using var fromStream = await stream.Content.ReadAsStreamAsync();
                 await using var toStream = await destination.Create();
                 await fromStream.CopyToAsync(toStream);
                 return true;
             }
 
-            private async Task<HttpResponseMessage?> ResolveDownloadStream(Archive a)
+            private async Task<(bool, HttpResponseMessage?)> ResolveDownloadStream(Archive a, bool quickMode)
             {
                 TOP:
                 string url;
@@ -168,7 +169,7 @@ namespace Wabbajack.Lib.Downloaders
                     if (csrfKey == null)
                     {
                         Utils.Log($"Returning null from IPS4 Downloader because no csrfKey was found");
-                        return null;
+                        return (false, null);
                     }
 
                     var sep = Site.EndsWith("?") ? "&" : "?";
@@ -199,10 +200,10 @@ namespace Wabbajack.Lib.Downloaders
                     if (a.Size != 0 && headerContentSize != 0 && a.Size != headerContentSize)
                     {
                         Utils.Log($"Bad Header Content sizes {a.Size} vs {headerContentSize}");
-                        return null;
+                        return (false, null);
                     }
 
-                    return streamResult;
+                    return (true, streamResult);
                 }
 
                 // Sometimes LL hands back a json object telling us to wait until a certain time
@@ -210,6 +211,7 @@ namespace Wabbajack.Lib.Downloaders
                 var secs = times.Download - times.CurrentTime;
                 for (int x = 0; x < secs; x++)
                 {
+                    if (quickMode) return (true, default);
                     Utils.Status($"Waiting for {secs} at the request of {Downloader.SiteName}", Percent.FactoryPutInRange(x, secs));
                     await Task.Delay(1000);
                 }
@@ -228,7 +230,8 @@ namespace Wabbajack.Lib.Downloaders
 
             public override async Task<bool> Verify(Archive a)
             {
-                var stream = await ResolveDownloadStream(a);
+                var (isValid, stream) = await ResolveDownloadStream(a, true);
+                if (!isValid) return false;
                 if (stream == null)
                     return false;
 

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -104,8 +104,9 @@ namespace Wabbajack.Lib.Downloaders
         {
             if (await Download(archive, destination))
             {
-                await destination.FileHashCachedAsync();
+                var downloadedHash = await destination.FileHashCachedAsync();
-                return DownloadResult.Success;
+                if (downloadedHash == archive.Hash || archive.Hash == default) 
+                    return DownloadResult.Success;
             }
 
             

Wabbajack.Server/Program.cs

@@ -12,6 +12,7 @@ namespace Wabbajack.Server
     {
         public static void Main(string[] args)
         {
+            LoggingSettings.LogToFile = true;
             Consts.IsServer = true;
             bool testMode = args.Contains("TESTMODE");
             CreateHostBuilder(args, testMode).Build().Run();

Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs

@@ -24,8 +24,21 @@ namespace Wabbajack.VirtualFileSystem
             Definitions.FileType.RAR_OLD,
             Definitions.FileType.RAR_NEW,
             Definitions.FileType._7Z);
-        
+
         private static Extension OMODExtension = new Extension(".omod");
+        private static Extension BSAExtension = new Extension(".bsa");
+
+        public static readonly HashSet<Extension> ExtractableExtensions = new HashSet<Extension>
+        {
+            new Extension(".bsa"),
+            new Extension(".ba2"),
+            new Extension(".7z"),
+            new Extension(".7zip"),
+            new Extension(".rar"),
+            new Extension(".zip"),
+            OMODExtension
+        };
+        
         
         /// <summary>
         /// When true, will allow 7z to use multiple threads and cache more data in memory, potentially
@@ -64,11 +77,16 @@ namespace Wabbajack.VirtualFileSystem
                     }
                 }
 
-                case Definitions.FileType.TES3:
                 case Definitions.FileType.BSA:
                 case Definitions.FileType.BA2:
                     return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
-                
+
+                case Definitions.FileType.TES3:
+                    if (sFn.Name.FileName.Extension == BSAExtension) 
+                        return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
+                    else
+                        throw new Exception($"Invalid file format {sFn.Name}");
+
 
                 default:
                     throw new Exception($"Invalid file format {sFn.Name}");

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -175,16 +175,27 @@ namespace Wabbajack.VirtualFileSystem
         public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile,
             IPath relPath, int depth = 0)
         {
-            await using var stream = await extractedFile.GetStream();
+            Hash hash = default;
-            var hash = await stream.xxHashAsync();
+            if (extractedFile is NativeFileStreamFactory)
-            stream.Position = 0;
+            {
+                hash = await ((AbsolutePath)extractedFile.Name).FileHashCachedAsync();
+            } 
+            else
+            {
+                await using var hstream = await extractedFile.GetStream();
+                hash = await hstream.xxHashAsync();
+            }
 
+            if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
+            {
+                return vself;
+            }
+
+            
+            await using var stream = await extractedFile.GetStream();
             var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);
             stream.Position = 0;
             
-            if (sig.HasValue && TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
-                return vself;
-            
             var self = new VirtualFile
             {
                 Context = context,
@@ -202,7 +213,7 @@ namespace Wabbajack.VirtualFileSystem
                 self.ExtendedHashes = await ExtendedHashes.FromStream(stream);
 
             // Can't extract, so return
-            if (!sig.HasValue) return self;
+            if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension)) return self;
 
             try
             {

Wabbajack/App.xaml.cs

@@ -11,6 +11,8 @@ namespace Wabbajack
     {
         public App()
         {
+            LoggingSettings.LogToFile = true;
+
             CLIOld.ParseOptions(Environment.GetCommandLineArgs());
             if (CLIArguments.Help)
                 CLIOld.DisplayHelpText();

azure-pipelines.yml

@@ -65,12 +65,12 @@ steps:
 
 - task: DownloadSecureFile@1
   inputs:
-    secureFile: 'CertFile.p12'
+    secureFile: 'CodeCert2020.pfx'
 - task: codesigning@2
   displayName: "Sign main app"
   condition: eq(variables['Build.SourceBranch'], 'refs/heads/master')
   inputs:
-    secureFileId: 'CertFile.p12'
+    secureFileId: 'CodeCert2020.pfx'
     signCertPassword: '$(CertPassword)'
     files: '$(System.DefaultWorkingDirectory)/PublishApp/*abbajack*.exe'
     hashingAlgorithm: 'SHA256'
@@ -80,7 +80,7 @@ steps:
   displayName: "Sign launcher"
   condition: eq(variables['Build.SourceBranch'], 'refs/heads/master')
   inputs:
-    secureFileId: 'CertFile.p12'
+    secureFileId: 'CodeCert2020.pfx'
     signCertPassword: '$(CertPassword)'
     files: '$(System.DefaultWorkingDirectory)/PublishLauncher/*abbajack*.exe'
     hashingAlgorithm: 'SHA256'