Several fixes that came up during testing.

* Fixed hashing progress bars
* Wired up the CPUStatus to the UI during installation
* Fixed xxHashing
* Verified modlist optimization works on a real modlist (noise)
* Added an optimizer test for extra files created by the user left in the install directory.

CHANGELOG.md

@@ -1,5 +1,12 @@
 ### Changelog
 
+#### Version - Next
+* Optimized install process, if you install on a directory that already contains an install
+  the minimal amount of work will be done to update the install, instead of doing a complete
+  from-scratch install
+* Vortex Support for some non-Bethesda games.
+* Reworked several internal systems (VFS and workqueues) for better reliability and stability
+
 #### Version 1.0 beta 1 - 11/6/2019
 * New Installation GUI
 * Files are now moved during installation instead of copied

Wabbajack.Common/Consts.cs

@@ -81,6 +81,8 @@ namespace Wabbajack.Common
             }
         }
 
+        public static string HashFileExtension => ".xxHash";
+
         public static string WabbajackCacheLocation = "http://build.wabbajack.org/nexus_api_cache/";
     }
 }

Wabbajack.Common/StatusFileStream.cs

@@ -0,0 +1,66 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.Common
+{
+    public class StatusFileStream : Stream
+    {
+        private string _message;
+        private FileStream _inner;
+
+        public StatusFileStream(FileStream fs, string message)
+        {
+            _inner = fs;
+            _message = message;
+        }
+
+        public override void Flush()
+        {
+            _inner.Flush();
+        }
+
+        public override long Seek(long offset, SeekOrigin origin)
+        {
+            return _inner.Seek(offset, origin);
+        }
+
+        public override void SetLength(long value)
+        {
+            _inner.SetLength(value);
+        }
+
+        public override int Read(byte[] buffer, int offset, int count)
+        {
+            UpdateStatus();
+            return _inner.Read(buffer, offset, count);
+        }
+
+        private void UpdateStatus()
+        {
+            if (_inner.Length != 0)
+                Utils.Status(_message, (int) (_inner.Position * 100 / _inner.Length));
+        }
+
+        public override void Write(byte[] buffer, int offset, int count)
+        {
+            UpdateStatus();
+            _inner.Write(buffer, offset, count);
+        }
+
+        public override bool CanRead => _inner.CanRead;
+        public override bool CanSeek => _inner.CanSeek;
+        public override bool CanWrite => _inner.CanWrite;
+        public override long Length => _inner.Length;
+
+        public override long Position
+        {
+            get => _inner.Position;
+            set => _inner.Position = value;
+        }
+    }
+}

Wabbajack.Common/Utils.cs

@@ -78,7 +78,10 @@ namespace Wabbajack.Common
 
         public static void Status(string msg, int progress = 0)
         {
-            _statusSubj.OnNext((msg, progress));
+            if (WorkQueue.CurrentQueue != null)
+                WorkQueue.CurrentQueue.Report(msg, progress);
+            else
+                _statusSubj.OnNext((msg, progress));
         }
 
         /// <summary>
@@ -111,8 +114,11 @@ namespace Wabbajack.Common
                 {
                     var config = new xxHashConfig();
                     config.HashSizeInBits = 64;
-                    var value = xxHashFactory.Instance.Create(config).ComputeHash(fs);
-                    return value.AsBase64String();
+                    using (var f = new StatusFileStream(fs, $"Hashing {Path.GetFileName(file)}"))    
+                    {
+                        var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
+                        return value.AsBase64String();
+                    }
                 }
             }
             catch (IOException ex)
@@ -122,6 +128,19 @@ namespace Wabbajack.Common
             }
         }
 
+        public static string FileHashCached(this string file)
+        {
+            var hashPath = file + Consts.HashFileExtension;
+            if (File.Exists(hashPath) && File.GetLastWriteTime(file) <= File.GetLastWriteTime(hashPath))
+            {
+                return File.ReadAllText(hashPath);
+            }
+
+            var hash = file.FileHash();
+            File.WriteAllText(hashPath, hash);
+            return hash;
+        }
+
         public static async Task<string> FileHashAsync(this string file, bool nullOnIOError = false)
         {
             try
@@ -729,4 +748,4 @@ namespace Wabbajack.Common
             return ErrorResponse.Success;
         }
     }
-}
+}

Wabbajack.Common/Wabbajack.Common.csproj

@@ -104,6 +104,7 @@
     <Compile Include="GOGHandler.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="SplittingStream.cs" />
+    <Compile Include="StatusFileStream.cs" />
     <Compile Include="StatusUpdate.cs" />
     <Compile Include="SteamHandler.cs" />
     <Compile Include="Utils.cs" />

Wabbajack.Lib/AInstaller.cs

@@ -1,14 +1,18 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Eventing.Reader;
 using System.IO;
 using System.IO.Compression;
 using System.Linq;
 using System.Windows.Navigation;
+using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.VirtualFileSystem;
 using Context = Wabbajack.VirtualFileSystem.Context;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
+using File = System.IO.File;
+using FileInfo = System.IO.FileInfo;
 using Path = Alphaleonis.Win32.Filesystem.Path;
 
 namespace Wabbajack.Lib
@@ -256,7 +260,10 @@ namespace Wabbajack.Lib
         {
             try
             {
-                archive.State.Download(archive, Path.Combine(DownloadFolder, archive.Name));
+                var path = Path.Combine(DownloadFolder, archive.Name);
+                archive.State.Download(archive, path);
+                path.FileHashCached();
+
             }
             catch (Exception ex)
             {
@@ -271,25 +278,14 @@ namespace Wabbajack.Lib
         public void HashArchives()
         {
             HashedArchives = Directory.EnumerateFiles(DownloadFolder)
-                .Where(e => !e.EndsWith(".sha"))
-                .PMap(Queue, e => (HashArchive(e), e))
+                .Where(e => !e.EndsWith(Consts.HashFileExtension))
+                .PMap(Queue, e => (e.FileHashCached(), e))
                 .OrderByDescending(e => File.GetLastWriteTime(e.Item2))
                 .GroupBy(e => e.Item1)
                 .Select(e => e.First())
                 .ToDictionary(e => e.Item1, e => e.Item2);
         }
 
-        public string HashArchive(string e)
-        {
-            var cache = e + ".sha";
-            if (cache.FileExists() && new FileInfo(cache).LastWriteTime >= new FileInfo(e).LastWriteTime)
-                return File.ReadAllText(cache);
-
-            Status($"Hashing {Path.GetFileName(e)}");
-            File.WriteAllText(cache, e.FileHash());
-            return HashArchive(e);
-        }
-
         /// <summary>
         /// The user may already have some files in the OutputFolder. If so we can go through these and
         /// figure out which need to be updated, deleted, or left alone
@@ -299,11 +295,26 @@ namespace Wabbajack.Lib
             Utils.Log("Optimizing Modlist directives");
             var indexed = ModList.Directives.ToDictionary(d => d.To);
 
+            Directory.EnumerateFiles(OutputFolder, "*", DirectoryEnumerationOptions.Recursive)
+                .PMap(Queue, f =>
+                {
+                    var relative_to = f.RelativeTo(OutputFolder);
+                    Utils.Status($"Checking if modlist file {relative_to}");
+                    if (indexed.ContainsKey(relative_to) || f.StartsWith(DownloadFolder + Path.DirectorySeparator))
+                    {
+                        return;
+                    }
+
+                    Utils.Log($"Deleting {relative_to} it's not part of this modlist");
+                    File.Delete(f);
+                });
+
             indexed.Values.PMap(Queue, d =>
             {
                 // Bit backwards, but we want to return null for 
                 // all files we *want* installed. We return the files
                 // to remove from the install list.
+                Status($"Optimizing {d.To}");
                 var path = Path.Combine(OutputFolder, d.To);
                 if (!File.Exists(path)) return null;
 

Wabbajack.Test/SanityTests.cs

@@ -85,6 +85,9 @@ namespace Wabbajack.Test
             var deleted_path = utils.PathOfInstalledFile(mod, @"Data\scripts\deleted.pex");
             var modified_path = utils.PathOfInstalledFile(mod, @"Data\scripts\modified.pex");
 
+            var extra_path = utils.PathOfInstalledFile(mod, @"something_i_made.foo");
+            File.WriteAllText(extra_path, "bleh");
+
 
             var unchanged_modified = File.GetLastWriteTime(unchanged_path);
             var modified_modified = File.GetLastWriteTime(modified_path);
@@ -92,6 +95,8 @@ namespace Wabbajack.Test
             File.WriteAllText(modified_path, "random data");
             File.Delete(deleted_path);
 
+            Assert.IsTrue(File.Exists(extra_path));
+
             CompileAndInstall(profile);
 
             utils.VerifyInstalledFile(mod, @"Data\scripts\unchanged.pex");
@@ -100,8 +105,7 @@ namespace Wabbajack.Test
 
             Assert.AreEqual(unchanged_modified, File.GetLastWriteTime(unchanged_path));
             Assert.AreNotEqual(modified_modified, File.GetLastWriteTime(modified_path));
-
-
+            Assert.IsFalse(File.Exists(extra_path));
         }
 
 

Wabbajack/View Models/InstallerVM.cs

@@ -1,4 +1,4 @@
-using Syroot.Windows.IO;
+using Syroot.Windows.IO;
 using System;
 using ReactiveUI;
 using System.Diagnostics;
@@ -16,6 +16,8 @@ using Wabbajack.Common;
 using Wabbajack.Lib;
 using ReactiveUI.Fody.Helpers;
 using System.Windows.Media;
+using DynamicData;
+using DynamicData.Binding;
 
 namespace Wabbajack
 {
@@ -296,6 +298,19 @@ namespace Wabbajack
             {
                 DownloadFolder = DownloadLocation.TargetPath
             };
+
+            // Compile progress updates and populate ObservableCollection
+            installer.QueueStatus
+                .ObserveOn(RxApp.TaskpoolScheduler)
+                .ToObservableChangeSet(x => x.ID)
+                .Batch(TimeSpan.FromMilliseconds(250), RxApp.TaskpoolScheduler)
+                .EnsureUniqueChanges()
+                .ObserveOn(RxApp.MainThreadScheduler)
+                .Sort(SortExpressionComparer<CPUStatus>.Ascending(s => s.ID), SortOptimisations.ComparesImmutableValuesOnly)
+                .Bind(this.MWVM.StatusList)
+                .Subscribe()
+                .DisposeWith(this.CompositeDisposable);
+
             Task.Run(async () =>
             {
                 try
@@ -317,4 +332,4 @@ namespace Wabbajack
             });
         }
     }
-}
+}