diff --git a/CHANGELOG.md b/CHANGELOG.md index fc17c291..9718ffb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ ### Changelog +#### Version - Next +* Optimized install process, if you install on a directory that already contains an install + the minimal amount of work will be done to update the install, instead of doing a complete + from-scratch install +* Vortex Support for some non-Bethesda games. +* Reworked several internal systems (VFS and workqueues) for better reliability and stability + #### Version 1.0 beta 1 - 11/6/2019 * New Installation GUI * Files are now moved during installation instead of copied diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 40f14348..c7a16b0b 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -81,6 +81,8 @@ namespace Wabbajack.Common } } + public static string HashFileExtension => ".xxHash"; + public static string WabbajackCacheLocation = "http://build.wabbajack.org/nexus_api_cache/"; } } diff --git a/Wabbajack.Common/StatusFileStream.cs b/Wabbajack.Common/StatusFileStream.cs new file mode 100644 index 00000000..c1ba32a8 --- /dev/null +++ b/Wabbajack.Common/StatusFileStream.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.Common +{ + public class StatusFileStream : Stream + { + private string _message; + private FileStream _inner; + + public StatusFileStream(FileStream fs, string message) + { + _inner = fs; + _message = message; + } + + public override void Flush() + { + _inner.Flush(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + return _inner.Seek(offset, origin); + } + + public override void SetLength(long value) + { + _inner.SetLength(value); + } + + public override int Read(byte[] buffer, int offset, int count) + { + UpdateStatus(); + return _inner.Read(buffer, offset, count); + } + + private void UpdateStatus() + { + if (_inner.Length != 0) + Utils.Status(_message, (int) (_inner.Position * 100 / _inner.Length)); + } + + public override void Write(byte[] buffer, int offset, int count) + { + UpdateStatus(); + _inner.Write(buffer, offset, count); + } + + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => _inner.CanWrite; + public override long Length => _inner.Length; + + public override long Position + { + get => _inner.Position; + set => _inner.Position = value; + } + } +} diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 5b84df67..c2d839c4 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -78,7 +78,10 @@ namespace Wabbajack.Common public static void Status(string msg, int progress = 0) { - _statusSubj.OnNext((msg, progress)); + if (WorkQueue.CurrentQueue != null) + WorkQueue.CurrentQueue.Report(msg, progress); + else + _statusSubj.OnNext((msg, progress)); } /// @@ -111,8 +114,11 @@ namespace Wabbajack.Common { var config = new xxHashConfig(); config.HashSizeInBits = 64; - var value = xxHashFactory.Instance.Create(config).ComputeHash(fs); - return value.AsBase64String(); + using (var f = new StatusFileStream(fs, $"Hashing {Path.GetFileName(file)}")) + { + var value = xxHashFactory.Instance.Create(config).ComputeHash(f); + return value.AsBase64String(); + } } } catch (IOException ex) @@ -122,6 +128,19 @@ namespace Wabbajack.Common } } + public static string FileHashCached(this string file) + { + var hashPath = file + Consts.HashFileExtension; + if (File.Exists(hashPath) && File.GetLastWriteTime(file) <= File.GetLastWriteTime(hashPath)) + { + return File.ReadAllText(hashPath); + } + + var hash = file.FileHash(); + File.WriteAllText(hashPath, hash); + return hash; + } + public static async Task FileHashAsync(this string file, bool nullOnIOError = false) { try @@ -729,4 +748,4 @@ namespace Wabbajack.Common return ErrorResponse.Success; } } -} \ No newline at end of file +} diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 2babb989..d614f9de 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -104,6 +104,7 @@ + diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 6c8d7512..2c775784 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -1,14 +1,18 @@ using System; using System.Collections.Generic; +using System.Diagnostics.Eventing.Reader; using System.IO; using System.IO.Compression; using System.Linq; using System.Windows.Navigation; +using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; using Wabbajack.VirtualFileSystem; using Context = Wabbajack.VirtualFileSystem.Context; using Directory = Alphaleonis.Win32.Filesystem.Directory; +using File = System.IO.File; +using FileInfo = System.IO.FileInfo; using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib @@ -256,7 +260,10 @@ namespace Wabbajack.Lib { try { - archive.State.Download(archive, Path.Combine(DownloadFolder, archive.Name)); + var path = Path.Combine(DownloadFolder, archive.Name); + archive.State.Download(archive, path); + path.FileHashCached(); + } catch (Exception ex) { @@ -271,25 +278,14 @@ namespace Wabbajack.Lib public void HashArchives() { HashedArchives = Directory.EnumerateFiles(DownloadFolder) - .Where(e => !e.EndsWith(".sha")) - .PMap(Queue, e => (HashArchive(e), e)) + .Where(e => !e.EndsWith(Consts.HashFileExtension)) + .PMap(Queue, e => (e.FileHashCached(), e)) .OrderByDescending(e => File.GetLastWriteTime(e.Item2)) .GroupBy(e => e.Item1) .Select(e => e.First()) .ToDictionary(e => e.Item1, e => e.Item2); } - public string HashArchive(string e) - { - var cache = e + ".sha"; - if (cache.FileExists() && new FileInfo(cache).LastWriteTime >= new FileInfo(e).LastWriteTime) - return File.ReadAllText(cache); - - Status($"Hashing {Path.GetFileName(e)}"); - File.WriteAllText(cache, e.FileHash()); - return HashArchive(e); - } - /// /// The user may already have some files in the OutputFolder. If so we can go through these and /// figure out which need to be updated, deleted, or left alone @@ -299,11 +295,26 @@ namespace Wabbajack.Lib Utils.Log("Optimizing Modlist directives"); var indexed = ModList.Directives.ToDictionary(d => d.To); + Directory.EnumerateFiles(OutputFolder, "*", DirectoryEnumerationOptions.Recursive) + .PMap(Queue, f => + { + var relative_to = f.RelativeTo(OutputFolder); + Utils.Status($"Checking if modlist file {relative_to}"); + if (indexed.ContainsKey(relative_to) || f.StartsWith(DownloadFolder + Path.DirectorySeparator)) + { + return; + } + + Utils.Log($"Deleting {relative_to} it's not part of this modlist"); + File.Delete(f); + }); + indexed.Values.PMap(Queue, d => { // Bit backwards, but we want to return null for // all files we *want* installed. We return the files // to remove from the install list. + Status($"Optimizing {d.To}"); var path = Path.Combine(OutputFolder, d.To); if (!File.Exists(path)) return null; diff --git a/Wabbajack.Test/SanityTests.cs b/Wabbajack.Test/SanityTests.cs index 22f2485a..067a774b 100644 --- a/Wabbajack.Test/SanityTests.cs +++ b/Wabbajack.Test/SanityTests.cs @@ -85,6 +85,9 @@ namespace Wabbajack.Test var deleted_path = utils.PathOfInstalledFile(mod, @"Data\scripts\deleted.pex"); var modified_path = utils.PathOfInstalledFile(mod, @"Data\scripts\modified.pex"); + var extra_path = utils.PathOfInstalledFile(mod, @"something_i_made.foo"); + File.WriteAllText(extra_path, "bleh"); + var unchanged_modified = File.GetLastWriteTime(unchanged_path); var modified_modified = File.GetLastWriteTime(modified_path); @@ -92,6 +95,8 @@ namespace Wabbajack.Test File.WriteAllText(modified_path, "random data"); File.Delete(deleted_path); + Assert.IsTrue(File.Exists(extra_path)); + CompileAndInstall(profile); utils.VerifyInstalledFile(mod, @"Data\scripts\unchanged.pex"); @@ -100,8 +105,7 @@ namespace Wabbajack.Test Assert.AreEqual(unchanged_modified, File.GetLastWriteTime(unchanged_path)); Assert.AreNotEqual(modified_modified, File.GetLastWriteTime(modified_path)); - - + Assert.IsFalse(File.Exists(extra_path)); } diff --git a/Wabbajack/View Models/InstallerVM.cs b/Wabbajack/View Models/InstallerVM.cs index 6fe0b8c4..b96e2c58 100644 --- a/Wabbajack/View Models/InstallerVM.cs +++ b/Wabbajack/View Models/InstallerVM.cs @@ -1,4 +1,4 @@ -using Syroot.Windows.IO; +using Syroot.Windows.IO; using System; using ReactiveUI; using System.Diagnostics; @@ -16,6 +16,8 @@ using Wabbajack.Common; using Wabbajack.Lib; using ReactiveUI.Fody.Helpers; using System.Windows.Media; +using DynamicData; +using DynamicData.Binding; namespace Wabbajack { @@ -296,6 +298,19 @@ namespace Wabbajack { DownloadFolder = DownloadLocation.TargetPath }; + + // Compile progress updates and populate ObservableCollection + installer.QueueStatus + .ObserveOn(RxApp.TaskpoolScheduler) + .ToObservableChangeSet(x => x.ID) + .Batch(TimeSpan.FromMilliseconds(250), RxApp.TaskpoolScheduler) + .EnsureUniqueChanges() + .ObserveOn(RxApp.MainThreadScheduler) + .Sort(SortExpressionComparer.Ascending(s => s.ID), SortOptimisations.ComparesImmutableValuesOnly) + .Bind(this.MWVM.StatusList) + .Subscribe() + .DisposeWith(this.CompositeDisposable); + Task.Run(async () => { try @@ -317,4 +332,4 @@ namespace Wabbajack }); } } -} \ No newline at end of file +}