From 86107ec8ecaebcd08920797a43e923e3461ae0a4 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 5 Aug 2020 16:01:45 -0600 Subject: [PATCH 1/2] Enable optional disk compression --- .../Compression.BSA.Test.csproj | 2 +- Wabbajack.App.Test/Wabbajack.App.Test.csproj | 4 +- .../Wabbajack.Common.Test.csproj | 2 +- Wabbajack.Common/Paths.cs | 698 +----------------- Wabbajack.Common/Paths/AbsolutePath.cs | 421 +++++++++++ Wabbajack.Common/Paths/FileCompaction.cs | 61 ++ Wabbajack.Common/Paths/IPath.cs | 12 + Wabbajack.Common/Paths/RelativePath.cs | 159 ++++ Wabbajack.Common/Wabbajack.Common.csproj | 2 +- Wabbajack.Lib/AInstaller.cs | 29 +- Wabbajack.Lib/MO2Installer.cs | 6 + Wabbajack.Lib/Wabbajack.Lib.csproj | 6 +- .../Wabbajack.Server.Test.csproj | 2 +- Wabbajack.Test/EndToEndTests.cs | 5 +- Wabbajack.Test/Wabbajack.Test.csproj | 2 +- .../Wabbajack.VirtualFileSystem.Test.csproj | 2 +- Wabbajack/Settings.cs | 3 + .../View Models/Installers/MO2InstallerVM.cs | 1 + .../Settings/ModlistGallerySettingsView.xaml | 23 +- .../ModlistGallerySettingsView.xaml.cs | 2 + Wabbajack/Wabbajack.csproj | 16 +- 21 files changed, 724 insertions(+), 734 deletions(-) create mode 100644 Wabbajack.Common/Paths/AbsolutePath.cs create mode 100644 Wabbajack.Common/Paths/FileCompaction.cs create mode 100644 Wabbajack.Common/Paths/IPath.cs create mode 100644 Wabbajack.Common/Paths/RelativePath.cs diff --git a/Compression.BSA.Test/Compression.BSA.Test.csproj b/Compression.BSA.Test/Compression.BSA.Test.csproj index 4bdf57dc..0142bc9e 100644 --- a/Compression.BSA.Test/Compression.BSA.Test.csproj +++ b/Compression.BSA.Test/Compression.BSA.Test.csproj @@ -11,7 +11,7 @@ - + diff --git a/Wabbajack.App.Test/Wabbajack.App.Test.csproj b/Wabbajack.App.Test/Wabbajack.App.Test.csproj index d151d8c8..ac0eaf1b 100644 --- a/Wabbajack.App.Test/Wabbajack.App.Test.csproj +++ b/Wabbajack.App.Test/Wabbajack.App.Test.csproj @@ -10,8 +10,8 @@ - - + + diff --git a/Wabbajack.Common.Test/Wabbajack.Common.Test.csproj b/Wabbajack.Common.Test/Wabbajack.Common.Test.csproj index 248e9d23..cf7de0f4 100644 --- a/Wabbajack.Common.Test/Wabbajack.Common.Test.csproj +++ b/Wabbajack.Common.Test/Wabbajack.Common.Test.csproj @@ -11,7 +11,7 @@ - + diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index d457c595..dcc9ba05 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -19,566 +19,8 @@ using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Common { - public interface IPath - { - /// - /// Get the final file name, for c:\bar\baz this is `baz` for c:\bar.zip this is `bar.zip` - /// for `bar.zip` this is `bar.zip` - /// - public RelativePath FileName { get; } - } - public struct AbsolutePath : IPath, IComparable, IEquatable - { - #region ObjectEquality - - public bool Equals(AbsolutePath other) - { - return string.Equals(_path, other._path, StringComparison.InvariantCultureIgnoreCase); - } - - public override bool Equals(object? obj) - { - return obj is AbsolutePath other && Equals(other); - } - - #endregion - - public override int GetHashCode() - { - return _path.GetHashCode(StringComparison.InvariantCultureIgnoreCase); - } - - public override string ToString() - { - return _path; - } - - private readonly string _nullable_path; - private string _path => _nullable_path ?? string.Empty; - - public AbsolutePath(string path, bool skipValidation = false) - { - _nullable_path = path.Replace("/", "\\").TrimEnd('\\'); - if (!skipValidation) - { - ValidateAbsolutePath(); - } - } - - public AbsolutePath(AbsolutePath path) - { - _nullable_path = path._path; - } - - private void ValidateAbsolutePath() - { - if (Path.IsPathRooted(_path)) - { - return; - } - - throw new InvalidDataException($"Absolute path must be absolute, got {_path}"); - } - - public string Normalize() - { - return _path.Replace("/", "\\").TrimEnd('\\'); - } - - public DriveInfo DriveInfo() - { - return new DriveInfo(Path.GetPathRoot(_path)); - } - - public Extension Extension => Extension.FromPath(_path); - - public ValueTask OpenRead() - { - return OpenShared(); - } - - public ValueTask Create() - { - var path = _path; - return CircuitBreaker.WithAutoRetryAsync(async () => File.Open(path, FileMode.Create, FileAccess.ReadWrite)); - } - - public ValueTask OpenWrite() - { - var path = _path; - return CircuitBreaker.WithAutoRetryAsync(async () => File.OpenWrite(path)); - } - - public async Task WriteAllTextAsync(string text) - { - await using var fs = File.Create(_path); - await fs.WriteAsync(Encoding.UTF8.GetBytes(text)); - } - - public bool Exists => File.Exists(_path) || Directory.Exists(_path); - public bool IsFile => File.Exists(_path); - public bool IsDirectory => Directory.Exists(_path); - - public async Task DeleteDirectory(bool dontDeleteIfNotEmpty = false) - { - if (IsDirectory) - { - if (dontDeleteIfNotEmpty && (EnumerateFiles().Any() || EnumerateDirectories().Any())) return; - await Utils.DeleteDirectory(this); - } - } - - public long Size => Exists ? new FileInfo(_path).Length : 0; - - public DateTime LastModified - { - get => File.GetLastWriteTime(_path); - set => File.SetLastWriteTime(_path, value); - } - - public DateTime LastModifiedUtc => File.GetLastWriteTimeUtc(_path); - public AbsolutePath Parent => (AbsolutePath)Path.GetDirectoryName(_path); - public RelativePath FileName => (RelativePath)Path.GetFileName(_path); - public RelativePath FileNameWithoutExtension => (RelativePath)Path.GetFileNameWithoutExtension(_path); - public bool IsEmptyDirectory => IsDirectory && !EnumerateFiles().Any(); - - public bool IsReadOnly - { - get - { - return new FileInfo(_path).IsReadOnly; - } - set - { - new FileInfo(_path).IsReadOnly = value; - } - } - - public void SetReadOnly(bool val) - { - IsReadOnly = true; - } - - /// - /// Returns the full path the folder that contains Wabbajack.Common. This will almost always be - /// where all the binaries for the project reside. - /// - /// - public static AbsolutePath EntryPoint - { - get - { - var location = Assembly.GetExecutingAssembly().Location ?? null; - if (location == null) - throw new ArgumentException("Could not find entry point."); - return ((AbsolutePath)location).Parent; - } - } - - public AbsolutePath Root => (AbsolutePath)Path.GetPathRoot(_path); - - /// - /// Moves this file to the specified location, will use Copy if required - /// - /// - /// Replace the destination file if it exists - public async Task MoveToAsync(AbsolutePath otherPath, bool overwrite = false) - { - if (Root != otherPath.Root) - { - if (otherPath.Exists && overwrite) - await otherPath.DeleteAsync(); - - await CopyToAsync(otherPath); - await DeleteAsync(); - return; - } - - var path = _path; - await CircuitBreaker.WithAutoRetryAsync(async () => File.Move(path, otherPath._path, overwrite ? MoveOptions.ReplaceExisting : MoveOptions.None)); - } - - public RelativePath RelativeTo(AbsolutePath p) - { - var relPath = Path.GetRelativePath(p._path, _path); - if (relPath == _path) - throw new ArgumentException($"{_path} is not a subpath of {p._path}"); - return new RelativePath(relPath); - } - - - public async Task ReadAllTextAsync() - { - await using var fs = File.OpenRead(_path); - return Encoding.UTF8.GetString(await fs.ReadAllAsync()); - } - - /// - /// Assuming the path is a folder, enumerate all the files in the folder - /// - /// if true, also returns files in sub-folders - /// pattern to match against - /// - public IEnumerable EnumerateFiles(bool recursive = true, string pattern = "*") - { - if (!IsDirectory) return new AbsolutePath[0]; - return Directory - .EnumerateFiles(_path, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly) - .Select(path => new AbsolutePath(path, true)); - } - - #region Operators - - public static explicit operator string(AbsolutePath path) - { - return path._path; - } - - public static explicit operator AbsolutePath(string path) - { - if (string.IsNullOrEmpty(path)) return default; - return !Path.IsPathRooted(path) ? ((RelativePath)path).RelativeToEntryPoint() : new AbsolutePath(path); - } - - public static bool operator ==(AbsolutePath a, AbsolutePath b) - { - return a.Equals(b); - } - - public static bool operator !=(AbsolutePath a, AbsolutePath b) - { - return !a.Equals(b); - } - - #endregion - - public void CreateDirectory() - { - Directory.CreateDirectory(_path); - } - - public async Task DeleteAsync() - { - try - { - if (!IsFile) return; - - if (IsReadOnly) IsReadOnly = false; - - var path = _path; - await CircuitBreaker.WithAutoRetryAsync(async () => File.Delete(path)); - } - catch (FileNotFoundException) - { - // ignore, it doesn't exist so why delete it? - } - } - - public void Delete() - { - if (!IsFile) return; - - if (IsReadOnly) IsReadOnly = false; - - var path = _path; - CircuitBreaker.WithAutoRetry(async () => File.Delete(path)); - } - - public bool InFolder(AbsolutePath folder) - { - return _path.StartsWith(folder._path + Path.DirectorySeparator); - } - - public async Task ReadAllBytesAsync() - { - await using var f = await OpenShared(); - return await f.ReadAllAsync(); - } - - public AbsolutePath WithExtension(Extension hashFileExtension) - { - return new AbsolutePath(_path + (string)hashFileExtension, true); - } - - public AbsolutePath ReplaceExtension(Extension extension) - { - return new AbsolutePath( - Path.Combine(Path.GetDirectoryName(_path), Path.GetFileNameWithoutExtension(_path) + (string)extension), - true); - } - - public AbsolutePath AppendToName(string toAppend) - { - return new AbsolutePath( - Path.Combine(Path.GetDirectoryName(_path), - Path.GetFileNameWithoutExtension(_path) + toAppend + (string)Extension)); - } - - public AbsolutePath Combine(params RelativePath[] paths) - { - return new AbsolutePath(Path.Combine(paths.Select(s => (string)s).Cons(_path).ToArray())); - } - - public AbsolutePath Combine(params string[] paths) - { - - return new AbsolutePath(Path.Combine(paths.Cons(_path).ToArray())); - } - - public IEnumerable ReadAllLines() - { - return File.ReadAllLines(_path); - } - - public async Task WriteAllBytesAsync(byte[] data) - { - await using var fs = await Create(); - await fs.WriteAsync(data); - } - - public async Task WriteAllAsync(Stream data, bool disposeDataAfter = true) - { - await using var fs = await Create(); - await data.CopyToAsync(fs); - if (disposeDataAfter) await data.DisposeAsync(); - } - - [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Auto)] - private static extern bool CreateHardLink(string lpFileName, string lpExistingFileName, IntPtr lpSecurityAttributes); - - public bool HardLinkTo(AbsolutePath destination) - { - Utils.Log($"Hard Linking {_path} to {destination}"); - return CreateHardLink((string)destination, (string)this, IntPtr.Zero); - } - - public async ValueTask HardLinkIfOversize(AbsolutePath destination) - { - if (!destination.Parent.Exists) - destination.Parent.CreateDirectory(); - - if (Root == destination.Root && Consts.SupportedBSAs.Contains(Extension)) - { - if (HardLinkTo(destination)) - return; - } - - await CopyToAsync(destination); - } - - public async Task> ReadAllLinesAsync() - { - return (await ReadAllTextAsync()).Split(new[] {'\n', '\r'}, StringSplitOptions.RemoveEmptyEntries); - } - - public static AbsolutePath GetCurrentDirectory() - { - return new AbsolutePath(Directory.GetCurrentDirectory()); - } - - public async Task CopyToAsync(AbsolutePath destFile) - { - await using var src = await OpenRead(); - await using var dest = await destFile.Create(); - await src.CopyToAsync(dest); - } - - public IEnumerable EnumerateDirectories(bool recursive = true) - { - return Directory.EnumerateDirectories(_path, "*", recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly) - .Select(p => (AbsolutePath)p); - } - - public async Task WriteAllLinesAsync(params string[] strings) - { - await WriteAllTextAsync(string.Join("\r\n",strings)); - } - - public int CompareTo(AbsolutePath other) - { - return string.Compare(_path, other._path, StringComparison.Ordinal); - } - - public string ReadAllText() - { - return File.ReadAllText(_path); - } - - public ValueTask OpenShared() - { - var path = _path; - return CircuitBreaker.WithAutoRetryAsync(async () => - File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)); - } - - public ValueTask WriteShared() - { - var path = _path; - return CircuitBreaker.WithAutoRetryAsync(async () => - File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)); - } - - public async Task CopyDirectoryToAsync(AbsolutePath destination) - { - destination.CreateDirectory(); - foreach (var file in EnumerateFiles()) - { - var dest = file.RelativeTo(this).RelativeTo(destination); - await file.CopyToAsync(dest); - } - } - } - - [JsonConverter(typeof(Utils.RelativePathConverter))] - public struct RelativePath : IPath, IEquatable, IComparable - { - private readonly string? _nullable_path; - private string _path => _nullable_path ?? string.Empty; - - public RelativePath(string path) - { - if (string.IsNullOrWhiteSpace(path)) - { - _nullable_path = null; - return; - } - var trimmed = path.Replace("/", "\\").Trim('\\'); - if (string.IsNullOrEmpty(trimmed)) - { - _nullable_path = null; - return; - } - - _nullable_path = trimmed; - Validate(); - } - - public override string ToString() - { - return _path; - } - - public Extension Extension => Extension.FromPath(_path); - - public override int GetHashCode() - { - return _path.GetHashCode(StringComparison.InvariantCultureIgnoreCase); - } - - public static RelativePath RandomFileName() - { - return (RelativePath)Guid.NewGuid().ToString(); - } - - - public RelativePath Munge() - { - return (RelativePath)_path.Replace('\\', '_').Replace('/', '_').Replace(':', '_'); - } - - private void Validate() - { - if (Path.IsPathRooted(_path)) - { - throw new InvalidDataException($"Cannot create relative path from absolute path string, got {_path}"); - } - } - - public AbsolutePath RelativeTo(AbsolutePath abs) - { - return new AbsolutePath(Path.Combine((string)abs, _path)); - } - - public AbsolutePath RelativeToEntryPoint() - { - return RelativeTo(AbsolutePath.EntryPoint); - } - - public AbsolutePath RelativeToWorkingDirectory() - { - return RelativeTo((AbsolutePath)Directory.GetCurrentDirectory()); - } - - public static explicit operator string(RelativePath path) - { - return path._path; - } - - public static explicit operator RelativePath(string path) - { - return new RelativePath(path); - } - - public AbsolutePath RelativeToSystemDirectory() - { - return RelativeTo((AbsolutePath)Environment.SystemDirectory); - } - - public RelativePath Parent => (RelativePath)Path.GetDirectoryName(_path); - - public RelativePath FileName => new RelativePath(Path.GetFileName(_path)); - - public RelativePath FileNameWithoutExtension => (RelativePath)Path.GetFileNameWithoutExtension(_path); - - public RelativePath TopParent - { - get - { - var curr = this; - - while (curr.Parent != default) - curr = curr.Parent; - - return curr; - } - } - - public bool Equals(RelativePath other) - { - return string.Equals(_path, other._path, StringComparison.InvariantCultureIgnoreCase); - } - - public override bool Equals(object? obj) - { - return obj is RelativePath other && Equals(other); - } - - public static bool operator ==(RelativePath a, RelativePath b) - { - return a.Equals(b); - } - - public static bool operator !=(RelativePath a, RelativePath b) - { - return !a.Equals(b); - } - - public bool StartsWith(string s) - { - return _path.StartsWith(s); - } - - public bool StartsWith(RelativePath s) - { - return _path.StartsWith(s._path); - } - - public RelativePath Combine(params RelativePath[] paths ) - { - return (RelativePath)Path.Combine(paths.Select(p => (string)p).Cons(_path).ToArray()); - } - - public RelativePath Combine(params string[] paths) - { - return (RelativePath)Path.Combine(paths.Cons(_path).ToArray()); - } - - public int CompareTo(RelativePath other) - { - return string.Compare(_path, other._path, StringComparison.Ordinal); - } - } + public static partial class Utils { @@ -738,144 +180,6 @@ namespace Wabbajack.Common } } - public struct HashRelativePath : IEquatable - { - private static RelativePath[] EMPTY_PATH; - public Hash BaseHash { get; } - public RelativePath[] Paths { get; } - static HashRelativePath() - { - EMPTY_PATH = new RelativePath[0]; - } - public HashRelativePath(Hash baseHash, params RelativePath[] paths) - { - BaseHash = baseHash; - Paths = paths; - } - - public override string ToString() - { - var paths = Paths == null ? EmptyPath : Paths; - return string.Join("|", paths.Select(t => t.ToString()).Cons(BaseHash.ToString())); - } - - private static RelativePath[] EmptyPath = Array.Empty(); - - public static bool operator ==(HashRelativePath a, HashRelativePath b) - { - if (a.Paths == null || b.Paths == null) return false; - - if (a.BaseHash != b.BaseHash || a.Paths.Length != b.Paths.Length) - { - return false; - } - - for (var idx = 0; idx < a.Paths.Length; idx += 1) - { - if (a.Paths[idx] != b.Paths[idx]) - { - return false; - } - } - - return true; - } - - public static bool operator !=(HashRelativePath a, HashRelativePath b) - { - return !(a == b); - } - - public bool Equals(HashRelativePath other) - { - return this == other; - } - - public override bool Equals(object? obj) - { - return obj is HashRelativePath other && Equals(other); - } - - public override int GetHashCode() - { - return HashCode.Combine(BaseHash, Paths); - } - - public static HashRelativePath FromStrings(string hash, params string[] paths) - { - return new HashRelativePath(Hash.FromBase64(hash), paths.Select(p => (RelativePath)p).ToArray()); - } - } - - public struct FullPath : IEquatable, IPath - { - public AbsolutePath Base { get; } - - public RelativePath[] Paths { get; } - - private readonly int _hash; - - public FullPath(AbsolutePath basePath, params RelativePath[] paths) - { - Base = basePath; - Paths = paths == null ? Array.Empty() : paths; - _hash = Base.GetHashCode(); - foreach (var itm in Paths) - { - _hash ^= itm.GetHashCode(); - } - } - - public override string ToString() - { - var paths = Paths == null ? EmptyPath : Paths; - return string.Join("|", paths.Select(t => (string)t).Cons((string)Base)); - } - - public override int GetHashCode() - { - return _hash; - } - - private static RelativePath[] EmptyPath = Array.Empty(); - - public static bool operator ==(FullPath a, FullPath b) - { - if (a.Paths == null || b.Paths == null) return false; - - if (a.Base != b.Base || a.Paths.Length != b.Paths.Length) - { - return false; - } - - for (var idx = 0; idx < a.Paths.Length; idx += 1) - { - if (a.Paths[idx] != b.Paths[idx]) - { - return false; - } - } - - return true; - } - - public static bool operator !=(FullPath a, FullPath b) - { - return !(a == b); - } - - public bool Equals(FullPath other) - { - return this == other; - } - - public override bool Equals(object? obj) - { - return obj is FullPath other && Equals(other); - } - - public RelativePath FileName => Paths.Length == 0 ? Base.FileName : Paths.Last().FileName; - } } diff --git a/Wabbajack.Common/Paths/AbsolutePath.cs b/Wabbajack.Common/Paths/AbsolutePath.cs new file mode 100644 index 00000000..af7e2f55 --- /dev/null +++ b/Wabbajack.Common/Paths/AbsolutePath.cs @@ -0,0 +1,421 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using Directory = Alphaleonis.Win32.Filesystem.Directory; +using DriveInfo = Alphaleonis.Win32.Filesystem.DriveInfo; +using File = Alphaleonis.Win32.Filesystem.File; +using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.Common +{ + + public struct AbsolutePath : IPath, IComparable, IEquatable + { + #region ObjectEquality + + public bool Equals(AbsolutePath other) + { + return string.Equals(_path, other._path, StringComparison.InvariantCultureIgnoreCase); + } + + public override bool Equals(object? obj) + { + return obj is AbsolutePath other && Equals(other); + } + + #endregion + + public override int GetHashCode() + { + return _path.GetHashCode(StringComparison.InvariantCultureIgnoreCase); + } + + public override string ToString() + { + return _path; + } + + private readonly string _nullable_path; + private string _path => _nullable_path ?? string.Empty; + + public AbsolutePath(string path, bool skipValidation = false) + { + _nullable_path = path.Replace("/", "\\").TrimEnd('\\'); + if (!skipValidation) + { + ValidateAbsolutePath(); + } + } + + public AbsolutePath(AbsolutePath path) + { + _nullable_path = path._path; + } + + private void ValidateAbsolutePath() + { + if (Path.IsPathRooted(_path)) + { + return; + } + + throw new InvalidDataException($"Absolute path must be absolute, got {_path}"); + } + + public string Normalize() + { + return _path.Replace("/", "\\").TrimEnd('\\'); + } + + public DriveInfo DriveInfo() + { + return new DriveInfo(Path.GetPathRoot(_path)); + } + + public Extension Extension => Extension.FromPath(_path); + + public ValueTask OpenRead() + { + return OpenShared(); + } + + public ValueTask Create() + { + var path = _path; + return CircuitBreaker.WithAutoRetryAsync(async () => File.Open(path, FileMode.Create, FileAccess.ReadWrite)); + } + + public ValueTask OpenWrite() + { + var path = _path; + return CircuitBreaker.WithAutoRetryAsync(async () => File.OpenWrite(path)); + } + + public async Task WriteAllTextAsync(string text) + { + await using var fs = File.Create(_path); + await fs.WriteAsync(Encoding.UTF8.GetBytes(text)); + } + + public bool Exists => File.Exists(_path) || Directory.Exists(_path); + public bool IsFile => File.Exists(_path); + public bool IsDirectory => Directory.Exists(_path); + + public async Task DeleteDirectory(bool dontDeleteIfNotEmpty = false) + { + if (IsDirectory) + { + if (dontDeleteIfNotEmpty && (EnumerateFiles().Any() || EnumerateDirectories().Any())) return; + await Utils.DeleteDirectory(this); + } + } + + public long Size => Exists ? new FileInfo(_path).Length : 0; + + public DateTime LastModified + { + get => File.GetLastWriteTime(_path); + set => File.SetLastWriteTime(_path, value); + } + + public DateTime LastModifiedUtc => File.GetLastWriteTimeUtc(_path); + public AbsolutePath Parent => (AbsolutePath)Path.GetDirectoryName(_path); + public RelativePath FileName => (RelativePath)Path.GetFileName(_path); + public RelativePath FileNameWithoutExtension => (RelativePath)Path.GetFileNameWithoutExtension(_path); + public bool IsEmptyDirectory => IsDirectory && !EnumerateFiles().Any(); + + public bool IsReadOnly + { + get + { + return new FileInfo(_path).IsReadOnly; + } + set + { + new FileInfo(_path).IsReadOnly = value; + } + } + + public void SetReadOnly(bool val) + { + IsReadOnly = true; + } + + /// + /// Returns the full path the folder that contains Wabbajack.Common. This will almost always be + /// where all the binaries for the project reside. + /// + /// + public static AbsolutePath EntryPoint + { + get + { + var location = Assembly.GetExecutingAssembly().Location ?? null; + if (location == null) + throw new ArgumentException("Could not find entry point."); + return ((AbsolutePath)location).Parent; + } + } + + public AbsolutePath Root => (AbsolutePath)Path.GetPathRoot(_path); + + /// + /// Moves this file to the specified location, will use Copy if required + /// + /// + /// Replace the destination file if it exists + public async Task MoveToAsync(AbsolutePath otherPath, bool overwrite = false) + { + if (Root != otherPath.Root) + { + if (otherPath.Exists && overwrite) + await otherPath.DeleteAsync(); + + await CopyToAsync(otherPath); + await DeleteAsync(); + return; + } + + var path = _path; + await CircuitBreaker.WithAutoRetryAsync(async () => File.Move(path, otherPath._path, overwrite ? MoveOptions.ReplaceExisting : MoveOptions.None)); + } + + public RelativePath RelativeTo(AbsolutePath p) + { + var relPath = Path.GetRelativePath(p._path, _path); + if (relPath == _path) + throw new ArgumentException($"{_path} is not a subpath of {p._path}"); + return new RelativePath(relPath); + } + + + public async Task ReadAllTextAsync() + { + await using var fs = File.OpenRead(_path); + return Encoding.UTF8.GetString(await fs.ReadAllAsync()); + } + + /// + /// Assuming the path is a folder, enumerate all the files in the folder + /// + /// if true, also returns files in sub-folders + /// pattern to match against + /// + public IEnumerable EnumerateFiles(bool recursive = true, string pattern = "*") + { + if (!IsDirectory) return new AbsolutePath[0]; + return Directory + .EnumerateFiles(_path, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly) + .Select(path => new AbsolutePath(path, true)); + } + + #region Operators + + public static explicit operator string(AbsolutePath path) + { + return path._path; + } + + public static explicit operator AbsolutePath(string path) + { + if (string.IsNullOrEmpty(path)) return default; + return !Path.IsPathRooted(path) ? ((RelativePath)path).RelativeToEntryPoint() : new AbsolutePath(path); + } + + public static bool operator ==(AbsolutePath a, AbsolutePath b) + { + return a.Equals(b); + } + + public static bool operator !=(AbsolutePath a, AbsolutePath b) + { + return !a.Equals(b); + } + + #endregion + + public void CreateDirectory() + { + Directory.CreateDirectory(_path); + } + + public async Task DeleteAsync() + { + try + { + if (!IsFile) return; + + if (IsReadOnly) IsReadOnly = false; + + var path = _path; + await CircuitBreaker.WithAutoRetryAsync(async () => File.Delete(path)); + } + catch (FileNotFoundException) + { + // ignore, it doesn't exist so why delete it? + } + } + + public void Delete() + { + if (!IsFile) return; + + if (IsReadOnly) IsReadOnly = false; + + var path = _path; + CircuitBreaker.WithAutoRetry(async () => File.Delete(path)); + } + + public bool InFolder(AbsolutePath folder) + { + return _path.StartsWith(folder._path + Path.DirectorySeparator); + } + + public async Task ReadAllBytesAsync() + { + await using var f = await OpenShared(); + return await f.ReadAllAsync(); + } + + public AbsolutePath WithExtension(Extension hashFileExtension) + { + return new AbsolutePath(_path + (string)hashFileExtension, true); + } + + public AbsolutePath ReplaceExtension(Extension extension) + { + return new AbsolutePath( + Path.Combine(Path.GetDirectoryName(_path), Path.GetFileNameWithoutExtension(_path) + (string)extension), + true); + } + + public AbsolutePath AppendToName(string toAppend) + { + return new AbsolutePath( + Path.Combine(Path.GetDirectoryName(_path), + Path.GetFileNameWithoutExtension(_path) + toAppend + (string)Extension)); + } + + public AbsolutePath Combine(params RelativePath[] paths) + { + return new AbsolutePath(Path.Combine(paths.Select(s => (string)s).Cons(_path).ToArray())); + } + + public AbsolutePath Combine(params string[] paths) + { + + return new AbsolutePath(Path.Combine(paths.Cons(_path).ToArray())); + } + + public IEnumerable ReadAllLines() + { + return File.ReadAllLines(_path); + } + + public async Task WriteAllBytesAsync(byte[] data) + { + await using var fs = await Create(); + await fs.WriteAsync(data); + } + + public async Task WriteAllAsync(Stream data, bool disposeDataAfter = true) + { + await using var fs = await Create(); + await data.CopyToAsync(fs); + if (disposeDataAfter) await data.DisposeAsync(); + } + + [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Auto)] + private static extern bool CreateHardLink(string lpFileName, string lpExistingFileName, IntPtr lpSecurityAttributes); + + public bool HardLinkTo(AbsolutePath destination) + { + Utils.Log($"Hard Linking {_path} to {destination}"); + return CreateHardLink((string)destination, (string)this, IntPtr.Zero); + } + + public async ValueTask HardLinkIfOversize(AbsolutePath destination) + { + if (!destination.Parent.Exists) + destination.Parent.CreateDirectory(); + + if (Root == destination.Root && Consts.SupportedBSAs.Contains(Extension)) + { + if (HardLinkTo(destination)) + return; + } + + await CopyToAsync(destination); + } + + public async Task> ReadAllLinesAsync() + { + return (await ReadAllTextAsync()).Split(new[] {'\n', '\r'}, StringSplitOptions.RemoveEmptyEntries); + } + + public static AbsolutePath GetCurrentDirectory() + { + return new AbsolutePath(Directory.GetCurrentDirectory()); + } + + public async Task CopyToAsync(AbsolutePath destFile) + { + await using var src = await OpenRead(); + await using var dest = await destFile.Create(); + await src.CopyToAsync(dest); + } + + public IEnumerable EnumerateDirectories(bool recursive = true) + { + return Directory.EnumerateDirectories(_path, "*", recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly) + .Select(p => (AbsolutePath)p); + } + + public async Task WriteAllLinesAsync(params string[] strings) + { + await WriteAllTextAsync(string.Join("\r\n",strings)); + } + + public int CompareTo(AbsolutePath other) + { + return string.Compare(_path, other._path, StringComparison.Ordinal); + } + + public string ReadAllText() + { + return File.ReadAllText(_path); + } + + public ValueTask OpenShared() + { + var path = _path; + return CircuitBreaker.WithAutoRetryAsync(async () => + File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)); + } + + public ValueTask WriteShared() + { + var path = _path; + return CircuitBreaker.WithAutoRetryAsync(async () => + File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)); + } + + public async Task CopyDirectoryToAsync(AbsolutePath destination) + { + destination.CreateDirectory(); + foreach (var file in EnumerateFiles()) + { + var dest = file.RelativeTo(this).RelativeTo(destination); + await file.CopyToAsync(dest); + } + } + } + +} diff --git a/Wabbajack.Common/Paths/FileCompaction.cs b/Wabbajack.Common/Paths/FileCompaction.cs new file mode 100644 index 00000000..a1a6e64a --- /dev/null +++ b/Wabbajack.Common/Paths/FileCompaction.cs @@ -0,0 +1,61 @@ +using System.Threading.Tasks; +using Wabbajack.Common.IO; + +namespace Wabbajack.Common +{ + public static class FileCompaction + { + private static AbsolutePath _compactExecutable; + private static bool? _haveCompact = null; + + private static AbsolutePath? GetCompactPath() + { + if (_haveCompact != null && _haveCompact.Value) return _compactExecutable; + if (_haveCompact != null) return null; + _compactExecutable = ((AbsolutePath)KnownFolders.SystemX86.Path).Combine("compact.exe"); + + if (!_compactExecutable.Exists) return null; + + _haveCompact = true; + return _compactExecutable; + } + + public enum Algorithm + { + XPRESS4K, + XPRESS8K, + XPRESS16K, + LZX + } + + public static async Task Compact(this AbsolutePath path, Algorithm algorithm) + { + if (!path.Exists) return false; + + + var exe = GetCompactPath(); + if (exe == null) return false; + + if (path.IsFile) + { + var proc = new ProcessHelper + { + Path = exe.Value, + Arguments = new object[] {"/C", "/EXE:" + algorithm, path}, + ThrowOnNonZeroExitCode = false + }; + return await proc.Start() == 0; + } + else + { + var proc = new ProcessHelper + { + Path = exe.Value, + Arguments = new object[] {"/C", "/S", "/EXE:" + algorithm, path}, + ThrowOnNonZeroExitCode = false + }; + return await proc.Start() == 0; + } + } + } +} diff --git a/Wabbajack.Common/Paths/IPath.cs b/Wabbajack.Common/Paths/IPath.cs new file mode 100644 index 00000000..da57dbe6 --- /dev/null +++ b/Wabbajack.Common/Paths/IPath.cs @@ -0,0 +1,12 @@ +namespace Wabbajack.Common +{ + public interface IPath + { + /// + /// Get the final file name, for c:\bar\baz this is `baz` for c:\bar.zip this is `bar.zip` + /// for `bar.zip` this is `bar.zip` + /// + public RelativePath FileName { get; } + } + +} diff --git a/Wabbajack.Common/Paths/RelativePath.cs b/Wabbajack.Common/Paths/RelativePath.cs new file mode 100644 index 00000000..d4c9ba6f --- /dev/null +++ b/Wabbajack.Common/Paths/RelativePath.cs @@ -0,0 +1,159 @@ +using System; +using System.IO; +using System.Linq; +using Newtonsoft.Json; +using Directory = Alphaleonis.Win32.Filesystem.Directory; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.Common +{ + [JsonConverter(typeof(Utils.RelativePathConverter))] + public struct RelativePath : IPath, IEquatable, IComparable + { + private readonly string? _nullable_path; + private string _path => _nullable_path ?? string.Empty; + + public RelativePath(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + _nullable_path = null; + return; + } + var trimmed = path.Replace("/", "\\").Trim('\\'); + if (string.IsNullOrEmpty(trimmed)) + { + _nullable_path = null; + return; + } + + _nullable_path = trimmed; + Validate(); + } + + public override string ToString() + { + return _path; + } + + public Extension Extension => Extension.FromPath(_path); + + public override int GetHashCode() + { + return _path.GetHashCode(StringComparison.InvariantCultureIgnoreCase); + } + + public static RelativePath RandomFileName() + { + return (RelativePath)Guid.NewGuid().ToString(); + } + + + public RelativePath Munge() + { + return (RelativePath)_path.Replace('\\', '_').Replace('/', '_').Replace(':', '_'); + } + + private void Validate() + { + if (Path.IsPathRooted(_path)) + { + throw new InvalidDataException($"Cannot create relative path from absolute path string, got {_path}"); + } + } + + public AbsolutePath RelativeTo(AbsolutePath abs) + { + return new AbsolutePath(Path.Combine((string)abs, _path)); + } + + public AbsolutePath RelativeToEntryPoint() + { + return RelativeTo(AbsolutePath.EntryPoint); + } + + public AbsolutePath RelativeToWorkingDirectory() + { + return RelativeTo((AbsolutePath)Directory.GetCurrentDirectory()); + } + + public static explicit operator string(RelativePath path) + { + return path._path; + } + + public static explicit operator RelativePath(string path) + { + return new RelativePath(path); + } + + public AbsolutePath RelativeToSystemDirectory() + { + return RelativeTo((AbsolutePath)Environment.SystemDirectory); + } + + public RelativePath Parent => (RelativePath)Path.GetDirectoryName(_path); + + public RelativePath FileName => new RelativePath(Path.GetFileName(_path)); + + public RelativePath FileNameWithoutExtension => (RelativePath)Path.GetFileNameWithoutExtension(_path); + + public RelativePath TopParent + { + get + { + var curr = this; + + while (curr.Parent != default) + curr = curr.Parent; + + return curr; + } + } + + public bool Equals(RelativePath other) + { + return string.Equals(_path, other._path, StringComparison.InvariantCultureIgnoreCase); + } + + public override bool Equals(object? obj) + { + return obj is RelativePath other && Equals(other); + } + + public static bool operator ==(RelativePath a, RelativePath b) + { + return a.Equals(b); + } + + public static bool operator !=(RelativePath a, RelativePath b) + { + return !a.Equals(b); + } + + public bool StartsWith(string s) + { + return _path.StartsWith(s); + } + + public bool StartsWith(RelativePath s) + { + return _path.StartsWith(s._path); + } + + public RelativePath Combine(params RelativePath[] paths ) + { + return (RelativePath)Path.Combine(paths.Select(p => (string)p).Cons(_path).ToArray()); + } + + public RelativePath Combine(params string[] paths) + { + return (RelativePath)Path.Combine(paths.Cons(_path).ToArray()); + } + + public int CompareTo(RelativePath other) + { + return string.Compare(_path, other._path, StringComparison.Ordinal); + } + } +} diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index ea60d498..decd2ff4 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -50,7 +50,7 @@ - + diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 2a0a2da4..2c5b2a5a 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -32,6 +32,8 @@ namespace Wabbajack.Lib public GameMetaData Game { get; } public SystemParameters? SystemParameters { get; set; } + + public bool UseCompression { get; set; } public AInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters? parameters, int steps, Game game) @@ -163,8 +165,7 @@ namespace Wabbajack.Lib to.LastModified = DateTime.Now; } - await vFiles.GroupBy(f => f.FromFile) - .PDoIndexed(queue, async (idx, group) => + foreach (var (idx, group) in vFiles.GroupBy(f => f.FromFile).Select((grp, i) => (i, grp))) { Utils.Status("Installing files", Percent.FactoryPutInRange(idx, vFiles.Count)); if (group.Key == null) @@ -186,14 +187,8 @@ namespace Wabbajack.Lib { await CopyFile(firstDest, OutputFolder.Combine(copy.To)); } - }); - Status("Unstaging files"); - await onFinish(); - - // Now patch all the files from this archive - await grouping.OfType() - .PMap(queue, async toPatch => + foreach (var toPatch in group.OfType()) { await using var patchStream = new MemoryStream(); Status($"Patching {toPatch.To.FileName}"); @@ -231,8 +226,20 @@ namespace Wabbajack.Lib await toFile.DeleteAsync(); Utils.ErrorThrow(new Exception($"Virus scan of patched executable reported possible malware: {toFile.ToString()} ({(long)hash})")); } - - }); + } + + if (UseCompression) + { + foreach (var file in group) + { + Utils.Status($"Compacting {file.To}"); + await file.To.RelativeTo(OutputFolder).Compact(FileCompaction.Algorithm.XPRESS16K); + } + } + } + + Status("Unstaging files"); + await onFinish(); } public async Task DownloadArchives() diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index 935898b8..652305c7 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -286,6 +286,9 @@ namespace Wabbajack.Lib Info($"Writing {bsa.To}"); await a.Build(OutputFolder.Combine(bsa.To)); streams.Do(s => s.Dispose()); + + if (UseCompression) + await OutputFolder.Combine(bsa.To).Compact(FileCompaction.Algorithm.XPRESS16K); } var bsaDir = OutputFolder.Combine(Consts.BSACreationDir); @@ -319,6 +322,9 @@ namespace Wabbajack.Lib await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID)); break; } + + if (UseCompression) + await outPath.Compact(FileCompaction.Algorithm.XPRESS16K); }); } diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index f249a04e..6487ec7b 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -17,7 +17,7 @@ 3.1.0 - 6.2.1 + 6.2.4 2.2.2.1 @@ -38,10 +38,10 @@ 2.1.0 - 11.4.17 + 11.5.17 - 11.4.17 + 11.5.17 0.26.0 diff --git a/Wabbajack.Server.Test/Wabbajack.Server.Test.csproj b/Wabbajack.Server.Test/Wabbajack.Server.Test.csproj index e3ec849a..dc3b9479 100644 --- a/Wabbajack.Server.Test/Wabbajack.Server.Test.csproj +++ b/Wabbajack.Server.Test/Wabbajack.Server.Test.csproj @@ -9,7 +9,7 @@ - + diff --git a/Wabbajack.Test/EndToEndTests.cs b/Wabbajack.Test/EndToEndTests.cs index 82ed5fce..8e60880e 100644 --- a/Wabbajack.Test/EndToEndTests.cs +++ b/Wabbajack.Test/EndToEndTests.cs @@ -169,7 +169,10 @@ namespace Wabbajack.Test modList: modlist, outputFolder: utils.InstallFolder, downloadFolder: utils.DownloadsFolder, - parameters: ACompilerTest.CreateDummySystemParameters()); + parameters: ACompilerTest.CreateDummySystemParameters()) + { + UseCompression = true + }; installer.GameFolder = utils.GameFolder; await installer.Begin(); } diff --git a/Wabbajack.Test/Wabbajack.Test.csproj b/Wabbajack.Test/Wabbajack.Test.csproj index 40eb3b5b..96f4c359 100644 --- a/Wabbajack.Test/Wabbajack.Test.csproj +++ b/Wabbajack.Test/Wabbajack.Test.csproj @@ -29,7 +29,7 @@ - + diff --git a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj index 5bbe6b5c..bd6eae3f 100644 --- a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj +++ b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj @@ -11,7 +11,7 @@ - + diff --git a/Wabbajack/Settings.cs b/Wabbajack/Settings.cs index 88f1c7d3..e69574cf 100644 --- a/Wabbajack/Settings.cs +++ b/Wabbajack/Settings.cs @@ -104,6 +104,9 @@ namespace Wabbajack public string Search { get; set; } private bool _isPersistent = true; public bool IsPersistent { get => _isPersistent; set => RaiseAndSetIfChanged(ref _isPersistent, value); } + + private bool _useCompression = true; + public bool UseCompression { get => _useCompression; set => RaiseAndSetIfChanged(ref _useCompression, value); } } [JsonName("PerformanceSettings")] diff --git a/Wabbajack/View Models/Installers/MO2InstallerVM.cs b/Wabbajack/View Models/Installers/MO2InstallerVM.cs index ed83ec2d..7f0118d0 100644 --- a/Wabbajack/View Models/Installers/MO2InstallerVM.cs +++ b/Wabbajack/View Models/Installers/MO2InstallerVM.cs @@ -156,6 +156,7 @@ namespace Wabbajack downloadFolder: DownloadLocation.TargetPath, parameters: SystemParametersConstructor.Create())) { + installer.UseCompression = Parent.MWVM.Settings.Filters.UseCompression; Parent.MWVM.Settings.Performance.AttachToBatchProcessor(installer); return await Task.Run(async () => diff --git a/Wabbajack/Views/Settings/ModlistGallerySettingsView.xaml b/Wabbajack/Views/Settings/ModlistGallerySettingsView.xaml index 7a37ef99..990c4032 100644 --- a/Wabbajack/Views/Settings/ModlistGallerySettingsView.xaml +++ b/Wabbajack/Views/Settings/ModlistGallerySettingsView.xaml @@ -41,6 +41,7 @@ + + Name="FilterPersistCheckBox" + Margin="0,5,0,0" + HorizontalAlignment="Left" + VerticalAlignment="Top" + Content="Gallery filters are saved on exit"> -