Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enable optional disk compression

Browse Source
This commit is contained in:
Timothy Baldridge 2020-08-05 16:01:45 -06:00
parent b8dff45979
commit 86107ec8ec
21 changed files with 724 additions and 734 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Compression.BSA.Test/Compression.BSA.Test.csproj
View File

@ -11,7 +11,7 @@
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.console" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
</ItemGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

4
Wabbajack.App.Test/Wabbajack.App.Test.csproj
View File

@ -10,8 +10,8 @@
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.console" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.2" />
<PackageReference Include="Xunit.StaFact" Version="0.3.18" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
<PackageReference Include="Xunit.StaFact" Version="1.0.37" />
</ItemGroup>
<ItemGroup>

2
Wabbajack.Common.Test/Wabbajack.Common.Test.csproj
View File

@ -11,7 +11,7 @@
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.console" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
</ItemGroup>
<ItemGroup>

698
Wabbajack.Common/Paths.cs
View File

@ -19,566 +19,8 @@ using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Common
{
public interface IPath
{
/// <summary>
/// Get the final file name, for c:\bar\baz this is `baz` for c:\bar.zip this is `bar.zip`
/// for `bar.zip` this is `bar.zip`
/// </summary>
public RelativePath FileName { get; }
}
public struct AbsolutePath : IPath, IComparable<AbsolutePath>, IEquatable<AbsolutePath>
{
#region ObjectEquality
public bool Equals(AbsolutePath other)
{
return string.Equals(_path, other._path, StringComparison.InvariantCultureIgnoreCase);
}
public override bool Equals(object? obj)
{
return obj is AbsolutePath other && Equals(other);
}
#endregion
public override int GetHashCode()
{
return _path.GetHashCode(StringComparison.InvariantCultureIgnoreCase);
}
public override string ToString()
{
return _path;
}
private readonly string _nullable_path;
private string _path => _nullable_path ?? string.Empty;
public AbsolutePath(string path, bool skipValidation = false)
{
_nullable_path = path.Replace("/", "\\").TrimEnd('\\');
if (!skipValidation)
{
ValidateAbsolutePath();
}
}
public AbsolutePath(AbsolutePath path)
{
_nullable_path = path._path;
}
private void ValidateAbsolutePath()
{
if (Path.IsPathRooted(_path))
{
return;
}
throw new InvalidDataException($"Absolute path must be absolute, got {_path}");
}
public string Normalize()
{
return _path.Replace("/", "\\").TrimEnd('\\');
}
public DriveInfo DriveInfo()
{
return new DriveInfo(Path.GetPathRoot(_path));
}
public Extension Extension => Extension.FromPath(_path);
public ValueTask<FileStream> OpenRead()
{
return OpenShared();
}
public ValueTask<FileStream> Create()
{
var path = _path;
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () => File.Open(path, FileMode.Create, FileAccess.ReadWrite));
}
public ValueTask<FileStream> OpenWrite()
{
var path = _path;
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () => File.OpenWrite(path));
}
public async Task WriteAllTextAsync(string text)
{
await using var fs = File.Create(_path);
await fs.WriteAsync(Encoding.UTF8.GetBytes(text));
}
public bool Exists => File.Exists(_path) || Directory.Exists(_path);
public bool IsFile => File.Exists(_path);
public bool IsDirectory => Directory.Exists(_path);
public async Task DeleteDirectory(bool dontDeleteIfNotEmpty = false)
{
if (IsDirectory)
{
if (dontDeleteIfNotEmpty && (EnumerateFiles().Any() || EnumerateDirectories().Any())) return;
await Utils.DeleteDirectory(this);
}
}
public long Size => Exists ? new FileInfo(_path).Length : 0;
public DateTime LastModified
{
get => File.GetLastWriteTime(_path);
set => File.SetLastWriteTime(_path, value);
}
public DateTime LastModifiedUtc => File.GetLastWriteTimeUtc(_path);
public AbsolutePath Parent => (AbsolutePath)Path.GetDirectoryName(_path);
public RelativePath FileName => (RelativePath)Path.GetFileName(_path);
public RelativePath FileNameWithoutExtension => (RelativePath)Path.GetFileNameWithoutExtension(_path);
public bool IsEmptyDirectory => IsDirectory && !EnumerateFiles().Any();
public bool IsReadOnly
{
get
{
return new FileInfo(_path).IsReadOnly;
}
set
{
new FileInfo(_path).IsReadOnly = value;
}
}
public void SetReadOnly(bool val)
{
IsReadOnly = true;
}
/// <summary>
/// Returns the full path the folder that contains Wabbajack.Common. This will almost always be
/// where all the binaries for the project reside.
/// </summary>
/// <exception cref="ArgumentException"></exception>
public static AbsolutePath EntryPoint
{
get
{
var location = Assembly.GetExecutingAssembly().Location ?? null;
if (location == null)
throw new ArgumentException("Could not find entry point.");
return ((AbsolutePath)location).Parent;
}
}
public AbsolutePath Root => (AbsolutePath)Path.GetPathRoot(_path);
/// <summary>
/// Moves this file to the specified location, will use Copy if required
/// </summary>
/// <param name="otherPath"></param>
/// <param name="overwrite">Replace the destination file if it exists</param>
public async Task MoveToAsync(AbsolutePath otherPath, bool overwrite = false)
{
if (Root != otherPath.Root)
{
if (otherPath.Exists && overwrite)
await otherPath.DeleteAsync();
await CopyToAsync(otherPath);
await DeleteAsync();
return;
}
var path = _path;
await CircuitBreaker.WithAutoRetryAsync<IOException>(async () => File.Move(path, otherPath._path, overwrite ? MoveOptions.ReplaceExisting : MoveOptions.None));
}
public RelativePath RelativeTo(AbsolutePath p)
{
var relPath = Path.GetRelativePath(p._path, _path);
if (relPath == _path)
throw new ArgumentException($"{_path} is not a subpath of {p._path}");
return new RelativePath(relPath);
}
public async Task<string> ReadAllTextAsync()
{
await using var fs = File.OpenRead(_path);
return Encoding.UTF8.GetString(await fs.ReadAllAsync());
}
/// <summary>
/// Assuming the path is a folder, enumerate all the files in the folder
/// </summary>
/// <param name="recursive">if true, also returns files in sub-folders</param>
/// <param name="pattern">pattern to match against</param>
/// <returns></returns>
public IEnumerable<AbsolutePath> EnumerateFiles(bool recursive = true, string pattern = "*")
{
if (!IsDirectory) return new AbsolutePath[0];
return Directory
.EnumerateFiles(_path, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)
.Select(path => new AbsolutePath(path, true));
}
#region Operators
public static explicit operator string(AbsolutePath path)
{
return path._path;
}
public static explicit operator AbsolutePath(string path)
{
if (string.IsNullOrEmpty(path)) return default;
return !Path.IsPathRooted(path) ? ((RelativePath)path).RelativeToEntryPoint() : new AbsolutePath(path);
}
public static bool operator ==(AbsolutePath a, AbsolutePath b)
{
return a.Equals(b);
}
public static bool operator !=(AbsolutePath a, AbsolutePath b)
{
return !a.Equals(b);
}
#endregion
public void CreateDirectory()
{
Directory.CreateDirectory(_path);
}
public async Task DeleteAsync()
{
try
{
if (!IsFile) return;
if (IsReadOnly) IsReadOnly = false;
var path = _path;
await CircuitBreaker.WithAutoRetryAsync<IOException>(async () => File.Delete(path));
}
catch (FileNotFoundException)
{
// ignore, it doesn't exist so why delete it?
}
}
public void Delete()
{
if (!IsFile) return;
if (IsReadOnly) IsReadOnly = false;
var path = _path;
CircuitBreaker.WithAutoRetry<IOException>(async () => File.Delete(path));
}
public bool InFolder(AbsolutePath folder)
{
return _path.StartsWith(folder._path + Path.DirectorySeparator);
}
public async Task<byte[]> ReadAllBytesAsync()
{
await using var f = await OpenShared();
return await f.ReadAllAsync();
}
public AbsolutePath WithExtension(Extension hashFileExtension)
{
return new AbsolutePath(_path + (string)hashFileExtension, true);
}
public AbsolutePath ReplaceExtension(Extension extension)
{
return new AbsolutePath(
Path.Combine(Path.GetDirectoryName(_path), Path.GetFileNameWithoutExtension(_path) + (string)extension),
true);
}
public AbsolutePath AppendToName(string toAppend)
{
return new AbsolutePath(
Path.Combine(Path.GetDirectoryName(_path),
Path.GetFileNameWithoutExtension(_path) + toAppend + (string)Extension));
}
public AbsolutePath Combine(params RelativePath[] paths)
{
return new AbsolutePath(Path.Combine(paths.Select(s => (string)s).Cons(_path).ToArray()));
}
public AbsolutePath Combine(params string[] paths)
{
return new AbsolutePath(Path.Combine(paths.Cons(_path).ToArray()));
}
public IEnumerable<string> ReadAllLines()
{
return File.ReadAllLines(_path);
}
public async Task WriteAllBytesAsync(byte[] data)
{
await using var fs = await Create();
await fs.WriteAsync(data);
}
public async Task WriteAllAsync(Stream data, bool disposeDataAfter = true)
{
await using var fs = await Create();
await data.CopyToAsync(fs);
if (disposeDataAfter) await data.DisposeAsync();
}
[DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Auto)]
private static extern bool CreateHardLink(string lpFileName, string lpExistingFileName, IntPtr lpSecurityAttributes);
public bool HardLinkTo(AbsolutePath destination)
{
Utils.Log($"Hard Linking {_path} to {destination}");
return CreateHardLink((string)destination, (string)this, IntPtr.Zero);
}
public async ValueTask HardLinkIfOversize(AbsolutePath destination)
{
if (!destination.Parent.Exists)
destination.Parent.CreateDirectory();
if (Root == destination.Root && Consts.SupportedBSAs.Contains(Extension))
{
if (HardLinkTo(destination))
return;
}
await CopyToAsync(destination);
}
public async Task<IEnumerable<string>> ReadAllLinesAsync()
{
return (await ReadAllTextAsync()).Split(new[] {'\n', '\r'}, StringSplitOptions.RemoveEmptyEntries);
}
public static AbsolutePath GetCurrentDirectory()
{
return new AbsolutePath(Directory.GetCurrentDirectory());
}
public async Task CopyToAsync(AbsolutePath destFile)
{
await using var src = await OpenRead();
await using var dest = await destFile.Create();
await src.CopyToAsync(dest);
}
public IEnumerable<AbsolutePath> EnumerateDirectories(bool recursive = true)
{
return Directory.EnumerateDirectories(_path, "*", recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)
.Select(p => (AbsolutePath)p);
}
public async Task WriteAllLinesAsync(params string[] strings)
{
await WriteAllTextAsync(string.Join("\r\n",strings));
}
public int CompareTo(AbsolutePath other)
{
return string.Compare(_path, other._path, StringComparison.Ordinal);
}
public string ReadAllText()
{
return File.ReadAllText(_path);
}
public ValueTask<FileStream> OpenShared()
{
var path = _path;
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () =>
File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
}
public ValueTask<FileStream> WriteShared()
{
var path = _path;
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () =>
File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite));
}
public async Task CopyDirectoryToAsync(AbsolutePath destination)
{
destination.CreateDirectory();
foreach (var file in EnumerateFiles())
{
var dest = file.RelativeTo(this).RelativeTo(destination);
await file.CopyToAsync(dest);
}
}
}
[JsonConverter(typeof(Utils.RelativePathConverter))]
public struct RelativePath : IPath, IEquatable<RelativePath>, IComparable<RelativePath>
{
private readonly string? _nullable_path;
private string _path => _nullable_path ?? string.Empty;
public RelativePath(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
_nullable_path = null;
return;
}
var trimmed = path.Replace("/", "\\").Trim('\\');
if (string.IsNullOrEmpty(trimmed))
{
_nullable_path = null;
return;
}
_nullable_path = trimmed;
Validate();
}
public override string ToString()
{
return _path;
}
public Extension Extension => Extension.FromPath(_path);
public override int GetHashCode()
{
return _path.GetHashCode(StringComparison.InvariantCultureIgnoreCase);
}
public static RelativePath RandomFileName()
{
return (RelativePath)Guid.NewGuid().ToString();
}
public RelativePath Munge()
{
return (RelativePath)_path.Replace('\\', '_').Replace('/', '_').Replace(':', '_');
}
private void Validate()
{
if (Path.IsPathRooted(_path))
{
throw new InvalidDataException($"Cannot create relative path from absolute path string, got {_path}");
}
}
public AbsolutePath RelativeTo(AbsolutePath abs)
{
return new AbsolutePath(Path.Combine((string)abs, _path));
}
public AbsolutePath RelativeToEntryPoint()
{
return RelativeTo(AbsolutePath.EntryPoint);
}
public AbsolutePath RelativeToWorkingDirectory()
{
return RelativeTo((AbsolutePath)Directory.GetCurrentDirectory());
}
public static explicit operator string(RelativePath path)
{
return path._path;
}
public static explicit operator RelativePath(string path)
{
return new RelativePath(path);
}
public AbsolutePath RelativeToSystemDirectory()
{
return RelativeTo((AbsolutePath)Environment.SystemDirectory);
}
public RelativePath Parent => (RelativePath)Path.GetDirectoryName(_path);
public RelativePath FileName => new RelativePath(Path.GetFileName(_path));
public RelativePath FileNameWithoutExtension => (RelativePath)Path.GetFileNameWithoutExtension(_path);
public RelativePath TopParent
{
get
{
var curr = this;
while (curr.Parent != default)
curr = curr.Parent;
return curr;
}
}
public bool Equals(RelativePath other)
{
return string.Equals(_path, other._path, StringComparison.InvariantCultureIgnoreCase);
}
public override bool Equals(object? obj)
{
return obj is RelativePath other && Equals(other);
}
public static bool operator ==(RelativePath a, RelativePath b)
{
return a.Equals(b);
}
public static bool operator !=(RelativePath a, RelativePath b)
{
return !a.Equals(b);
}
public bool StartsWith(string s)
{
return _path.StartsWith(s);
}
public bool StartsWith(RelativePath s)
{
return _path.StartsWith(s._path);
}
public RelativePath Combine(params RelativePath[] paths )
{
return (RelativePath)Path.Combine(paths.Select(p => (string)p).Cons(_path).ToArray());
}
public RelativePath Combine(params string[] paths)
{
return (RelativePath)Path.Combine(paths.Cons(_path).ToArray());
}
public int CompareTo(RelativePath other)
{
return string.Compare(_path, other._path, StringComparison.Ordinal);
}
}
public static partial class Utils
{
@ -738,144 +180,6 @@ namespace Wabbajack.Common
}
}
public struct HashRelativePath : IEquatable<HashRelativePath>
{
private static RelativePath[] EMPTY_PATH;
public Hash BaseHash { get; }
public RelativePath[] Paths { get; }
static HashRelativePath()
{
EMPTY_PATH = new RelativePath[0];
}
public HashRelativePath(Hash baseHash, params RelativePath[] paths)
{
BaseHash = baseHash;
Paths = paths;
}
public override string ToString()
{
var paths = Paths == null ? EmptyPath : Paths;
return string.Join("|", paths.Select(t => t.ToString()).Cons(BaseHash.ToString()));
}
private static RelativePath[] EmptyPath = Array.Empty<RelativePath>();
public static bool operator ==(HashRelativePath a, HashRelativePath b)
{
if (a.Paths == null || b.Paths == null) return false;
if (a.BaseHash != b.BaseHash || a.Paths.Length != b.Paths.Length)
{
return false;
}
for (var idx = 0; idx < a.Paths.Length; idx += 1)
{
if (a.Paths[idx] != b.Paths[idx])
{
return false;
}
}
return true;
}
public static bool operator !=(HashRelativePath a, HashRelativePath b)
{
return !(a == b);
}
public bool Equals(HashRelativePath other)
{
return this == other;
}
public override bool Equals(object? obj)
{
return obj is HashRelativePath other && Equals(other);
}
public override int GetHashCode()
{
return HashCode.Combine(BaseHash, Paths);
}
public static HashRelativePath FromStrings(string hash, params string[] paths)
{
return new HashRelativePath(Hash.FromBase64(hash), paths.Select(p => (RelativePath)p).ToArray());
}
}
public struct FullPath : IEquatable<FullPath>, IPath
{
public AbsolutePath Base { get; }
public RelativePath[] Paths { get; }
private readonly int _hash;
public FullPath(AbsolutePath basePath, params RelativePath[] paths)
{
Base = basePath;
Paths = paths == null ? Array.Empty<RelativePath>() : paths;
_hash = Base.GetHashCode();
foreach (var itm in Paths)
{
_hash ^= itm.GetHashCode();
}
}
public override string ToString()
{
var paths = Paths == null ? EmptyPath : Paths;
return string.Join("|", paths.Select(t => (string)t).Cons((string)Base));
}
public override int GetHashCode()
{
return _hash;
}
private static RelativePath[] EmptyPath = Array.Empty<RelativePath>();
public static bool operator ==(FullPath a, FullPath b)
{
if (a.Paths == null || b.Paths == null) return false;
if (a.Base != b.Base || a.Paths.Length != b.Paths.Length)
{
return false;
}
for (var idx = 0; idx < a.Paths.Length; idx += 1)
{
if (a.Paths[idx] != b.Paths[idx])
{
return false;
}
}
return true;
}
public static bool operator !=(FullPath a, FullPath b)
{
return !(a == b);
}
public bool Equals(FullPath other)
{
return this == other;
}
public override bool Equals(object? obj)
{
return obj is FullPath other && Equals(other);
}
public RelativePath FileName => Paths.Length == 0 ? Base.FileName : Paths.Last().FileName;
}
}

421
Wabbajack.Common/Paths/AbsolutePath.cs Normal file
View File

@ -0,0 +1,421 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using DriveInfo = Alphaleonis.Win32.Filesystem.DriveInfo;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Common
{
public struct AbsolutePath : IPath, IComparable<AbsolutePath>, IEquatable<AbsolutePath>
{
#region ObjectEquality
public bool Equals(AbsolutePath other)
{
return string.Equals(_path, other._path, StringComparison.InvariantCultureIgnoreCase);
}
public override bool Equals(object? obj)
{
return obj is AbsolutePath other && Equals(other);
}
#endregion
public override int GetHashCode()
{
return _path.GetHashCode(StringComparison.InvariantCultureIgnoreCase);
}
public override string ToString()
{
return _path;
}
private readonly string _nullable_path;
private string _path => _nullable_path ?? string.Empty;
public AbsolutePath(string path, bool skipValidation = false)
{
_nullable_path = path.Replace("/", "\\").TrimEnd('\\');
if (!skipValidation)
{
ValidateAbsolutePath();
}
}
public AbsolutePath(AbsolutePath path)
{
_nullable_path = path._path;
}
private void ValidateAbsolutePath()
{
if (Path.IsPathRooted(_path))
{
return;
}
throw new InvalidDataException($"Absolute path must be absolute, got {_path}");
}
public string Normalize()
{
return _path.Replace("/", "\\").TrimEnd('\\');
}
public DriveInfo DriveInfo()
{
return new DriveInfo(Path.GetPathRoot(_path));
}
public Extension Extension => Extension.FromPath(_path);
public ValueTask<FileStream> OpenRead()
{
return OpenShared();
}
public ValueTask<FileStream> Create()
{
var path = _path;
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () => File.Open(path, FileMode.Create, FileAccess.ReadWrite));
}
public ValueTask<FileStream> OpenWrite()
{
var path = _path;
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () => File.OpenWrite(path));
}
public async Task WriteAllTextAsync(string text)
{
await using var fs = File.Create(_path);
await fs.WriteAsync(Encoding.UTF8.GetBytes(text));
}
public bool Exists => File.Exists(_path) || Directory.Exists(_path);
public bool IsFile => File.Exists(_path);
public bool IsDirectory => Directory.Exists(_path);
public async Task DeleteDirectory(bool dontDeleteIfNotEmpty = false)
{
if (IsDirectory)
{
if (dontDeleteIfNotEmpty && (EnumerateFiles().Any() || EnumerateDirectories().Any())) return;
await Utils.DeleteDirectory(this);
}
}
public long Size => Exists ? new FileInfo(_path).Length : 0;
public DateTime LastModified
{
get => File.GetLastWriteTime(_path);
set => File.SetLastWriteTime(_path, value);
}
public DateTime LastModifiedUtc => File.GetLastWriteTimeUtc(_path);
public AbsolutePath Parent => (AbsolutePath)Path.GetDirectoryName(_path);
public RelativePath FileName => (RelativePath)Path.GetFileName(_path);
public RelativePath FileNameWithoutExtension => (RelativePath)Path.GetFileNameWithoutExtension(_path);
public bool IsEmptyDirectory => IsDirectory && !EnumerateFiles().Any();
public bool IsReadOnly
{
get
{
return new FileInfo(_path).IsReadOnly;
}
set
{
new FileInfo(_path).IsReadOnly = value;
}
}
public void SetReadOnly(bool val)
{
IsReadOnly = true;
}
/// <summary>
/// Returns the full path the folder that contains Wabbajack.Common. This will almost always be
/// where all the binaries for the project reside.
/// </summary>
/// <exception cref="ArgumentException"></exception>
public static AbsolutePath EntryPoint
{
get
{
var location = Assembly.GetExecutingAssembly().Location ?? null;
if (location == null)
throw new ArgumentException("Could not find entry point.");
return ((AbsolutePath)location).Parent;
}
}
public AbsolutePath Root => (AbsolutePath)Path.GetPathRoot(_path);
/// <summary>
/// Moves this file to the specified location, will use Copy if required
/// </summary>
/// <param name="otherPath"></param>
/// <param name="overwrite">Replace the destination file if it exists</param>
public async Task MoveToAsync(AbsolutePath otherPath, bool overwrite = false)
{
if (Root != otherPath.Root)
{
if (otherPath.Exists && overwrite)
await otherPath.DeleteAsync();
await CopyToAsync(otherPath);
await DeleteAsync();
return;
}
var path = _path;
await CircuitBreaker.WithAutoRetryAsync<IOException>(async () => File.Move(path, otherPath._path, overwrite ? MoveOptions.ReplaceExisting : MoveOptions.None));
}
public RelativePath RelativeTo(AbsolutePath p)
{
var relPath = Path.GetRelativePath(p._path, _path);
if (relPath == _path)
throw new ArgumentException($"{_path} is not a subpath of {p._path}");
return new RelativePath(relPath);
}
public async Task<string> ReadAllTextAsync()
{
await using var fs = File.OpenRead(_path);
return Encoding.UTF8.GetString(await fs.ReadAllAsync());
}
/// <summary>
/// Assuming the path is a folder, enumerate all the files in the folder
/// </summary>
/// <param name="recursive">if true, also returns files in sub-folders</param>
/// <param name="pattern">pattern to match against</param>
/// <returns></returns>
public IEnumerable<AbsolutePath> EnumerateFiles(bool recursive = true, string pattern = "*")
{
if (!IsDirectory) return new AbsolutePath[0];
return Directory
.EnumerateFiles(_path, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)
.Select(path => new AbsolutePath(path, true));
}
#region Operators
public static explicit operator string(AbsolutePath path)
{
return path._path;
}
public static explicit operator AbsolutePath(string path)
{
if (string.IsNullOrEmpty(path)) return default;
return !Path.IsPathRooted(path) ? ((RelativePath)path).RelativeToEntryPoint() : new AbsolutePath(path);
}
public static bool operator ==(AbsolutePath a, AbsolutePath b)
{
return a.Equals(b);
}
public static bool operator !=(AbsolutePath a, AbsolutePath b)
{
return !a.Equals(b);
}
#endregion
public void CreateDirectory()
{
Directory.CreateDirectory(_path);
}
public async Task DeleteAsync()
{
try
{
if (!IsFile) return;
if (IsReadOnly) IsReadOnly = false;
var path = _path;
await CircuitBreaker.WithAutoRetryAsync<IOException>(async () => File.Delete(path));
}
catch (FileNotFoundException)
{
// ignore, it doesn't exist so why delete it?
}
}
public void Delete()
{
if (!IsFile) return;
if (IsReadOnly) IsReadOnly = false;
var path = _path;
CircuitBreaker.WithAutoRetry<IOException>(async () => File.Delete(path));
}
public bool InFolder(AbsolutePath folder)
{
return _path.StartsWith(folder._path + Path.DirectorySeparator);
}
public async Task<byte[]> ReadAllBytesAsync()
{
await using var f = await OpenShared();
return await f.ReadAllAsync();
}
public AbsolutePath WithExtension(Extension hashFileExtension)
{
return new AbsolutePath(_path + (string)hashFileExtension, true);
}
public AbsolutePath ReplaceExtension(Extension extension)
{
return new AbsolutePath(
Path.Combine(Path.GetDirectoryName(_path), Path.GetFileNameWithoutExtension(_path) + (string)extension),
true);
}
public AbsolutePath AppendToName(string toAppend)
{
return new AbsolutePath(
Path.Combine(Path.GetDirectoryName(_path),
Path.GetFileNameWithoutExtension(_path) + toAppend + (string)Extension));
}
public AbsolutePath Combine(params RelativePath[] paths)
{
return new AbsolutePath(Path.Combine(paths.Select(s => (string)s).Cons(_path).ToArray()));
}
public AbsolutePath Combine(params string[] paths)
{
return new AbsolutePath(Path.Combine(paths.Cons(_path).ToArray()));
}
public IEnumerable<string> ReadAllLines()
{
return File.ReadAllLines(_path);
}
public async Task WriteAllBytesAsync(byte[] data)
{
await using var fs = await Create();
await fs.WriteAsync(data);
}
public async Task WriteAllAsync(Stream data, bool disposeDataAfter = true)
{
await using var fs = await Create();
await data.CopyToAsync(fs);
if (disposeDataAfter) await data.DisposeAsync();
}
[DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Auto)]
private static extern bool CreateHardLink(string lpFileName, string lpExistingFileName, IntPtr lpSecurityAttributes);
public bool HardLinkTo(AbsolutePath destination)
{
Utils.Log($"Hard Linking {_path} to {destination}");
return CreateHardLink((string)destination, (string)this, IntPtr.Zero);
}
public async ValueTask HardLinkIfOversize(AbsolutePath destination)
{
if (!destination.Parent.Exists)
destination.Parent.CreateDirectory();
if (Root == destination.Root && Consts.SupportedBSAs.Contains(Extension))
{
if (HardLinkTo(destination))
return;
}
await CopyToAsync(destination);
}
public async Task<IEnumerable<string>> ReadAllLinesAsync()
{
return (await ReadAllTextAsync()).Split(new[] {'\n', '\r'}, StringSplitOptions.RemoveEmptyEntries);
}
public static AbsolutePath GetCurrentDirectory()
{
return new AbsolutePath(Directory.GetCurrentDirectory());
}
public async Task CopyToAsync(AbsolutePath destFile)
{
await using var src = await OpenRead();
await using var dest = await destFile.Create();
await src.CopyToAsync(dest);
}
public IEnumerable<AbsolutePath> EnumerateDirectories(bool recursive = true)
{
return Directory.EnumerateDirectories(_path, "*", recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)
.Select(p => (AbsolutePath)p);
}
public async Task WriteAllLinesAsync(params string[] strings)
{
await WriteAllTextAsync(string.Join("\r\n",strings));
}
public int CompareTo(AbsolutePath other)
{
return string.Compare(_path, other._path, StringComparison.Ordinal);
}
public string ReadAllText()
{
return File.ReadAllText(_path);
}
public ValueTask<FileStream> OpenShared()
{
var path = _path;
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () =>
File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
}
public ValueTask<FileStream> WriteShared()
{
var path = _path;
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () =>
File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite));
}
public async Task CopyDirectoryToAsync(AbsolutePath destination)
{
destination.CreateDirectory();
foreach (var file in EnumerateFiles())
{
var dest = file.RelativeTo(this).RelativeTo(destination);
await file.CopyToAsync(dest);
}
}
}
}

61
Wabbajack.Common/Paths/FileCompaction.cs Normal file
View File

@ -0,0 +1,61 @@
using System.Threading.Tasks;
using Wabbajack.Common.IO;
namespace Wabbajack.Common
{
public static class FileCompaction
{
private static AbsolutePath _compactExecutable;
private static bool? _haveCompact = null;
private static AbsolutePath? GetCompactPath()
{
if (_haveCompact != null && _haveCompact.Value) return _compactExecutable;
if (_haveCompact != null) return null;
_compactExecutable = ((AbsolutePath)KnownFolders.SystemX86.Path).Combine("compact.exe");
if (!_compactExecutable.Exists) return null;
_haveCompact = true;
return _compactExecutable;
}
public enum Algorithm
{
XPRESS4K,
XPRESS8K,
XPRESS16K,
LZX
}
public static async Task<bool> Compact(this AbsolutePath path, Algorithm algorithm)
{
if (!path.Exists) return false;
var exe = GetCompactPath();
if (exe == null) return false;
if (path.IsFile)
{
var proc = new ProcessHelper
{
Path = exe.Value,
Arguments = new object[] {"/C", "/EXE:" + algorithm, path},
ThrowOnNonZeroExitCode = false
};
return await proc.Start() == 0;
}
else
{
var proc = new ProcessHelper
{
Path = exe.Value,
Arguments = new object[] {"/C", "/S", "/EXE:" + algorithm, path},
ThrowOnNonZeroExitCode = false
};
return await proc.Start() == 0;
}
}
}
}

12
Wabbajack.Common/Paths/IPath.cs Normal file
View File

@ -0,0 +1,12 @@
namespace Wabbajack.Common
{
public interface IPath
{
/// <summary>
/// Get the final file name, for c:\bar\baz this is `baz` for c:\bar.zip this is `bar.zip`
/// for `bar.zip` this is `bar.zip`
/// </summary>
public RelativePath FileName { get; }
}
}

159
Wabbajack.Common/Paths/RelativePath.cs Normal file
View File

@ -0,0 +1,159 @@
using System;
using System.IO;
using System.Linq;
using Newtonsoft.Json;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Common
{
[JsonConverter(typeof(Utils.RelativePathConverter))]
public struct RelativePath : IPath, IEquatable<RelativePath>, IComparable<RelativePath>
{
private readonly string? _nullable_path;
private string _path => _nullable_path ?? string.Empty;
public RelativePath(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
_nullable_path = null;
return;
}
var trimmed = path.Replace("/", "\\").Trim('\\');
if (string.IsNullOrEmpty(trimmed))
{
_nullable_path = null;
return;
}
_nullable_path = trimmed;
Validate();
}
public override string ToString()
{
return _path;
}
public Extension Extension => Extension.FromPath(_path);
public override int GetHashCode()
{
return _path.GetHashCode(StringComparison.InvariantCultureIgnoreCase);
}
public static RelativePath RandomFileName()
{
return (RelativePath)Guid.NewGuid().ToString();
}
public RelativePath Munge()
{
return (RelativePath)_path.Replace('\\', '_').Replace('/', '_').Replace(':', '_');
}
private void Validate()
{
if (Path.IsPathRooted(_path))
{
throw new InvalidDataException($"Cannot create relative path from absolute path string, got {_path}");
}
}
public AbsolutePath RelativeTo(AbsolutePath abs)
{
return new AbsolutePath(Path.Combine((string)abs, _path));
}
public AbsolutePath RelativeToEntryPoint()
{
return RelativeTo(AbsolutePath.EntryPoint);
}
public AbsolutePath RelativeToWorkingDirectory()
{
return RelativeTo((AbsolutePath)Directory.GetCurrentDirectory());
}
public static explicit operator string(RelativePath path)
{
return path._path;
}
public static explicit operator RelativePath(string path)
{
return new RelativePath(path);
}
public AbsolutePath RelativeToSystemDirectory()
{
return RelativeTo((AbsolutePath)Environment.SystemDirectory);
}
public RelativePath Parent => (RelativePath)Path.GetDirectoryName(_path);
public RelativePath FileName => new RelativePath(Path.GetFileName(_path));
public RelativePath FileNameWithoutExtension => (RelativePath)Path.GetFileNameWithoutExtension(_path);
public RelativePath TopParent
{
get
{
var curr = this;
while (curr.Parent != default)
curr = curr.Parent;
return curr;
}
}
public bool Equals(RelativePath other)
{
return string.Equals(_path, other._path, StringComparison.InvariantCultureIgnoreCase);
}
public override bool Equals(object? obj)
{
return obj is RelativePath other && Equals(other);
}
public static bool operator ==(RelativePath a, RelativePath b)
{
return a.Equals(b);
}
public static bool operator !=(RelativePath a, RelativePath b)
{
return !a.Equals(b);
}
public bool StartsWith(string s)
{
return _path.StartsWith(s);
}
public bool StartsWith(RelativePath s)
{
return _path.StartsWith(s._path);
}
public RelativePath Combine(params RelativePath[] paths )
{
return (RelativePath)Path.Combine(paths.Select(p => (string)p).Cons(_path).ToArray());
}
public RelativePath Combine(params string[] paths)
{
return (RelativePath)Path.Combine(paths.Cons(_path).ToArray());
}
public int CompareTo(RelativePath other)
{
return string.Compare(_path, other._path, StringComparison.Ordinal);
}
}
}

2
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -50,7 +50,7 @@
<PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0-preview.6.20305.6" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="Octodiff" Version="1.2.1" />
<PackageReference Include="ReactiveUI" Version="11.4.17" />
<PackageReference Include="ReactiveUI" Version="11.5.17" />
<PackageReference Include="RocksDbNative" Version="6.2.2" />
<PackageReference Include="RocksDbSharp" Version="6.2.2" />
<PackageReference Include="SharpZipLib" Version="1.2.0" />

29
Wabbajack.Lib/AInstaller.cs
View File

@ -32,6 +32,8 @@ namespace Wabbajack.Lib
public GameMetaData Game { get; }
public SystemParameters? SystemParameters { get; set; }
public bool UseCompression { get; set; }
public AInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters? parameters, int steps, Game game)
@ -163,8 +165,7 @@ namespace Wabbajack.Lib
to.LastModified = DateTime.Now;
}
await vFiles.GroupBy(f => f.FromFile)
.PDoIndexed(queue, async (idx, group) =>
foreach (var (idx, group) in vFiles.GroupBy(f => f.FromFile).Select((grp, i) => (i, grp)))
{
Utils.Status("Installing files", Percent.FactoryPutInRange(idx, vFiles.Count));
if (group.Key == null)
@ -186,14 +187,8 @@ namespace Wabbajack.Lib
{
await CopyFile(firstDest, OutputFolder.Combine(copy.To));
}
});
Status("Unstaging files");
await onFinish();
// Now patch all the files from this archive
await grouping.OfType<PatchedFromArchive>()
.PMap(queue, async toPatch =>
foreach (var toPatch in group.OfType<PatchedFromArchive>())
{
await using var patchStream = new MemoryStream();
Status($"Patching {toPatch.To.FileName}");
@ -231,8 +226,20 @@ namespace Wabbajack.Lib
await toFile.DeleteAsync();
Utils.ErrorThrow(new Exception($"Virus scan of patched executable reported possible malware: {toFile.ToString()} ({(long)hash})"));
}
});
}
if (UseCompression)
{
foreach (var file in group)
{
Utils.Status($"Compacting {file.To}");
await file.To.RelativeTo(OutputFolder).Compact(FileCompaction.Algorithm.XPRESS16K);
}
}
}
Status("Unstaging files");
await onFinish();
}
public async Task DownloadArchives()

6
Wabbajack.Lib/MO2Installer.cs
View File

@ -286,6 +286,9 @@ namespace Wabbajack.Lib
Info($"Writing {bsa.To}");
await a.Build(OutputFolder.Combine(bsa.To));
streams.Do(s => s.Dispose());
if (UseCompression)
await OutputFolder.Combine(bsa.To).Compact(FileCompaction.Algorithm.XPRESS16K);
}
var bsaDir = OutputFolder.Combine(Consts.BSACreationDir);
@ -319,6 +322,9 @@ namespace Wabbajack.Lib
await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID));
break;
}
if (UseCompression)
await outPath.Compact(FileCompaction.Algorithm.XPRESS16K);
});
}

6
Wabbajack.Lib/Wabbajack.Lib.csproj
View File

@ -17,7 +17,7 @@
<Version>3.1.0</Version>
</PackageReference>
<PackageReference Include="Fody">
<Version>6.2.1</Version>
<Version>6.2.4</Version>
</PackageReference>
<PackageReference Include="Genbox.AlphaFS">
<Version>2.2.2.1</Version>
@ -38,10 +38,10 @@
<Version>2.1.0</Version>
</PackageReference>
<PackageReference Include="ReactiveUI">
<Version>11.4.17</Version>
<Version>11.5.17</Version>
</PackageReference>
<PackageReference Include="ReactiveUI.Fody">
<Version>11.4.17</Version>
<Version>11.5.17</Version>
</PackageReference>
<PackageReference Include="SharpCompress">
<Version>0.26.0</Version>

2
Wabbajack.Server.Test/Wabbajack.Server.Test.csproj
View File

@ -9,7 +9,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
<PackageReference Include="coverlet.collector" Version="1.3.0" />
<PackageReference Include="XunitContext" Version="2.0.1" />
</ItemGroup>

5
Wabbajack.Test/EndToEndTests.cs
View File

@ -169,7 +169,10 @@ namespace Wabbajack.Test
modList: modlist,
outputFolder: utils.InstallFolder,
downloadFolder: utils.DownloadsFolder,
parameters: ACompilerTest.CreateDummySystemParameters());
parameters: ACompilerTest.CreateDummySystemParameters())
{
UseCompression = true
};
installer.GameFolder = utils.GameFolder;
await installer.Begin();
}

2
Wabbajack.Test/Wabbajack.Test.csproj
View File

@ -29,7 +29,7 @@
<PackageReference Include="CefSharp.OffScreen" Version="83.4.20" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
<PackageReference Include="coverlet.collector" Version="1.3.0" />
<PackageReference Include="XunitContext" Version="2.0.1" />
</ItemGroup>

2
Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj
View File

@ -11,7 +11,7 @@
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.console" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
</ItemGroup>
<ItemGroup>

3
Wabbajack/Settings.cs
View File

@ -104,6 +104,9 @@ namespace Wabbajack
public string Search { get; set; }
private bool _isPersistent = true;
public bool IsPersistent { get => _isPersistent; set => RaiseAndSetIfChanged(ref _isPersistent, value); }
private bool _useCompression = true;
public bool UseCompression { get => _useCompression; set => RaiseAndSetIfChanged(ref _useCompression, value); }
}
[JsonName("PerformanceSettings")]

1
Wabbajack/View Models/Installers/MO2InstallerVM.cs
View File

@ -156,6 +156,7 @@ namespace Wabbajack
downloadFolder: DownloadLocation.TargetPath,
parameters: SystemParametersConstructor.Create()))
{
installer.UseCompression = Parent.MWVM.Settings.Filters.UseCompression;
Parent.MWVM.Settings.Performance.AttachToBatchProcessor(installer);
return await Task.Run(async () =>

23
Wabbajack/Views/Settings/ModlistGallerySettingsView.xaml
View File

@ -41,6 +41,7 @@
<Grid.RowDefinitions>
<RowDefinition />
<RowDefinition />
<RowDefinition />
</Grid.RowDefinitions>
<Grid.Resources>
<Style BasedOn="{StaticResource MainButtonStyle}" TargetType="Button">
@ -54,16 +55,26 @@
</Style>
</Grid.Resources>
<CheckBox Grid.Row="0"
Name="FilterPersistCheckBox"
Margin="0,5,0,0"
HorizontalAlignment="Left"
VerticalAlignment="Top"
Content="Gallery filters are saved on exit">
Name="FilterPersistCheckBox"
Margin="0,5,0,0"
HorizontalAlignment="Left"
VerticalAlignment="Top"
Content="Gallery filters are saved on exit">
<CheckBox.LayoutTransform>
<ScaleTransform ScaleX="1.2" ScaleY="1.2" />
</CheckBox.LayoutTransform>
</CheckBox>
<Button Grid.Row="1"
<CheckBox Grid.Row="1"
Name="UseCompressionCheckBox"
Margin="0,5,0,0"
HorizontalAlignment="Left"
VerticalAlignment="Top"
Content="Use NTFS LZS compression during install">
<CheckBox.LayoutTransform>
<ScaleTransform ScaleX="1.2" ScaleY="1.2" />
</CheckBox.LayoutTransform>
</CheckBox>
<Button Grid.Row="2"
Name="ClearCefCache"
Margin="0,5,0,0"
Content="Clear In-App Browser Cache" />

2
Wabbajack/Views/Settings/ModlistGallerySettingsView.xaml.cs
View File

@ -34,6 +34,8 @@ namespace Wabbajack
// Bind Values
this.Bind(this.ViewModel, x => x.IsPersistent, x => x.FilterPersistCheckBox.IsChecked)
.DisposeWith(disposable);
this.Bind(this.ViewModel, x => x.UseCompression, x => x.UseCompressionCheckBox.IsChecked)
.DisposeWith(disposable);
this.ClearCefCache.Click += (sender, args) => {Driver.ClearCache();};
});

16
Wabbajack/Wabbajack.csproj
View File

@ -56,24 +56,24 @@
<ItemGroup>
<PackageReference Include="CefSharp.Wpf" Version="83.4.20" />
<PackageReference Include="DynamicData" Version="6.16.1" />
<PackageReference Include="DynamicData" Version="6.16.3" />
<PackageReference Include="Extended.Wpf.Toolkit" Version="4.0.1" />
<PackageReference Include="Fody" Version="6.2.1">
<PackageReference Include="Fody" Version="6.2.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
<PackageReference Include="GitInfo" Version="2.0.30">
<PackageReference Include="GitInfo" Version="2.0.31">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="MahApps.Metro" Version="2.1.1" />
<PackageReference Include="MahApps.Metro.IconPacks" Version="4.3.0" />
<PackageReference Include="MahApps.Metro" Version="2.2.0" />
<PackageReference Include="MahApps.Metro.IconPacks" Version="4.4.0" />
<PackageReference Include="PInvoke.Gdi32" Version="0.6.49" />
<PackageReference Include="PInvoke.User32" Version="0.6.49" />
<PackageReference Include="ReactiveUI" Version="11.4.17" />
<PackageReference Include="ReactiveUI.Fody" Version="11.4.17" />
<PackageReference Include="ReactiveUI.WPF" Version="11.4.17" />
<PackageReference Include="ReactiveUI" Version="11.5.17" />
<PackageReference Include="ReactiveUI.Fody" Version="11.5.17" />
<PackageReference Include="ReactiveUI.WPF" Version="11.5.17" />
<PackageReference Include="SharpDX.DXGI" Version="4.2.0" />
<PackageReference Include="WindowsAPICodePack-Shell" Version="1.1.1" />
<PackageReference Include="WPFThemes.DarkBlend" Version="1.0.8" />