Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tons of WIP changes for paths

Browse Source
This commit is contained in:
Timothy Baldridge 2020-03-23 06:57:18 -06:00
parent d6123a7fb2
commit b37728eefd
21 changed files with 627 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Compression.BSA/BA2Builder.cs
View File

@ -122,7 +122,7 @@ namespace Compression.BSA
public uint FileHash => _state.NameHash; public uint FileHash => _state.NameHash;
public uint DirHash => _state.DirHash; public uint DirHash => _state.DirHash;
public string FullName => _state.Path; public string FullName => (string)_state.Path;
public int Index => _state.Index; public int Index => _state.Index;
public void WriteHeader(BinaryWriter bw) public void WriteHeader(BinaryWriter bw)
@ -247,7 +247,7 @@ namespace Compression.BSA
public uint FileHash => _state.NameHash; public uint FileHash => _state.NameHash;
public uint DirHash => _state.DirHash; public uint DirHash => _state.DirHash;
public string FullName => _state.Path; public string FullName => (string)_state.Path;
public int Index => _state.Index; public int Index => _state.Index;
public void WriteHeader(BinaryWriter wtr) public void WriteHeader(BinaryWriter wtr)

17
Compression.BSA/BA2Reader.cs
View File

@ -4,6 +4,7 @@ using System.IO;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using ICSharpCode.SharpZipLib.Zip.Compression; using ICSharpCode.SharpZipLib.Zip.Compression;
using Wabbajack.Common;
using File = Alphaleonis.Win32.Filesystem.File; using File = Alphaleonis.Win32.Filesystem.File;
namespace Compression.BSA namespace Compression.BSA
@ -23,7 +24,7 @@ namespace Compression.BSA
public class BA2Reader : IBSAReader public class BA2Reader : IBSAReader
{ {
internal string _filename; internal AbsolutePath _filename;
private Stream _stream; private Stream _stream;
internal BinaryReader _rdr; internal BinaryReader _rdr;
internal uint _version; internal uint _version;
@ -35,7 +36,7 @@ namespace Compression.BSA
public bool HasNameTable => _nameTableOffset > 0; public bool HasNameTable => _nameTableOffset > 0;
public BA2Reader(string filename) : this(File.OpenRead(filename)) public BA2Reader(AbsolutePath filename) : this(filename.OpenRead())
{ {
_filename = filename; _filename = filename;
} }
@ -174,7 +175,7 @@ namespace Compression.BSA
public string FullPath { get; set; } public string FullPath { get; set; }
public string Path => FullPath; public RelativePath Path => new RelativePath(FullPath);
public uint Size => (uint)_chunks.Sum(f => f._fullSz) + HeaderSize + sizeof(uint); public uint Size => (uint)_chunks.Sum(f => f._fullSz) + HeaderSize + sizeof(uint);
public FileStateObject State => new BA2DX10EntryState(this); public FileStateObject State => new BA2DX10EntryState(this);
@ -186,7 +187,7 @@ namespace Compression.BSA
WriteHeader(bw); WriteHeader(bw);
using (var fs = File.OpenRead(_bsa._filename)) using (var fs = _bsa._filename.OpenRead())
using (var br = new BinaryReader(fs)) using (var br = new BinaryReader(fs))
{ {
foreach (var chunk in _chunks) foreach (var chunk in _chunks)
@ -328,7 +329,7 @@ namespace Compression.BSA
public BA2DX10EntryState() { } public BA2DX10EntryState() { }
public BA2DX10EntryState(BA2DX10Entry ba2Dx10Entry) public BA2DX10EntryState(BA2DX10Entry ba2Dx10Entry)
{ {
Path = ba2Dx10Entry.FullPath; Path = ba2Dx10Entry.Path;
NameHash = ba2Dx10Entry._nameHash; NameHash = ba2Dx10Entry._nameHash;
Extension = ba2Dx10Entry._extension; Extension = ba2Dx10Entry._extension;
DirHash = ba2Dx10Entry._dirHash; DirHash = ba2Dx10Entry._dirHash;
@ -438,13 +439,13 @@ namespace Compression.BSA
public string FullPath { get; set; } public string FullPath { get; set; }
public string Path => FullPath; public RelativePath Path => new RelativePath(FullPath);
public uint Size => _realSize; public uint Size => _realSize;
public FileStateObject State => new BA2FileEntryState(this); public FileStateObject State => new BA2FileEntryState(this);
public void CopyDataTo(Stream output) public void CopyDataTo(Stream output)
{ {
using (var fs = File.OpenRead(_bsa._filename)) using (var fs = _bsa._filename.OpenRead())
{ {
fs.Seek((long) _offset, SeekOrigin.Begin); fs.Seek((long) _offset, SeekOrigin.Begin);
uint len = Compressed ? _size : _realSize; uint len = Compressed ? _size : _realSize;
@ -479,7 +480,7 @@ namespace Compression.BSA
Flags = ba2FileEntry._flags; Flags = ba2FileEntry._flags;
Align = ba2FileEntry._align; Align = ba2FileEntry._align;
Compressed = ba2FileEntry.Compressed; Compressed = ba2FileEntry.Compressed;
Path = ba2FileEntry.FullPath; Path = ba2FileEntry.Path;
Extension = ba2FileEntry._extension; Extension = ba2FileEntry._extension;
Index = ba2FileEntry._index; Index = ba2FileEntry._index;
} }

26
Compression.BSA/BSABuilder.cs
View File

@ -59,11 +59,11 @@ namespace Compression.BSA
set => _version = (uint) value; set => _version = (uint) value;
} }
public IEnumerable<string> FolderNames public IEnumerable<RelativePath> FolderNames
{ {
get get
{ {
return _files.Select(f => Path.GetDirectoryName(f.Path)).Distinct(); return _files.Select(f => f.Path.Parent).Distinct();
} }
} }
@ -128,13 +128,11 @@ namespace Compression.BSA
public void RegenFolderRecords() public void RegenFolderRecords()
{ {
_folders = _files.GroupBy(f => Path.GetDirectoryName(f.Path.ToLowerInvariant())) _folders = _files.GroupBy(f => f.Path.Parent)
.Select(f => new FolderRecordBuilder(this, f.Key, f.ToList())) .Select(f => new FolderRecordBuilder(this, f.Key, f.ToList()))
.OrderBy(f => f._hash) .OrderBy(f => f._hash)
.ToList(); .ToList();
var lnk = _files.Where(f => f.Path.EndsWith(".lnk")).FirstOrDefault();
foreach (var folder in _folders) foreach (var folder in _folders)
foreach (var file in folder._files) foreach (var file in folder._files)
file._folder = folder; file._folder = folder;
@ -156,13 +154,13 @@ namespace Compression.BSA
internal ulong _offset; internal ulong _offset;
internal uint _recordSize; internal uint _recordSize;
public FolderRecordBuilder(BSABuilder bsa, string folderName, IEnumerable<FileEntry> files) public FolderRecordBuilder(BSABuilder bsa, RelativePath folderName, IEnumerable<FileEntry> files)
{ {
_files = files.OrderBy(f => f._hash); _files = files.OrderBy(f => f._hash);
Name = folderName.ToLowerInvariant(); Name = folderName;
_bsa = bsa; _bsa = bsa;
// Folders don't have extensions, so let's make sure we cut it out // Folders don't have extensions, so let's make sure we cut it out
_hash = Name.GetBSAHash(""); _hash = Name.GetBSAHash();
_fileCount = (uint) files.Count(); _fileCount = (uint) files.Count();
_nameBytes = folderName.ToBZString(_bsa.HeaderType); _nameBytes = folderName.ToBZString(_bsa.HeaderType);
_recordSize = sizeof(ulong) + sizeof(uint) + sizeof(uint); _recordSize = sizeof(ulong) + sizeof(uint) + sizeof(uint);
@ -170,7 +168,7 @@ namespace Compression.BSA
public ulong Hash => _hash; public ulong Hash => _hash;
public string Name { get; } public RelativePath Name { get; }
public ulong SelfSize public ulong SelfSize
{ {
@ -232,17 +230,17 @@ namespace Compression.BSA
internal byte[] _nameBytes; internal byte[] _nameBytes;
private long _offsetOffset; private long _offsetOffset;
internal int _originalSize; internal int _originalSize;
internal string _path; internal RelativePath _path;
private byte[] _pathBSBytes; private byte[] _pathBSBytes;
internal byte[] _pathBytes; internal byte[] _pathBytes;
private Stream _srcData; private Stream _srcData;
public static FileEntry Create(BSABuilder bsa, string path, Stream src, bool flipCompression) public static FileEntry Create(BSABuilder bsa, RelativePath path, Stream src, bool flipCompression)
{ {
var entry = new FileEntry(); var entry = new FileEntry();
entry._bsa = bsa; entry._bsa = bsa;
entry._path = path.ToLowerInvariant(); entry._path = path;
entry._name = System.IO.Path.GetFileName(entry._path); entry._name = (string)entry._path.FileName;
entry._hash = entry._name.GetBSAHash(); entry._hash = entry._name.GetBSAHash();
entry._nameBytes = entry._name.ToTermString(bsa.HeaderType); entry._nameBytes = entry._name.ToTermString(bsa.HeaderType);
entry._pathBytes = entry._path.ToTermString(bsa.HeaderType); entry._pathBytes = entry._path.ToTermString(bsa.HeaderType);
@ -267,7 +265,7 @@ namespace Compression.BSA
} }
} }
public string Path => _path; public RelativePath Path => _path;
public bool FlipCompression => _flipCompression; public bool FlipCompression => _flipCompression;

5
Compression.BSA/BSADispatch.cs
View File

@ -1,14 +1,15 @@
using System.IO; using System.IO;
using System.Text; using System.Text;
using Wabbajack.Common;
namespace Compression.BSA namespace Compression.BSA
{ {
public static class BSADispatch public static class BSADispatch
{ {
public static IBSAReader OpenRead(string filename) public static IBSAReader OpenRead(AbsolutePath filename)
{ {
var fourcc = ""; var fourcc = "";
using (var file = File.OpenRead(filename)) using (var file = filename.OpenRead())
{ {
fourcc = Encoding.ASCII.GetString(new BinaryReader(file).ReadBytes(4)); fourcc = Encoding.ASCII.GetString(new BinaryReader(file).ReadBytes(4));
} }

12
Compression.BSA/BSAReader.cs
View File

@ -4,6 +4,7 @@ using System.IO;
using System.Text; using System.Text;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using K4os.Compression.LZ4.Streams; using K4os.Compression.LZ4.Streams;
using Wabbajack.Common;
using File = Alphaleonis.Win32.Filesystem.File; using File = Alphaleonis.Win32.Filesystem.File;
namespace Compression.BSA namespace Compression.BSA
@ -52,7 +53,7 @@ namespace Compression.BSA
internal uint _archiveFlags; internal uint _archiveFlags;
internal uint _fileCount; internal uint _fileCount;
internal uint _fileFlags; internal uint _fileFlags;
internal string _fileName; internal AbsolutePath _fileName;
internal uint _folderCount; internal uint _folderCount;
internal uint _folderRecordOffset; internal uint _folderRecordOffset;
private List<FolderRecord> _folders; private List<FolderRecord> _folders;
@ -63,7 +64,7 @@ namespace Compression.BSA
internal uint _totalFolderNameLength; internal uint _totalFolderNameLength;
internal uint _version; internal uint _version;
public BSAReader(string filename) : this(File.OpenRead(filename)) public BSAReader(AbsolutePath filename) : this(filename.OpenRead())
{ {
_fileName = filename; _fileName = filename;
} }
@ -270,12 +271,11 @@ namespace Compression.BSA
src.BaseStream.Position = old_pos; src.BaseStream.Position = old_pos;
} }
public string Path public RelativePath Path
{ {
get get
{ {
if (string.IsNullOrEmpty(Folder.Name)) return _name; return string.IsNullOrEmpty(Folder.Name) ? new RelativePath(_name) : new RelativePath(Folder.Name + "\\" + _name);
return Folder.Name + "\\" + _name;
} }
} }
@ -304,7 +304,7 @@ namespace Compression.BSA
public void CopyDataTo(Stream output) public void CopyDataTo(Stream output)
{ {
using (var in_file = File.OpenRead(_bsa._fileName)) using (var in_file = _bsa._fileName.OpenRead())
using (var rdr = new BinaryReader(in_file)) using (var rdr = new BinaryReader(in_file))
{ {
rdr.BaseStream.Position = _dataOffset; rdr.BaseStream.Position = _dataOffset;

5
Compression.BSA/IBSAReader.cs
View File

@ -1,6 +1,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using Wabbajack.Common;
namespace Compression.BSA namespace Compression.BSA
{ {
@ -31,7 +32,7 @@ namespace Compression.BSA
public class FileStateObject public class FileStateObject
{ {
public int Index { get; set; } public int Index { get; set; }
public string Path { get; set; } public RelativePath Path { get; set; }
} }
public interface IFile public interface IFile
@ -39,7 +40,7 @@ namespace Compression.BSA
/// <summary> /// <summary>
/// The path of the file inside the archive /// The path of the file inside the archive
/// </summary> /// </summary>
string Path { get; } RelativePath Path { get; }
/// <summary> /// <summary>
/// The uncompressed file size /// The uncompressed file size

2
Compression.BSA/TES3Builder.cs
View File

@ -48,7 +48,7 @@ namespace Compression.BSA
{ {
if (bw.BaseStream.Position != orgPos + state.NameOffset) if (bw.BaseStream.Position != orgPos + state.NameOffset)
throw new InvalidDataException("Offsets don't match when writing TES3 BSA"); throw new InvalidDataException("Offsets don't match when writing TES3 BSA");
bw.Write(Encoding.ASCII.GetBytes(state.Path)); bw.Write(Encoding.ASCII.GetBytes((string)state.Path));
bw.Write((byte)0); bw.Write((byte)0);
} }

13
Compression.BSA/TES3Reader.cs
View File

@ -2,6 +2,7 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Text; using System.Text;
using Wabbajack.Common;
namespace Compression.BSA namespace Compression.BSA
{ {
@ -13,12 +14,12 @@ namespace Compression.BSA
private uint _fileCount; private uint _fileCount;
private TES3FileEntry[] _files; private TES3FileEntry[] _files;
internal long _dataOffset; internal long _dataOffset;
internal string _filename; internal AbsolutePath _filename;
public TES3Reader(string filename) public TES3Reader(AbsolutePath filename)
{ {
_filename = filename; _filename = filename;
using var fs = File.OpenRead(filename); using var fs = filename.OpenRead();
using var br = new BinaryReader(fs); using var br = new BinaryReader(fs);
_versionNumber = br.ReadUInt32(); _versionNumber = br.ReadUInt32();
_hashTableOffset = br.ReadUInt32(); _hashTableOffset = br.ReadUInt32();
@ -46,7 +47,7 @@ namespace Compression.BSA
for (int i = 0; i < _fileCount; i++) for (int i = 0; i < _fileCount; i++)
{ {
br.BaseStream.Position = origPos + _files[i].NameOffset; br.BaseStream.Position = origPos + _files[i].NameOffset;
_files[i].Path = br.ReadStringTerm(VersionType.TES3); _files[i].Path = new RelativePath(br.ReadStringTerm(VersionType.TES3));
} }
br.BaseStream.Position = _hashTableOffset + 12; br.BaseStream.Position = _hashTableOffset + 12;
@ -95,7 +96,7 @@ namespace Compression.BSA
public class TES3FileEntry : IFile public class TES3FileEntry : IFile
{ {
public string Path { get; set; } public RelativePath Path { get; set; }
public uint Size { get; set; } public uint Size { get; set; }
public FileStateObject State => public FileStateObject State =>
new TES3FileState new TES3FileState
@ -111,7 +112,7 @@ namespace Compression.BSA
public void CopyDataTo(Stream output) public void CopyDataTo(Stream output)
{ {
using var fs = File.OpenRead(Archive._filename); using var fs = Archive._filename.OpenRead();
fs.Position = Archive._dataOffset + Offset; fs.Position = Archive._dataOffset + Offset;
fs.CopyToLimit(output, (int)Size); fs.CopyToLimit(output, (int)Size);
} }

19
Compression.BSA/Utils.cs
View File

@ -2,6 +2,7 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Text; using System.Text;
using Wabbajack.Common;
using Path = Alphaleonis.Win32.Filesystem.Path; using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Compression.BSA namespace Compression.BSA
@ -64,9 +65,9 @@ namespace Compression.BSA
/// </summary> /// </summary>
/// <param name="val"></param> /// <param name="val"></param>
/// <returns></returns> /// <returns></returns>
public static byte[] ToBZString(this string val, VersionType version) public static byte[] ToBZString(this RelativePath val, VersionType version)
{ {
var b = GetEncoding(version).GetBytes(val); var b = GetEncoding(version).GetBytes((string)val);
var b2 = new byte[b.Length + 2]; var b2 = new byte[b.Length + 2];
b.CopyTo(b2, 1); b.CopyTo(b2, 1);
b2[0] = (byte) (b.Length + 1); b2[0] = (byte) (b.Length + 1);
@ -78,9 +79,9 @@ namespace Compression.BSA
/// </summary> /// </summary>
/// <param name="val"></param> /// <param name="val"></param>
/// <returns></returns> /// <returns></returns>
public static byte[] ToBSString(this string val) public static byte[] ToBSString(this RelativePath val)
{ {
var b = Encoding.ASCII.GetBytes(val); var b = Encoding.ASCII.GetBytes((string)val);
var b2 = new byte[b.Length + 1]; var b2 = new byte[b.Length + 1];
b.CopyTo(b2, 1); b.CopyTo(b2, 1);
b2[0] = (byte) b.Length; b2[0] = (byte) b.Length;
@ -102,12 +103,22 @@ namespace Compression.BSA
return b2; return b2;
} }
public static byte[] ToTermString(this RelativePath val, VersionType version)
{
return ((string)val).ToTermString(version);
}
public static ulong GetBSAHash(this string name) public static ulong GetBSAHash(this string name)
{ {
name = name.Replace('/', '\\'); name = name.Replace('/', '\\');
return GetBSAHash(Path.ChangeExtension(name, null), Path.GetExtension(name)); return GetBSAHash(Path.ChangeExtension(name, null), Path.GetExtension(name));
} }
public static ulong GetBSAHash(this RelativePath name)
{
return ((string)name).GetBSAHash();
}
public static ulong GetBSAHash(this string name, string ext) public static ulong GetBSAHash(this string name, string ext)
{ {
name = name.ToLowerInvariant(); name = name.ToLowerInvariant();

12
Wabbajack.Common/Consts.cs
View File

@ -1,5 +1,6 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq;
using System.Reflection; using System.Reflection;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using Alphaleonis.Win32.Filesystem; using Alphaleonis.Win32.Filesystem;
@ -20,17 +21,22 @@ namespace Wabbajack.Common
public static string MegaPrefix = "https://mega.nz/#!"; public static string MegaPrefix = "https://mega.nz/#!";
public static HashSet<string> SupportedArchives = new HashSet<string>(StringComparer.OrdinalIgnoreCase) {".zip", ".rar", ".7z", ".7zip", ".fomod", ".omod", ".exe", ".dat"}; public static readonly HashSet<Extension> SupportedArchives = new[]{".zip", ".rar", ".7z", ".7zip", ".fomod", ".omod", ".exe", ".dat"}
.Select(s => new Extension(s)).ToHashSet();
// HashSet with archive extensions that need to be tested before extraction // HashSet with archive extensions that need to be tested before extraction
public static HashSet<string> TestArchivesBeforeExtraction = new HashSet<string>(StringComparer.OrdinalIgnoreCase) {".dat"}; public static HashSet<Extension> TestArchivesBeforeExtraction = new []{".dat"}.Select(s => new Extension(s)).ToHashSet();
public static HashSet<string> SupportedBSAs = new HashSet<string>(StringComparer.OrdinalIgnoreCase) {".bsa", ".ba2"}; public static readonly HashSet<Extension> SupportedBSAs = new[] {".bsa", ".ba2"}
.Select(s => new Extension(s)).ToHashSet();
public static HashSet<string> ConfigFileExtensions = new HashSet<string>(StringComparer.OrdinalIgnoreCase) {".json", ".ini", ".yml", ".xml"}; public static HashSet<string> ConfigFileExtensions = new HashSet<string>(StringComparer.OrdinalIgnoreCase) {".json", ".ini", ".yml", ".xml"};
public static HashSet<string> ESPFileExtensions = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { ".esp", ".esm", ".esl"}; public static HashSet<string> ESPFileExtensions = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { ".esp", ".esm", ".esl"};
public static HashSet<string> AssetFileExtensions = new HashSet<string>(StringComparer.OrdinalIgnoreCase) {".dds", ".tga", ".nif", ".psc", ".pex"}; public static HashSet<string> AssetFileExtensions = new HashSet<string>(StringComparer.OrdinalIgnoreCase) {".dds", ".tga", ".nif", ".psc", ".pex"};
public static readonly Extension EXE = new Extension(".exe");
public static readonly Extension OMOD = new Extension(".omod");
public static string NexusCacheDirectory = "nexus_link_cache"; public static string NexusCacheDirectory = "nexus_link_cache";
public static string WABBAJACK_INCLUDE = "WABBAJACK_INCLUDE"; public static string WABBAJACK_INCLUDE = "WABBAJACK_INCLUDE";

14
Wabbajack.Common/Enumerable.cs Normal file
View File

@ -0,0 +1,14 @@
using System.Collections.Generic;
namespace Wabbajack.Common
{
public static partial class Utils
{
public static IEnumerable<T> Cons<T>(this IEnumerable<T> coll, T next)
{
yield return next;
foreach (var itm in coll) yield return itm;
}
}
}

22
Wabbajack.Common/Hash.cs
View File

@ -111,13 +111,13 @@ namespace Wabbajack.Common
return sha.Hash.ToHex(); return sha.Hash.ToHex();
} }
public static Hash FileHash(this string file, bool nullOnIoError = false) public static Hash FileHash(this AbsolutePath file, bool nullOnIoError = false)
{ {
try try
{ {
using var fs = File.OpenRead(file); using var fs = file.OpenRead();
var config = new xxHashConfig {HashSizeInBits = 64}; var config = new xxHashConfig {HashSizeInBits = 64};
using var f = new StatusFileStream(fs, $"Hashing {Path.GetFileName(file)}"); using var f = new StatusFileStream(fs, $"Hashing {(string)file.FileName}");
return new Hash(BitConverter.ToUInt64(xxHashFactory.Instance.Create(config).ComputeHash(f).Hash)); return new Hash(BitConverter.ToUInt64(xxHashFactory.Instance.Create(config).ComputeHash(f).Hash));
} }
catch (IOException) catch (IOException)
@ -127,7 +127,7 @@ namespace Wabbajack.Common
} }
} }
public static Hash FileHashCached(this string file, bool nullOnIoError = false) public static Hash FileHashCached(this AbsolutePath file, bool nullOnIoError = false)
{ {
if (TryGetHashCache(file, out var foundHash)) return foundHash; if (TryGetHashCache(file, out var foundHash)) return foundHash;
@ -137,7 +137,7 @@ namespace Wabbajack.Common
return hash; return hash;
} }
public static bool TryGetHashCache(string file, out Hash hash) public static bool TryGetHashCache(AbsolutePath file, out Hash hash)
{ {
var hashFile = file + Consts.HashFileExtension; var hashFile = file + Consts.HashFileExtension;
hash = Hash.Empty; hash = Hash.Empty;
@ -151,24 +151,24 @@ namespace Wabbajack.Common
if (version != HashCacheVersion) return false; if (version != HashCacheVersion) return false;
var lastModified = br.ReadUInt64(); var lastModified = br.ReadUInt64();
if (lastModified != File.GetLastWriteTimeUtc(file).AsUnixTime()) return false; if (lastModified != file.LastModifiedUtc.AsUnixTime()) return false;
hash = new Hash(br.ReadUInt64()); hash = new Hash(br.ReadUInt64());
return true; return true;
} }
private const uint HashCacheVersion = 0x01; private const uint HashCacheVersion = 0x01;
private static void WriteHashCache(string file, Hash hash) private static void WriteHashCache(AbsolutePath file, Hash hash)
{ {
using var fs = File.Create(file + Consts.HashFileExtension); using var fs = File.Create(file + Consts.HashFileExtension);
using var bw = new BinaryWriter(fs); using var bw = new BinaryWriter(fs);
bw.Write(HashCacheVersion); bw.Write(HashCacheVersion);
var lastModified = File.GetLastWriteTimeUtc(file).AsUnixTime(); var lastModified = file.LastModifiedUtc.AsUnixTime();
bw.Write(lastModified); bw.Write(lastModified);
bw.Write((ulong)hash); bw.Write((ulong)hash);
} }
public static async Task<Hash> FileHashCachedAsync(this string file, bool nullOnIOError = false) public static async Task<Hash> FileHashCachedAsync(this AbsolutePath file, bool nullOnIOError = false)
{ {
if (TryGetHashCache(file, out var foundHash)) return foundHash; if (TryGetHashCache(file, out var foundHash)) return foundHash;
@ -178,11 +178,11 @@ namespace Wabbajack.Common
return hash; return hash;
} }
public static async Task<Hash> FileHashAsync(this string file, bool nullOnIOError = false) public static async Task<Hash> FileHashAsync(this AbsolutePath file, bool nullOnIOError = false)
{ {
try try
{ {
await using var fs = File.OpenRead(file); await using var fs = file.OpenRead();
var config = new xxHashConfig {HashSizeInBits = 64}; var config = new xxHashConfig {HashSizeInBits = 64};
var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(fs); var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(fs);
return new Hash(BitConverter.ToUInt64(value.Hash)); return new Hash(BitConverter.ToUInt64(value.Hash));

393
Wabbajack.Common/Paths.cs Normal file
View File

@ -0,0 +1,393 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Directory = System.IO.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Common
{
public class AbstractPath
{
public RelativePath FileName
{
get
{
switch (this)
{
case AbsolutePath abs:
return abs.FileName;
case RelativePath rel:
return rel.FileName;
}
return null;
}
}
}
public class AbsolutePath : AbstractPath
{
#region ObjectEquality
protected bool Equals(AbsolutePath other)
{
return _path == other._path;
}
public override bool Equals(object obj)
{
if (ReferenceEquals(null, obj))
{
return false;
}
if (ReferenceEquals(this, obj))
{
return true;
}
if (obj.GetType() != this.GetType())
{
return false;
}
return Equals((AbsolutePath) obj);
}
#endregion
public override int GetHashCode()
{
return (_path != null ? _path.GetHashCode() : 0);
}
private readonly string _path;
private Extension _extension;
public AbsolutePath(string path)
{
_path = path.ToLowerInvariant().Replace("/", "\\").TrimEnd('\\');
ValidateAbsolutePath();
}
public AbsolutePath(string path, bool skipValidation)
{
_path = path.ToLowerInvariant().Replace("/", "\\").TrimEnd('\\');
if (!skipValidation)
ValidateAbsolutePath();
}
public AbsolutePath(AbsolutePath path)
{
_path = path._path;
}
private void ValidateAbsolutePath()
{
if (Path.IsPathRooted(_path)) return;
throw new InvalidDataException($"Absolute path must be absolute");
}
public Extension Extension
{
get
{
if (_extension != null) return _extension;
var extension = Path.GetExtension(_path);
if (string.IsNullOrEmpty(extension))
return null;
_extension = (Extension)extension;
return _extension;
}
}
public FileStream OpenRead()
{
return File.OpenRead(_path);
}
public FileStream Create()
{
return File.Create(_path);
}
public FileStream OpenWrite()
{
return File.OpenWrite(_path);
}
public async Task WriteAllTextAsync(string text)
{
await using var fs = File.Create(_path);
await fs.WriteAsync(Encoding.UTF8.GetBytes(text));
}
public bool Exists => File.Exists(_path) || Directory.Exists(_path);
public bool IsFile => File.Exists(_path);
public bool IsDirectory => Directory.Exists(_path);
public long Size => (new FileInfo(_path)).Length;
public DateTime LastModified => File.GetLastWriteTime(_path);
public DateTime LastModifiedUtc => File.GetLastWriteTimeUtc(_path);
public AbsolutePath Parent => (AbsolutePath)Path.GetDirectoryName(_path);
public RelativePath FileName => (RelativePath)Path.GetFileName(_path);
public void Copy(AbsolutePath otherPath)
{
File.Copy(_path, otherPath._path);
}
public void Move(AbsolutePath otherPath, bool overwrite = false)
{
File.Move(_path, otherPath._path, overwrite ? MoveOptions.ReplaceExisting : MoveOptions.None);
}
public RelativePath RelativeTo(AbsolutePath p)
{
if (_path.Substring(0, p._path.Length + 1) != p._path + "\\")
throw new InvalidDataException("Not a parent path");
return new RelativePath(_path.Substring(p._path.Length + 1));
}
public async Task<string> ReadAllTextAsync()
{
await using var fs = File.OpenRead(_path);
return Encoding.UTF8.GetString(await fs.ReadAllAsync());
}
/// <summary>
/// Assuming the path is a folder, enumerate all the files in the folder
/// </summary>
/// <param name="recursive">if true, also returns files in sub-folders</param>
/// <returns></returns>
public IEnumerable<AbsolutePath> EnumerateFiles(bool recursive = true)
{
return Directory
.EnumerateFiles(_path, "*", recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)
.Select(path => new AbsolutePath(path, true));
}
#region Operators
public static explicit operator string(AbsolutePath path)
{
return path._path;
}
public static explicit operator AbsolutePath(string path)
{
return !Path.IsPathRooted(path) ? ((RelativePath)path).RelativeToEntryPoint() : new AbsolutePath(path);
}
public static bool operator ==(AbsolutePath a, AbsolutePath b)
{
return a._path == b._path;
}
public static bool operator !=(AbsolutePath a, AbsolutePath b)
{
return a._path != b._path;
}
#endregion
public void CreateDirectory()
{
Directory.CreateDirectory(_path);
}
}
public class RelativePath : AbstractPath
{
private readonly string _path;
private Extension _extension;
public RelativePath(string path)
{
_path = path.ToLowerInvariant().Replace("/", "\\").Trim('\\');
Validate();
}
public static RelativePath RandomFileName()
{
return (RelativePath)Guid.NewGuid().ToString();
}
private void Validate()
{
if (Path.IsPathRooted(_path))
throw new InvalidDataException("Cannot create relative path from absolute path string");
}
public AbsolutePath RelativeTo(AbsolutePath abs)
{
return new AbsolutePath(Path.Combine((string)abs, _path));
}
public AbsolutePath RelativeToEntryPoint()
{
return RelativeTo(((AbsolutePath)Assembly.GetEntryAssembly().Location).Parent);
}
public AbsolutePath RelativeToWorkingDirectory()
{
return RelativeTo((AbsolutePath)Directory.GetCurrentDirectory());
}
public static explicit operator string(RelativePath path)
{
return path._path;
}
public static explicit operator RelativePath(string path)
{
return new RelativePath(path);
}
public AbsolutePath RelativeToSystemDirectory()
{
return RelativeTo((AbsolutePath)Environment.SystemDirectory);
}
public RelativePath Parent => (RelativePath)Path.GetDirectoryName(_path);
}
public class Extension
{
#region ObjectEquality
protected bool Equals(Extension other)
{
return _extension == other._extension;
}
public override bool Equals(object obj)
{
if (ReferenceEquals(null, obj))
{
return false;
}
if (ReferenceEquals(this, obj))
{
return true;
}
if (obj.GetType() != this.GetType())
{
return false;
}
return Equals((Extension) obj);
}
public override int GetHashCode()
{
return (_extension != null ? _extension.GetHashCode() : 0);
}
#endregion
private readonly string _extension;
public Extension(string extension)
{
_extension = string.Intern(extension);
Validate();
}
private void Validate()
{
if (!_extension.StartsWith("."))
throw new InvalidDataException($"Extensions must start with '.'");
}
public static explicit operator string(Extension path)
{
return path._extension;
}
public static explicit operator Extension(string path)
{
return new Extension(path);
}
public static bool operator ==(Extension a, Extension b)
{
// Super fast comparison because extensions are interned
return ReferenceEquals(a._extension, b._extension);
}
public static bool operator !=(Extension a, Extension b)
{
return !ReferenceEquals(a._extension, b._extension);
}
}
public class HashRelativePath
{
public Hash BaseHash { get; }
public RelativePath[] Paths { get; }
public string ToString()
{
return string.Join("|", Paths.Select(t => t.ToString()).Cons(BaseHash.ToString()));
}
public static bool operator ==(HashRelativePath a, HashRelativePath b)
{
if (a.BaseHash != b.BaseHash || a.Paths.Length == b.Paths.Length)
return false;
for (int idx = 0; idx < a.Paths.Length; idx += 1)
if (a.Paths[idx] != b.Paths[idx])
return false;
return true;
}
public static bool operator !=(HashRelativePath a, HashRelativePath b)
{
return !(a == b);
}
}
public class FullPath
{
public AbsolutePath Base { get; }
public RelativePath[] Paths { get; }
public FullPath(AbsolutePath basePath, RelativePath[] paths)
{
Base = basePath;
Paths = Paths;
}
public string ToString()
{
return string.Join("|", Paths.Select(t => t.ToString()).Cons(Base.ToString()));
}
public static bool operator ==(FullPath a, FullPath b)
{
if (a.Base != b.Base || a.Paths.Length == b.Paths.Length)
return false;
for (int idx = 0; idx < a.Paths.Length; idx += 1)
if (a.Paths[idx] != b.Paths[idx])
return false;
return true;
}
public static bool operator !=(FullPath a, FullPath b)
{
return !(a == b);
}
}
}

10
Wabbajack.Common/StatusFeed/Errors/7zipReturnError.cs
View File

@ -8,21 +8,21 @@ namespace Wabbajack.Common.StatusFeed.Errors
{ {
public class _7zipReturnError : AErrorMessage public class _7zipReturnError : AErrorMessage
{ {
public string Destination { get; } public AbsolutePath Destination { get; }
public string Filename; public AbsolutePath Filename { get; }
public int Code; public int Code;
public string _7zip_output; public string _7zip_output;
public override string ShortDescription => $"7Zip returned an error while executing"; public override string ShortDescription => $"7Zip returned an error while executing";
public override string ExtendedDescription => public override string ExtendedDescription =>
$@"7Zip.exe should always return 0 when it finishes executing. While extracting {Filename} 7Zip encountered some error and $@"7Zip.exe should always return 0 when it finishes executing. While extracting {(string)Filename} 7Zip encountered some error and
instead returned {Code} which indicates there was an error. The archive might be corrupt or in a format that 7Zip cannot handle. Please verify the file is valid and that you instead returned {Code} which indicates there was an error. The archive might be corrupt or in a format that 7Zip cannot handle. Please verify the file is valid and that you
haven't run out of disk space in the {Destination} folder. haven't run out of disk space in the {(string)Destination} folder.
7Zip Output: 7Zip Output:
{_7zip_output}"; {_7zip_output}";
public _7zipReturnError(int code, string filename, string destination, string output) public _7zipReturnError(int code, AbsolutePath filename, AbsolutePath destination, string output)
{ {
Code = code; Code = code;
Filename = filename; Filename = filename;

26
Wabbajack.Common/Util/TempFolder.cs
View File

@ -9,36 +9,32 @@ namespace Wabbajack.Common
{ {
public class TempFolder : IDisposable public class TempFolder : IDisposable
{ {
public DirectoryInfo Dir { get; private set; } public AbsolutePath Dir { get; }
public bool DeleteAfter = true; public bool DeleteAfter = true;
public TempFolder(bool deleteAfter = true) public TempFolder(bool deleteAfter = true)
{ {
this.Dir = new DirectoryInfo(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName())); Dir = new AbsolutePath(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()));
this.Dir.Create(); if (!Dir.Exists)
this.DeleteAfter = deleteAfter; Dir.CreateDirectory();
DeleteAfter = deleteAfter;
} }
public TempFolder(DirectoryInfo dir, bool deleteAfter = true) public TempFolder(AbsolutePath dir, bool deleteAfter = true)
{ {
this.Dir = dir; Dir = dir;
if (!dir.Exists) if (!dir.Exists)
{ {
this.Dir.Create(); Dir.Create();
} }
this.DeleteAfter = deleteAfter; DeleteAfter = deleteAfter;
}
public TempFolder(string addedFolderPath, bool deleteAfter = true)
: this(new DirectoryInfo(Path.Combine(Path.GetTempPath(), addedFolderPath)), deleteAfter: deleteAfter)
{
} }
public void Dispose() public void Dispose()
{ {
if (DeleteAfter) if (DeleteAfter && Dir.Exists)
{ {
Utils.DeleteDirectory(this.Dir.FullName); Utils.DeleteDirectory(Dir);
} }
} }
} }

4
Wabbajack.Common/Utils.cs
View File

@ -1039,12 +1039,12 @@ namespace Wabbajack.Common
/// delete a folder. If you don't like this code, it's unlikely to change without a ton of testing. /// delete a folder. If you don't like this code, it's unlikely to change without a ton of testing.
/// </summary> /// </summary>
/// <param name="path"></param> /// <param name="path"></param>
public static async void DeleteDirectory(string path) public static async void DeleteDirectory(AbsolutePath path)
{ {
var process = new ProcessHelper var process = new ProcessHelper
{ {
Path = "cmd.exe", Path = "cmd.exe",
Arguments = new object[] {"/c", "del", "/f", "/q", "/s", $"\"{path}\"", "&&", "rmdir", "/q", "/s", $"\"{path}\""}, Arguments = new object[] {"/c", "del", "/f", "/q", "/s", $"\"{(string)path}\"", "&&", "rmdir", "/q", "/s", $"\"{(string)path}\""},
}; };
var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output) var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p => .ForEachAsync(p =>

80
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -22,12 +22,12 @@ namespace Wabbajack.VirtualFileSystem
static Context() static Context()
{ {
Utils.Log("Cleaning VFS, this may take a bit of time"); Utils.Log("Cleaning VFS, this may take a bit of time");
Utils.DeleteDirectory(_stagingFolder); Utils.DeleteDirectory(StagingFolder);
} }
public const ulong FileVersion = 0x03; public const ulong FileVersion = 0x03;
public const string Magic = "WABBAJACK VFS FILE"; public const string Magic = "WABBAJACK VFS FILE";
private static readonly string _stagingFolder = "vfs_staging"; private static readonly AbsolutePath StagingFolder = ((RelativePath)"vfs_staging").RelativeToWorkingDirectory();
public IndexRoot Index { get; private set; } = IndexRoot.Empty; public IndexRoot Index { get; private set; } = IndexRoot.Empty;
/// <summary> /// <summary>
@ -47,23 +47,18 @@ namespace Wabbajack.VirtualFileSystem
Queue = queue; Queue = queue;
UseExtendedHashes = extendedHashes; UseExtendedHashes = extendedHashes;
} }
public static TemporaryDirectory GetTemporaryFolder()
public TemporaryDirectory GetTemporaryFolder()
{ {
return new TemporaryDirectory(Path.Combine(_stagingFolder, Guid.NewGuid().ToString())); return new TemporaryDirectory(((RelativePath)Guid.NewGuid().ToString()).RelativeTo(StagingFolder));
} }
public async Task<IndexRoot> AddRoot(string root) public async Task<IndexRoot> AddRoot(AbsolutePath root)
{ {
if (!Path.IsPathRooted(root)) var filtered = Index.AllFiles.Where(file => file.IsNative && ((AbsolutePath) file.Name).Exists).ToList();
throw new InvalidDataException($"Path is not absolute: {root}");
var filtered = Index.AllFiles.Where(file => File.Exists(file.Name)).ToList();
var byPath = filtered.ToImmutableDictionary(f => f.Name); var byPath = filtered.ToImmutableDictionary(f => f.Name);
var filesToIndex = Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive).Distinct().ToList(); var filesToIndex = root.EnumerateFiles().Distinct().ToList();
var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing {root}", filesToIndex.Count)); var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing {root}", filesToIndex.Count));
@ -72,8 +67,7 @@ namespace Wabbajack.VirtualFileSystem
{ {
if (byPath.TryGetValue(f, out var found)) if (byPath.TryGetValue(f, out var found))
{ {
var fi = new FileInfo(f); if (found.LastModified == f.LastModifiedUtc.AsUnixTime() && found.Size == f.Size)
if (found.LastModified == fi.LastWriteTimeUtc.Ticks && found.Size == fi.Length)
return found; return found;
} }
@ -90,27 +84,21 @@ namespace Wabbajack.VirtualFileSystem
return newIndex; return newIndex;
} }
public async Task<IndexRoot> AddRoots(List<string> roots) public async Task<IndexRoot> AddRoots(List<AbsolutePath> roots)
{ {
if (!roots.All(p => Path.IsPathRooted(p))) var native = Index.AllFiles.Where(file => file.IsNative).ToDictionary(file => file.StagedPath);
throw new InvalidDataException($"Paths are not absolute");
var filtered = Index.AllFiles.Where(file => File.Exists(file.Name)).ToList(); var filtered = Index.AllFiles.Where(file => ((AbsolutePath)file.Name).Exists).ToList();
var byPath = filtered.ToImmutableDictionary(f => f.Name); var filesToIndex = roots.SelectMany(root => root.EnumerateFiles()).ToList();
var filesToIndex = roots.SelectMany(root => Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive)).ToList();
var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing roots", filesToIndex.Count));
var allFiles = await filesToIndex var allFiles = await filesToIndex
.PMap(Queue, async f => .PMap(Queue, async f =>
{ {
Utils.Status($"Indexing {Path.GetFileName(f)}"); Utils.Status($"Indexing {Path.GetFileName((string)f)}");
if (byPath.TryGetValue(f, out var found)) if (native.TryGetValue(f, out var found))
{ {
var fi = new FileInfo(f); if (found.LastModified == f.LastModifiedUtc.AsUnixTime() && found.Size == f.Size)
if (found.LastModified == fi.LastWriteTimeUtc.Ticks && found.Size == fi.Length)
return found; return found;
} }
@ -224,7 +212,7 @@ namespace Wabbajack.VirtualFileSystem
foreach (var group in grouped) foreach (var group in grouped)
{ {
var tmpPath = Path.Combine(_stagingFolder, Guid.NewGuid().ToString()); var tmpPath = Path.Combine(StagingFolder, Guid.NewGuid().ToString());
await FileExtractor.ExtractAll(Queue, group.Key.StagedPath, tmpPath); await FileExtractor.ExtractAll(Queue, group.Key.StagedPath, tmpPath);
paths.Add(tmpPath); paths.Add(tmpPath);
foreach (var file in group) foreach (var file in group)
@ -354,10 +342,10 @@ namespace Wabbajack.VirtualFileSystem
public static IndexRoot Empty = new IndexRoot(); public static IndexRoot Empty = new IndexRoot();
public IndexRoot(ImmutableList<VirtualFile> aFiles, public IndexRoot(ImmutableList<VirtualFile> aFiles,
ImmutableDictionary<string, VirtualFile> byFullPath, ImmutableDictionary<FullPath, VirtualFile> byFullPath,
ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> byHash, ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> byHash,
ImmutableDictionary<string, VirtualFile> byRoot, ImmutableDictionary<AbsolutePath, VirtualFile> byRoot,
ImmutableDictionary<string, ImmutableStack<VirtualFile>> byName) ImmutableDictionary<AbstractPath, ImmutableStack<VirtualFile>> byName)
{ {
AllFiles = aFiles; AllFiles = aFiles;
ByFullPath = byFullPath; ByFullPath = byFullPath;
@ -369,18 +357,18 @@ namespace Wabbajack.VirtualFileSystem
public IndexRoot() public IndexRoot()
{ {
AllFiles = ImmutableList<VirtualFile>.Empty; AllFiles = ImmutableList<VirtualFile>.Empty;
ByFullPath = ImmutableDictionary<string, VirtualFile>.Empty; ByFullPath = ImmutableDictionary<FullPath, VirtualFile>.Empty;
ByHash = ImmutableDictionary<Hash, ImmutableStack<VirtualFile>>.Empty; ByHash = ImmutableDictionary<Hash, ImmutableStack<VirtualFile>>.Empty;
ByRootPath = ImmutableDictionary<string, VirtualFile>.Empty; ByRootPath = ImmutableDictionary<AbsolutePath, VirtualFile>.Empty;
ByName = ImmutableDictionary<string, ImmutableStack<VirtualFile>>.Empty; ByName = ImmutableDictionary<AbstractPath, ImmutableStack<VirtualFile>>.Empty;
} }
public ImmutableList<VirtualFile> AllFiles { get; } public ImmutableList<VirtualFile> AllFiles { get; }
public ImmutableDictionary<string, VirtualFile> ByFullPath { get; } public ImmutableDictionary<FullPath, VirtualFile> ByFullPath { get; }
public ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> ByHash { get; } public ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> ByHash { get; }
public ImmutableDictionary<string, ImmutableStack<VirtualFile>> ByName { get; set; } public ImmutableDictionary<AbstractPath, ImmutableStack<VirtualFile>> ByName { get; set; }
public ImmutableDictionary<string, VirtualFile> ByRootPath { get; } public ImmutableDictionary<AbsolutePath, VirtualFile> ByRootPath { get; }
public async Task<IndexRoot> Integrate(ICollection<VirtualFile> files) public async Task<IndexRoot> Integrate(ICollection<VirtualFile> files)
{ {
@ -397,7 +385,7 @@ namespace Wabbajack.VirtualFileSystem
var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToGroupedImmutableDictionary(f => f.Name)); .ToGroupedImmutableDictionary(f => f.Name));
var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.Name)); var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.Name as AbsolutePath));
var result = new IndexRoot(allFiles, var result = new IndexRoot(allFiles,
await byFullPath, await byFullPath,
@ -408,27 +396,27 @@ namespace Wabbajack.VirtualFileSystem
return result; return result;
} }
public VirtualFile FileForArchiveHashPath(string[] argArchiveHashPath) public VirtualFile FileForArchiveHashPath(HashRelativePath argArchiveHashPath)
{ {
var cur = ByHash[Hash.FromBase64(argArchiveHashPath[0])].First(f => f.Parent == null); var cur = ByHash[argArchiveHashPath.BaseHash].First(f => f.Parent == null);
return argArchiveHashPath.Skip(1).Aggregate(cur, (current, itm) => ByName[itm].First(f => f.Parent == current)); return argArchiveHashPath.Paths.Aggregate(cur, (current, itm) => ByName[itm].First(f => f.Parent == current));
} }
} }
public class TemporaryDirectory : IDisposable public class TemporaryDirectory : IDisposable
{ {
public TemporaryDirectory(string name) public TemporaryDirectory(AbsolutePath name)
{ {
FullName = name; FullName = name;
if (!Directory.Exists(FullName)) if (!FullName.Exists)
Directory.CreateDirectory(FullName); FullName.CreateDirectory();
} }
public string FullName { get; } public AbsolutePath FullName { get; }
public void Dispose() public void Dispose()
{ {
if (Directory.Exists(FullName)) if (FullName.Exists)
Utils.DeleteDirectory(FullName); Utils.DeleteDirectory(FullName);
} }
} }

81
Wabbajack.VirtualFileSystem/FileExtractor.cs
View File

@ -16,15 +16,15 @@ namespace Wabbajack.VirtualFileSystem
public class FileExtractor public class FileExtractor
{ {
public static async Task ExtractAll(WorkQueue queue, string source, string dest) public static async Task ExtractAll(WorkQueue queue, AbsolutePath source, AbsolutePath dest)
{ {
try try
{ {
if (Consts.SupportedBSAs.Any(b => source.ToLower().EndsWith(b))) if (Consts.SupportedBSAs.Contains(source.Extension))
await ExtractAllWithBSA(queue, source, dest); await ExtractAllWithBSA(queue, source, dest);
else if (source.EndsWith(".omod")) else if (source.Extension == Consts.OMOD)
ExtractAllWithOMOD(source, dest); ExtractAllWithOMOD(source, dest);
else if (source.EndsWith(".exe")) else if (source.Extension == Consts.EXE)
ExtractAllWithInno(source, dest); ExtractAllWithInno(source, dest);
else else
ExtractAllWith7Zip(source, dest); ExtractAllWith7Zip(source, dest);
@ -35,14 +35,14 @@ namespace Wabbajack.VirtualFileSystem
} }
} }
private static void ExtractAllWithInno(string source, string dest) private static void ExtractAllWithInno(AbsolutePath source, AbsolutePath dest)
{ {
Utils.Log($"Extracting {Path.GetFileName(source)}"); Utils.Log($"Extracting {(string)source.FileName}");
var info = new ProcessStartInfo var info = new ProcessStartInfo
{ {
FileName = @"Extractors\innounp.exe", FileName = @"Extractors\innounp.exe",
Arguments = $"-x -y -b -d\"{dest}\" \"{source}\"", Arguments = $"-x -y -b -d\"{(string)dest}\" \"{(string)source}\"",
RedirectStandardError = true, RedirectStandardError = true,
RedirectStandardInput = true, RedirectStandardInput = true,
RedirectStandardOutput = true, RedirectStandardOutput = true,
@ -64,7 +64,7 @@ namespace Wabbajack.VirtualFileSystem
Utils.Error(e, "Error while setting process priority level for innounp.exe"); Utils.Error(e, "Error while setting process priority level for innounp.exe");
} }
var name = Path.GetFileName(source); var name = source.FileName;
try try
{ {
while (!p.HasExited) while (!p.HasExited)
@ -77,7 +77,7 @@ namespace Wabbajack.VirtualFileSystem
continue; continue;
int.TryParse(line.Substring(0, 3), out var percentInt); int.TryParse(line.Substring(0, 3), out var percentInt);
Utils.Status($"Extracting {name} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d)); Utils.Status($"Extracting {(string)name} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d));
} }
} }
catch (Exception e) catch (Exception e)
@ -85,7 +85,7 @@ namespace Wabbajack.VirtualFileSystem
Utils.Error(e, "Error while reading StandardOutput for innounp.exe"); Utils.Error(e, "Error while reading StandardOutput for innounp.exe");
} }
p.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Extracting {name}"); p.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Extracting {(string)name}");
if (p.ExitCode == 0) if (p.ExitCode == 0)
return; return;
@ -113,59 +113,52 @@ namespace Wabbajack.VirtualFileSystem
} }
} }
private static void ExtractAllWithOMOD(string source, string dest) private static void ExtractAllWithOMOD(AbsolutePath source, AbsolutePath dest)
{ {
Utils.Log($"Extracting {Path.GetFileName(source)}"); Utils.Log($"Extracting {(string)source.FileName}");
Framework.Settings.TempPath = dest; Framework.Settings.TempPath = (string)dest;
Framework.Settings.CodeProgress = new OMODProgress(); Framework.Settings.CodeProgress = new OMODProgress();
var omod = new OMOD(source); var omod = new OMOD((string)source);
omod.GetDataFiles(); omod.GetDataFiles();
omod.GetPlugins(); omod.GetPlugins();
} }
private static async Task ExtractAllWithBSA(WorkQueue queue, string source, string dest) private static async Task ExtractAllWithBSA(WorkQueue queue, AbsolutePath source, AbsolutePath dest)
{ {
try try
{ {
using (var arch = BSADispatch.OpenRead(source)) using var arch = BSADispatch.OpenRead(source);
{
await arch.Files await arch.Files
.PMap(queue, f => .PMap(queue, f =>
{ {
var path = f.Path; Utils.Status($"Extracting {(string)f.Path}");
if (f.Path.StartsWith("\\")) var outPath = f.Path.RelativeTo(dest);
path = f.Path.Substring(1); var parent = outPath.Parent;
Utils.Status($"Extracting {path}");
var outPath = Path.Combine(dest, path);
var parent = Path.GetDirectoryName(outPath);
if (!Directory.Exists(parent)) if (!parent.IsDirectory)
Directory.CreateDirectory(parent); parent.CreateDirectory();
using (var fs = File.Open(outPath, System.IO.FileMode.Create)) using var fs = outPath.Create();
{
f.CopyDataTo(fs); f.CopyDataTo(fs);
}
}); });
} }
}
catch (Exception ex) catch (Exception ex)
{ {
Utils.ErrorThrow(ex, $"While Extracting {source}"); Utils.ErrorThrow(ex, $"While Extracting {source}");
} }
} }
private static void ExtractAllWith7Zip(string source, string dest) private static void ExtractAllWith7Zip(AbsolutePath source, AbsolutePath dest)
{ {
Utils.Log(new GenericInfo($"Extracting {Path.GetFileName(source)}", $"The contents of {source} are being extracted to {dest} using 7zip.exe")); Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}", $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe"));
var info = new ProcessStartInfo var info = new ProcessStartInfo
{ {
FileName = @"Extractors\7z.exe", FileName = @"Extractors\7z.exe",
Arguments = $"x -bsp1 -y -o\"{dest}\" \"{source}\" -mmt=off", Arguments = $"x -bsp1 -y -o\"{(string)dest}\" \"{(string)source}\" -mmt=off",
RedirectStandardError = true, RedirectStandardError = true,
RedirectStandardInput = true, RedirectStandardInput = true,
RedirectStandardOutput = true, RedirectStandardOutput = true,
@ -185,7 +178,7 @@ namespace Wabbajack.VirtualFileSystem
{ {
} }
var name = Path.GetFileName(source); var name = source.FileName;
try try
{ {
while (!p.HasExited) while (!p.HasExited)
@ -197,7 +190,7 @@ namespace Wabbajack.VirtualFileSystem
if (line.Length <= 4 || line[3] != '%') continue; if (line.Length <= 4 || line[3] != '%') continue;
int.TryParse(line.Substring(0, 3), out var percentInt); int.TryParse(line.Substring(0, 3), out var percentInt);
Utils.Status($"Extracting {name} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d)); Utils.Status($"Extracting {(string)name} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d));
} }
} }
catch (Exception) catch (Exception)
@ -219,13 +212,13 @@ namespace Wabbajack.VirtualFileSystem
/// </summary> /// </summary>
/// <param name="v"></param> /// <param name="v"></param>
/// <returns></returns> /// <returns></returns>
public static bool CanExtract(string v) public static bool CanExtract(AbsolutePath v)
{ {
var ext = Path.GetExtension(v.ToLower()); var ext = v.Extension;
if(ext != ".exe" && !Consts.TestArchivesBeforeExtraction.Contains(ext)) if(ext != _exeExtension && !Consts.TestArchivesBeforeExtraction.Contains(ext))
return Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext); return Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext);
if (ext == ".exe") if (ext == _exeExtension)
{ {
var info = new ProcessStartInfo var info = new ProcessStartInfo
{ {
@ -243,7 +236,7 @@ namespace Wabbajack.VirtualFileSystem
p.Start(); p.Start();
ChildProcessTracker.AddProcess(p); ChildProcessTracker.AddProcess(p);
var name = Path.GetFileName(v); var name = v.FileName;
while (!p.HasExited) while (!p.HasExited)
{ {
var line = p.StandardOutput.ReadLine(); var line = p.StandardOutput.ReadLine();
@ -253,7 +246,7 @@ namespace Wabbajack.VirtualFileSystem
if (line[0] != '#') if (line[0] != '#')
continue; continue;
Utils.Status($"Testing {name} - {line.Trim()}"); Utils.Status($"Testing {(string)name} - {line.Trim()}");
} }
p.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Testing {name}"); p.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Testing {name}");
@ -299,10 +292,12 @@ namespace Wabbajack.VirtualFileSystem
} }
public static bool MightBeArchive(string path) private static Extension _exeExtension = new Extension(".exe");
public static bool MightBeArchive(AbsolutePath path)
{ {
var ext = Path.GetExtension(path.ToLower()); var ext = path.Extension;
return ext == ".exe" || Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext); return ext == _exeExtension || Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext);
} }
} }
} }

2
Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs
View File

@ -8,7 +8,7 @@ namespace Wabbajack.VirtualFileSystem
/// </summary> /// </summary>
public class IndexedVirtualFile public class IndexedVirtualFile
{ {
public string Name { get; set; } public AbstractPath Name { get; set; }
public Hash Hash { get; set; } public Hash Hash { get; set; }
public long Size { get; set; } public long Size { get; set; }
public List<IndexedVirtualFile> Children { get; set; } = new List<IndexedVirtualFile>(); public List<IndexedVirtualFile> Children { get; set; } = new List<IndexedVirtualFile>();

2
Wabbajack.VirtualFileSystem/PortableFile.cs
View File

@ -4,7 +4,7 @@ namespace Wabbajack.VirtualFileSystem
{ {
public class PortableFile public class PortableFile
{ {
public string Name { get; set; } public AbstractPath Name { get; set; }
public Hash Hash { get; set; } public Hash Hash { get; set; }
public Hash ParentHash { get; set; } public Hash ParentHash { get; set; }
public long Size { get; set; } public long Size { get; set; }

121
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -18,26 +18,26 @@ namespace Wabbajack.VirtualFileSystem
{ {
public class VirtualFile public class VirtualFile
{ {
private string _fullPath; private FullPath _fullPath;
private string _stagedPath; private AbsolutePath _stagedPath;
public string Name { get; internal set; } public AbstractPath Name { get; internal set; }
public string FullPath public FullPath FullPath
{ {
get get
{ {
if (_fullPath != null) return _fullPath; if (_fullPath != null) return _fullPath;
var cur = this; var cur = this;
var acc = new LinkedList<string>(); var acc = new LinkedList<AbstractPath>();
while (cur != null) while (cur != null)
{ {
acc.AddFirst(cur.Name); acc.AddFirst(cur.Name);
cur = cur.Parent; cur = cur.Parent;
} }
_fullPath = string.Join("|", acc); _fullPath = new FullPath(acc.First() as AbsolutePath, acc.Skip(1).OfType<RelativePath>().ToArray());
return _fullPath; return _fullPath;
} }
} }
@ -46,20 +46,20 @@ namespace Wabbajack.VirtualFileSystem
public ExtendedHashes ExtendedHashes { get; set; } public ExtendedHashes ExtendedHashes { get; set; }
public long Size { get; internal set; } public long Size { get; internal set; }
public long LastModified { get; internal set; } public ulong LastModified { get; internal set; }
public long LastAnalyzed { get; internal set; } public ulong LastAnalyzed { get; internal set; }
public VirtualFile Parent { get; internal set; } public VirtualFile Parent { get; internal set; }
public Context Context { get; set; } public Context Context { get; set; }
public string StagedPath public AbsolutePath StagedPath
{ {
get get
{ {
if (IsNative) if (IsNative)
return Name; return Name as AbsolutePath;
if (_stagedPath == null) if (_stagedPath == null)
throw new UnstagedFileException(FullPath); throw new UnstagedFileException(FullPath);
return _stagedPath; return _stagedPath;
@ -147,20 +147,19 @@ namespace Wabbajack.VirtualFileSystem
} }
} }
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, string abs_path, public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, AbsolutePath absPath,
string rel_path, bool topLevel) AbstractPath relPath, bool topLevel)
{ {
var hash = abs_path.FileHash(); var hash = absPath.FileHash();
var fi = new FileInfo(abs_path);
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(abs_path)) if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(absPath))
{ {
var result = await TryGetContentsFromServer(hash); var result = await TryGetContentsFromServer(hash);
if (result != null) if (result != null)
{ {
Utils.Log($"Downloaded VFS data for {Path.GetFileName(abs_path)}"); Utils.Log($"Downloaded VFS data for {(string)absPath}");
VirtualFile Convert(IndexedVirtualFile file, string path, VirtualFile vparent) VirtualFile Convert(IndexedVirtualFile file, AbstractPath path, VirtualFile vparent)
{ {
var vself = new VirtualFile var vself = new VirtualFile
{ {
@ -168,8 +167,8 @@ namespace Wabbajack.VirtualFileSystem
Name = path, Name = path,
Parent = vparent, Parent = vparent,
Size = file.Size, Size = file.Size,
LastModified = fi.LastWriteTimeUtc.Ticks, LastModified = absPath.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.Ticks, LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = file.Hash, Hash = file.Hash,
}; };
@ -179,32 +178,32 @@ namespace Wabbajack.VirtualFileSystem
return vself; return vself;
} }
return Convert(result, rel_path, parent); return Convert(result, relPath, parent);
} }
} }
var self = new VirtualFile var self = new VirtualFile
{ {
Context = context, Context = context,
Name = rel_path, Name = relPath,
Parent = parent, Parent = parent,
Size = fi.Length, Size = absPath.Size,
LastModified = fi.LastWriteTimeUtc.Ticks, LastModified = absPath.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.Ticks, LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = hash Hash = hash
}; };
if (context.UseExtendedHashes) if (context.UseExtendedHashes)
self.ExtendedHashes = ExtendedHashes.FromFile(abs_path); self.ExtendedHashes = ExtendedHashes.FromFile(absPath);
if (FileExtractor.CanExtract(abs_path)) if (FileExtractor.CanExtract(absPath))
{ {
using (var tempFolder = context.GetTemporaryFolder()) using (var tempFolder = Context.GetTemporaryFolder())
{ {
await FileExtractor.ExtractAll(context.Queue, abs_path, tempFolder.FullName); await FileExtractor.ExtractAll(context.Queue, absPath, tempFolder.FullName);
var list = await Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories) var list = await tempFolder.FullName.EnumerateFiles()
.PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName), false)); .PMap(context.Queue, absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), false));
self.Children = list.ToImmutableList(); self.Children = list.ToImmutableList();
} }
@ -236,59 +235,29 @@ namespace Wabbajack.VirtualFileSystem
} }
public void Write(MemoryStream ms) private void Write(Stream stream)
{ {
using (var bw = new BinaryWriter(ms, Encoding.UTF8, true)) stream.WriteAsMessagePack(this);
{
Write(bw);
}
}
private void Write(BinaryWriter bw)
{
bw.Write(Name);
bw.Write(FullPath);
bw.Write(Hash);
bw.Write(Size);
bw.Write(LastModified);
bw.Write(LastAnalyzed);
bw.Write(Children.Count);
foreach (var child in Children)
child.Write(bw);
} }
public static VirtualFile Read(Context context, byte[] data) public static VirtualFile Read(Context context, byte[] data)
{ {
using (var ms = new MemoryStream(data)) using var ms = new MemoryStream(data);
using (var br = new BinaryReader(ms)) return Read(context, null, ms);
{
return Read(context, null, br);
}
} }
private static VirtualFile Read(Context context, VirtualFile parent, BinaryReader br) private static VirtualFile Read(Context context, VirtualFile parent, Stream br)
{ {
var vf = new VirtualFile var vf = br.ReadAsMessagePack<VirtualFile>();
{ vf.Parent = parent;
Context = context, vf.Context = context;
Parent = parent, vf.Children ??= ImmutableList<VirtualFile>.Empty;
Name = br.ReadString(),
_fullPath = br.ReadString(),
Hash = br.ReadHash(),
Size = br.ReadInt64(),
LastModified = br.ReadInt64(),
LastAnalyzed = br.ReadInt64(),
Children = ImmutableList<VirtualFile>.Empty
};
var childrenCount = br.ReadInt32();
for (var idx = 0; idx < childrenCount; idx += 1) vf.Children = vf.Children.Add(Read(context, vf, br));
return vf; return vf;
} }
public static VirtualFile CreateFromPortable(Context context, public static VirtualFile CreateFromPortable(Context context,
Dictionary<Hash, IEnumerable<PortableFile>> state, Dictionary<Hash, string> links, Dictionary<Hash, IEnumerable<PortableFile>> state, Dictionary<Hash, AbsolutePath> links,
PortableFile portableFile) PortableFile portableFile)
{ {
var vf = new VirtualFile var vf = new VirtualFile
@ -337,16 +306,16 @@ namespace Wabbajack.VirtualFileSystem
public FileStream OpenRead() public FileStream OpenRead()
{ {
return File.OpenRead(StagedPath); return StagedPath.OpenRead();
} }
} }
public class ExtendedHashes public class ExtendedHashes
{ {
public static ExtendedHashes FromFile(string file) public static ExtendedHashes FromFile(AbsolutePath file)
{ {
var hashes = new ExtendedHashes(); var hashes = new ExtendedHashes();
using (var stream = File.OpenRead(file)) using (var stream = file.OpenRead())
{ {
hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex(); hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex();
stream.Position = 0; stream.Position = 0;
@ -386,9 +355,9 @@ namespace Wabbajack.VirtualFileSystem
public class UnstagedFileException : Exception public class UnstagedFileException : Exception
{ {
private readonly string _fullPath; private readonly FullPath _fullPath;
public UnstagedFileException(string fullPath) : base($"File {fullPath} is unstaged, cannot get staged name") public UnstagedFileException(FullPath fullPath) : base($"File {fullPath} is unstaged, cannot get staged name")
{ {
_fullPath = fullPath; _fullPath = fullPath;
} }