2021-09-27 12:42:46 +00:00
|
|
|
using System;
|
2019-11-14 22:22:53 +00:00
|
|
|
using System.Collections.Generic;
|
|
|
|
using System.Collections.Immutable;
|
|
|
|
using System.IO;
|
|
|
|
using System.Linq;
|
2021-09-27 12:42:46 +00:00
|
|
|
using System.Threading;
|
2019-12-04 01:26:26 +00:00
|
|
|
using System.Threading.Tasks;
|
2021-09-27 12:42:46 +00:00
|
|
|
using Microsoft.Extensions.Logging;
|
2019-11-14 22:22:53 +00:00
|
|
|
using Wabbajack.Common;
|
2021-06-16 05:16:25 +00:00
|
|
|
using Wabbajack.Common.FileSignatures;
|
2021-09-27 12:42:46 +00:00
|
|
|
using Wabbajack.DTOs.Streams;
|
|
|
|
using Wabbajack.DTOs.Texture;
|
|
|
|
using Wabbajack.Hashing.PHash;
|
|
|
|
using Wabbajack.Hashing.xxHash64;
|
|
|
|
using Wabbajack.Paths;
|
|
|
|
using Wabbajack.Paths.IO;
|
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
namespace Wabbajack.VFS;
|
|
|
|
|
|
|
|
public class VirtualFile
|
2019-11-14 22:22:53 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
private static readonly HashSet<Extension> TextureExtensions = new()
|
|
|
|
{new Extension(".dds"), new Extension(".tga")};
|
2021-01-05 22:09:32 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
private static readonly SignatureChecker DDSSig = new(FileType.DDS);
|
2021-01-05 22:09:32 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
private IEnumerable<VirtualFile> _thisAndAllChildren;
|
2020-03-23 12:57:18 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public IPath Name { get; internal set; }
|
2019-11-14 22:22:53 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public RelativePath RelativeName => (RelativePath) Name;
|
2020-03-24 12:21:19 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public AbsolutePath AbsoluteName => (AbsolutePath) Name;
|
2020-03-24 12:21:19 +00:00
|
|
|
|
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public FullPath FullPath { get; private set; }
|
2019-11-14 22:22:53 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public Hash Hash { get; internal set; }
|
|
|
|
public ImageState? ImageState { get; internal set; }
|
|
|
|
public long Size { get; internal set; }
|
2019-11-14 22:22:53 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public ulong LastModified { get; internal set; }
|
2019-11-14 22:22:53 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public ulong LastAnalyzed { get; internal set; }
|
2019-11-14 22:22:53 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public VirtualFile Parent { get; internal set; }
|
2019-11-14 22:22:53 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public Context Context { get; set; }
|
2019-11-14 22:22:53 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
/// <summary>
|
|
|
|
/// Returns the nesting factor for this file. Native files will have a nesting of 1, the factor
|
|
|
|
/// goes up for each nesting of a file in an archive.
|
|
|
|
/// </summary>
|
|
|
|
public int NestingFactor
|
|
|
|
{
|
|
|
|
get
|
2019-11-14 22:22:53 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
var cnt = 0;
|
|
|
|
var cur = this;
|
|
|
|
while (cur != null)
|
2019-11-14 22:22:53 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
cnt += 1;
|
|
|
|
cur = cur.Parent;
|
2019-11-14 22:22:53 +00:00
|
|
|
}
|
2021-10-23 16:51:17 +00:00
|
|
|
|
|
|
|
return cnt;
|
2019-11-14 22:22:53 +00:00
|
|
|
}
|
2021-10-23 16:51:17 +00:00
|
|
|
}
|
2019-11-14 22:22:53 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public ImmutableList<VirtualFile> Children { get; internal set; } = ImmutableList<VirtualFile>.Empty;
|
2019-11-14 22:22:53 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public bool IsArchive => Children != null && Children.Count > 0;
|
2019-11-14 22:22:53 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public bool IsNative => Parent == null;
|
2019-11-14 22:22:53 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public IEnumerable<VirtualFile> ThisAndAllChildren
|
|
|
|
{
|
|
|
|
get
|
2019-11-24 23:03:36 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
if (_thisAndAllChildren == null)
|
|
|
|
_thisAndAllChildren = Children.SelectMany(child => child.ThisAndAllChildren).Append(this).ToList();
|
2019-11-24 23:03:36 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
return _thisAndAllChildren;
|
2019-11-24 23:03:36 +00:00
|
|
|
}
|
2021-10-23 16:51:17 +00:00
|
|
|
}
|
2019-11-14 22:22:53 +00:00
|
|
|
|
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
/// <summary>
|
|
|
|
/// Returns all the virtual files in the path to this file, starting from the root file.
|
|
|
|
/// </summary>
|
|
|
|
public IEnumerable<VirtualFile> FilesInFullPath
|
|
|
|
{
|
|
|
|
get
|
2019-11-14 22:22:53 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
var stack = ImmutableStack<VirtualFile>.Empty;
|
|
|
|
var cur = this;
|
|
|
|
while (cur != null)
|
2019-11-14 22:22:53 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
stack = stack.Push(cur);
|
|
|
|
cur = cur.Parent;
|
2019-11-14 22:22:53 +00:00
|
|
|
}
|
2021-10-23 16:51:17 +00:00
|
|
|
|
|
|
|
return stack;
|
2019-11-14 22:22:53 +00:00
|
|
|
}
|
2021-10-23 16:51:17 +00:00
|
|
|
}
|
2019-11-14 22:22:53 +00:00
|
|
|
|
2020-03-24 12:21:19 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public VirtualFile TopParent => IsNative ? this : Parent.TopParent;
|
2020-06-20 23:10:43 +00:00
|
|
|
|
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public T ThisAndAllChildrenReduced<T>(T acc, Func<T, VirtualFile, T> fn)
|
|
|
|
{
|
|
|
|
acc = fn(acc, this);
|
|
|
|
return Children.Aggregate(acc, (current, itm) => itm.ThisAndAllChildrenReduced(current, fn));
|
|
|
|
}
|
2020-03-24 12:21:19 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public void ThisAndAllChildrenReduced(Action<VirtualFile> fn)
|
|
|
|
{
|
|
|
|
fn(this);
|
|
|
|
foreach (var itm in Children)
|
|
|
|
itm.ThisAndAllChildrenReduced(fn);
|
|
|
|
}
|
2021-09-27 12:42:46 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path,
|
|
|
|
VirtualFile vparent, IStreamFactory extractedFile)
|
|
|
|
{
|
|
|
|
var vself = new VirtualFile
|
2020-05-12 21:28:09 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
Context = context,
|
|
|
|
Name = path,
|
|
|
|
Parent = vparent,
|
|
|
|
Size = file.Size,
|
|
|
|
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
|
|
|
|
LastAnalyzed = DateTime.Now.AsUnixTime(),
|
|
|
|
Hash = file.Hash,
|
|
|
|
ImageState = file.ImageState
|
|
|
|
};
|
|
|
|
|
|
|
|
vself.FillFullPath();
|
|
|
|
|
|
|
|
vself.Children = file.Children.Select(f => ConvertFromIndexedFile(context, f, f.Name, vself, extractedFile))
|
|
|
|
.ToImmutableList();
|
|
|
|
|
|
|
|
return vself;
|
|
|
|
}
|
2020-05-12 21:28:09 +00:00
|
|
|
|
2021-01-05 22:09:32 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
internal IndexedVirtualFile ToIndexedVirtualFile()
|
|
|
|
{
|
|
|
|
return new IndexedVirtualFile
|
2020-05-12 21:28:09 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
Hash = Hash,
|
|
|
|
ImageState = ImageState,
|
|
|
|
Name = Name,
|
|
|
|
Children = Children.Select(c => c.ToIndexedVirtualFile()).ToList(),
|
|
|
|
Size = Size
|
|
|
|
};
|
|
|
|
}
|
2021-07-17 23:03:22 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public static async Task<VirtualFile> Analyze(Context context, VirtualFile? parent,
|
|
|
|
IStreamFactory extractedFile,
|
|
|
|
IPath relPath, CancellationToken token, int depth = 0)
|
|
|
|
{
|
|
|
|
Hash hash;
|
|
|
|
using (var job = await context.Limiter.Begin("Hash file", 0, token))
|
2019-11-14 22:22:53 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
if (extractedFile is NativeFileStreamFactory)
|
2020-09-18 03:27:59 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
var absPath = (AbsolutePath) extractedFile.Name;
|
|
|
|
job.Size = absPath.Size();
|
|
|
|
hash = await context.HashCache.FileHashCachedAsync(absPath, token);
|
2020-09-18 03:27:59 +00:00
|
|
|
}
|
2021-10-23 16:51:17 +00:00
|
|
|
else
|
|
|
|
{
|
|
|
|
await using var hstream = await extractedFile.GetStream();
|
|
|
|
job.Size = hstream.Length;
|
|
|
|
hash = await hstream.HashingCopy(Stream.Null, token, job);
|
|
|
|
}
|
|
|
|
}
|
2020-01-10 04:47:06 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
if (context.VfsCache.TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
|
|
|
|
return vself;
|
2020-09-18 03:27:59 +00:00
|
|
|
|
2021-09-27 12:42:46 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
await using var stream = await extractedFile.GetStream();
|
|
|
|
var sig = await FileExtractor.FileExtractor.ArchiveSigs.MatchesAsync(stream);
|
|
|
|
stream.Position = 0;
|
2021-09-27 12:42:46 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
var self = new VirtualFile
|
|
|
|
{
|
|
|
|
Context = context,
|
|
|
|
Name = relPath,
|
|
|
|
Parent = parent,
|
|
|
|
Size = stream.Length,
|
|
|
|
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
|
|
|
|
LastAnalyzed = DateTime.Now.AsUnixTime(),
|
|
|
|
Hash = hash
|
|
|
|
};
|
2019-11-15 23:13:27 +00:00
|
|
|
|
2020-09-08 02:22:23 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
if (TextureExtensions.Contains(relPath.FileName.Extension) && await DDSSig.MatchesAsync(stream) != null)
|
|
|
|
try
|
2020-09-08 02:22:23 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
using var job = await context.Limiter.Begin("Perceptual hash", self.Size, token);
|
|
|
|
self.ImageState = await ImageLoader.Load(stream);
|
|
|
|
await job.Report((int) self.Size, token);
|
|
|
|
stream.Position = 0;
|
2020-09-08 02:22:23 +00:00
|
|
|
}
|
2021-10-23 16:51:17 +00:00
|
|
|
catch (Exception)
|
2020-04-30 12:17:18 +00:00
|
|
|
{
|
|
|
|
}
|
2019-11-15 23:13:27 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
self.FillFullPath(depth);
|
|
|
|
|
|
|
|
|
|
|
|
// Can't extract, so return
|
|
|
|
if (!sig.HasValue ||
|
|
|
|
!FileExtractor.FileExtractor.ExtractableExtensions.Contains(relPath.FileName.Extension))
|
|
|
|
{
|
2021-09-27 12:42:46 +00:00
|
|
|
await context.VfsCache.WriteToCache(self);
|
2021-07-05 21:26:30 +00:00
|
|
|
return self;
|
|
|
|
}
|
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
try
|
2020-04-24 13:56:03 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
var list = await context.Extractor.GatheringExtract(extractedFile,
|
|
|
|
_ => true,
|
|
|
|
async (path, sfactory) => await Analyze(context, self, sfactory, path, token, depth + 1),
|
|
|
|
token);
|
2020-04-24 13:56:03 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
self.Children = list.Values.ToImmutableList();
|
2020-04-24 13:56:03 +00:00
|
|
|
}
|
2021-10-23 16:51:17 +00:00
|
|
|
catch (EndOfStreamException)
|
2020-03-24 21:42:28 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
return self;
|
2020-01-10 04:47:06 +00:00
|
|
|
}
|
2021-10-23 16:51:17 +00:00
|
|
|
catch (Exception ex)
|
2019-11-14 22:22:53 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
context.Logger.LogError(ex, "Error while examining the contents of {path}", relPath.FileName);
|
|
|
|
throw;
|
2019-11-14 22:22:53 +00:00
|
|
|
}
|
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
await context.VfsCache.WriteToCache(self);
|
|
|
|
return self;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
internal void FillFullPath()
|
|
|
|
{
|
|
|
|
var depth = 0;
|
|
|
|
var self = this;
|
|
|
|
while (self.Parent != null)
|
2019-11-14 22:22:53 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
depth += 1;
|
|
|
|
self = self.Parent;
|
2019-11-14 22:22:53 +00:00
|
|
|
}
|
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
FillFullPath(depth);
|
|
|
|
}
|
2021-09-27 12:42:46 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
internal void FillFullPath(int depth)
|
|
|
|
{
|
|
|
|
if (depth == 0)
|
|
|
|
{
|
|
|
|
FullPath = new FullPath((AbsolutePath) Name);
|
2020-03-24 12:21:19 +00:00
|
|
|
}
|
2021-10-23 16:51:17 +00:00
|
|
|
else
|
2020-03-24 12:21:19 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
var paths = new RelativePath[depth];
|
|
|
|
var self = this;
|
|
|
|
for (var idx = depth; idx != 0; idx -= 1)
|
2020-03-24 12:21:19 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
paths[idx - 1] = self.RelativeName;
|
|
|
|
self = self.Parent;
|
2020-03-24 12:21:19 +00:00
|
|
|
}
|
2021-09-27 12:42:46 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
FullPath = new FullPath(self.AbsoluteName, paths);
|
2019-11-14 22:22:53 +00:00
|
|
|
}
|
2021-10-23 16:51:17 +00:00
|
|
|
}
|
2019-11-14 22:22:53 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
public void Write(BinaryWriter bw)
|
|
|
|
{
|
|
|
|
bw.Write(Name.ToString() ?? string.Empty);
|
|
|
|
bw.Write(Size);
|
|
|
|
bw.Write(LastModified);
|
|
|
|
bw.Write(LastModified);
|
|
|
|
bw.Write((ulong) Hash);
|
|
|
|
bw.Write(Children.Count);
|
|
|
|
foreach (var child in Children)
|
|
|
|
child.Write(bw);
|
|
|
|
}
|
|
|
|
|
|
|
|
public static VirtualFile Read(Context context, byte[] data)
|
|
|
|
{
|
|
|
|
using var ms = new MemoryStream(data);
|
|
|
|
using var br = new BinaryReader(ms);
|
|
|
|
return Read(context, null, br);
|
|
|
|
}
|
|
|
|
|
|
|
|
private static VirtualFile Read(Context context, VirtualFile parent, BinaryReader br)
|
|
|
|
{
|
|
|
|
var vf = new VirtualFile
|
|
|
|
{
|
|
|
|
Name = br.ReadIPath(),
|
|
|
|
Size = br.ReadInt64(),
|
|
|
|
LastModified = br.ReadUInt64(),
|
|
|
|
LastAnalyzed = br.ReadUInt64(),
|
|
|
|
Hash = Hash.FromULong(br.ReadUInt64()),
|
|
|
|
Context = context,
|
|
|
|
Parent = parent,
|
|
|
|
Children = ImmutableList<VirtualFile>.Empty
|
|
|
|
};
|
|
|
|
vf.FullPath = new FullPath(vf.AbsoluteName);
|
|
|
|
var children = br.ReadInt32();
|
|
|
|
for (var i = 0; i < children; i++)
|
2019-11-15 13:06:34 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
var child = Read(context, vf, br, (AbsolutePath) vf.Name, new RelativePath[0]);
|
|
|
|
vf.Children = vf.Children.Add(child);
|
|
|
|
}
|
2019-11-15 13:06:34 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
return vf;
|
|
|
|
}
|
2020-03-24 12:21:19 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
private static VirtualFile Read(Context context, VirtualFile parent, BinaryReader br, AbsolutePath top,
|
|
|
|
RelativePath[] subpaths)
|
|
|
|
{
|
|
|
|
var name = (RelativePath) br.ReadIPath();
|
|
|
|
subpaths = subpaths.Add(name);
|
|
|
|
var vf = new VirtualFile
|
|
|
|
{
|
|
|
|
Name = name,
|
|
|
|
Size = br.ReadInt64(),
|
|
|
|
LastModified = br.ReadUInt64(),
|
|
|
|
LastAnalyzed = br.ReadUInt64(),
|
|
|
|
Hash = Hash.FromULong(br.ReadUInt64()),
|
|
|
|
Context = context,
|
|
|
|
Parent = parent,
|
|
|
|
Children = ImmutableList<VirtualFile>.Empty,
|
|
|
|
FullPath = new FullPath(top, subpaths)
|
|
|
|
};
|
|
|
|
|
|
|
|
var children = br.ReadInt32();
|
|
|
|
for (var i = 0; i < children; i++)
|
|
|
|
{
|
|
|
|
var child = Read(context, vf, br, top, subpaths);
|
|
|
|
vf.Children = vf.Children.Add(child);
|
2019-11-15 13:06:34 +00:00
|
|
|
}
|
2021-07-17 05:32:37 +00:00
|
|
|
|
2021-10-23 16:51:17 +00:00
|
|
|
return vf;
|
|
|
|
}
|
|
|
|
|
|
|
|
public HashRelativePath MakeRelativePaths()
|
|
|
|
{
|
|
|
|
var paths = new RelativePath[FilesInFullPath.Count() - 1];
|
|
|
|
|
|
|
|
var idx = 0;
|
|
|
|
foreach (var itm in FilesInFullPath.Skip(1))
|
2021-07-17 05:32:37 +00:00
|
|
|
{
|
2021-10-23 16:51:17 +00:00
|
|
|
paths[idx] = (RelativePath) itm.Name;
|
|
|
|
idx += 1;
|
2021-07-17 05:32:37 +00:00
|
|
|
}
|
2021-10-23 16:51:17 +00:00
|
|
|
|
|
|
|
var path = new HashRelativePath(FilesInFullPath.First().Hash, paths);
|
|
|
|
return path;
|
|
|
|
}
|
|
|
|
|
|
|
|
public VirtualFile InSameFolder(RelativePath relativePath)
|
|
|
|
{
|
|
|
|
var newPath = FullPath.InSameFolder(relativePath);
|
|
|
|
return Context.Index.ByFullPath.TryGetValue(newPath, out var found) ? found : null;
|
2019-11-14 22:22:53 +00:00
|
|
|
}
|
2021-09-27 12:42:46 +00:00
|
|
|
}
|