wabbajack/Wabbajack.VFS/VirtualFile.cs

372 lines
11 KiB
C#
Raw Normal View History

2021-09-27 12:42:46 +00:00
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
2021-09-27 12:42:46 +00:00
using System.Threading;
using System.Threading.Tasks;
2021-09-27 12:42:46 +00:00
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
2021-06-16 05:16:25 +00:00
using Wabbajack.Common.FileSignatures;
2021-09-27 12:42:46 +00:00
using Wabbajack.DTOs.Streams;
using Wabbajack.DTOs.Texture;
using Wabbajack.Hashing.PHash;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
2021-10-23 16:51:17 +00:00
namespace Wabbajack.VFS;
public class VirtualFile
{
2021-10-23 16:51:17 +00:00
private static readonly HashSet<Extension> TextureExtensions = new()
{new Extension(".dds"), new Extension(".tga")};
2021-01-05 22:09:32 +00:00
2021-10-23 16:51:17 +00:00
private static readonly SignatureChecker DDSSig = new(FileType.DDS);
2021-01-05 22:09:32 +00:00
2021-10-23 16:51:17 +00:00
private IEnumerable<VirtualFile> _thisAndAllChildren;
2020-03-23 12:57:18 +00:00
2021-10-23 16:51:17 +00:00
public IPath Name { get; internal set; }
2021-10-23 16:51:17 +00:00
public RelativePath RelativeName => (RelativePath) Name;
2020-03-24 12:21:19 +00:00
2021-10-23 16:51:17 +00:00
public AbsolutePath AbsoluteName => (AbsolutePath) Name;
2020-03-24 12:21:19 +00:00
2021-10-23 16:51:17 +00:00
public FullPath FullPath { get; private set; }
2021-10-23 16:51:17 +00:00
public Hash Hash { get; internal set; }
public ImageState? ImageState { get; internal set; }
public long Size { get; internal set; }
2021-10-23 16:51:17 +00:00
public ulong LastModified { get; internal set; }
2021-10-23 16:51:17 +00:00
public ulong LastAnalyzed { get; internal set; }
2021-10-23 16:51:17 +00:00
public VirtualFile Parent { get; internal set; }
2021-10-23 16:51:17 +00:00
public Context Context { get; set; }
2021-10-23 16:51:17 +00:00
/// <summary>
/// Returns the nesting factor for this file. Native files will have a nesting of 1, the factor
/// goes up for each nesting of a file in an archive.
/// </summary>
public int NestingFactor
{
get
{
2021-10-23 16:51:17 +00:00
var cnt = 0;
var cur = this;
while (cur != null)
{
2021-10-23 16:51:17 +00:00
cnt += 1;
cur = cur.Parent;
}
2021-10-23 16:51:17 +00:00
return cnt;
}
2021-10-23 16:51:17 +00:00
}
2021-10-23 16:51:17 +00:00
public ImmutableList<VirtualFile> Children { get; internal set; } = ImmutableList<VirtualFile>.Empty;
2021-10-23 16:51:17 +00:00
public bool IsArchive => Children != null && Children.Count > 0;
2021-10-23 16:51:17 +00:00
public bool IsNative => Parent == null;
2021-10-23 16:51:17 +00:00
public IEnumerable<VirtualFile> ThisAndAllChildren
{
get
2019-11-24 23:03:36 +00:00
{
2021-10-23 16:51:17 +00:00
if (_thisAndAllChildren == null)
_thisAndAllChildren = Children.SelectMany(child => child.ThisAndAllChildren).Append(this).ToList();
2019-11-24 23:03:36 +00:00
2021-10-23 16:51:17 +00:00
return _thisAndAllChildren;
2019-11-24 23:03:36 +00:00
}
2021-10-23 16:51:17 +00:00
}
2021-10-23 16:51:17 +00:00
/// <summary>
/// Returns all the virtual files in the path to this file, starting from the root file.
/// </summary>
public IEnumerable<VirtualFile> FilesInFullPath
{
get
{
2021-10-23 16:51:17 +00:00
var stack = ImmutableStack<VirtualFile>.Empty;
var cur = this;
while (cur != null)
{
2021-10-23 16:51:17 +00:00
stack = stack.Push(cur);
cur = cur.Parent;
}
2021-10-23 16:51:17 +00:00
return stack;
}
2021-10-23 16:51:17 +00:00
}
2020-03-24 12:21:19 +00:00
2021-10-23 16:51:17 +00:00
public VirtualFile TopParent => IsNative ? this : Parent.TopParent;
2021-10-23 16:51:17 +00:00
public T ThisAndAllChildrenReduced<T>(T acc, Func<T, VirtualFile, T> fn)
{
acc = fn(acc, this);
return Children.Aggregate(acc, (current, itm) => itm.ThisAndAllChildrenReduced(current, fn));
}
2020-03-24 12:21:19 +00:00
2021-10-23 16:51:17 +00:00
public void ThisAndAllChildrenReduced(Action<VirtualFile> fn)
{
fn(this);
foreach (var itm in Children)
itm.ThisAndAllChildrenReduced(fn);
}
2021-09-27 12:42:46 +00:00
2021-10-23 16:51:17 +00:00
private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path,
VirtualFile vparent, IStreamFactory extractedFile)
{
var vself = new VirtualFile
{
2021-10-23 16:51:17 +00:00
Context = context,
Name = path,
Parent = vparent,
Size = file.Size,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = file.Hash,
ImageState = file.ImageState
};
vself.FillFullPath();
vself.Children = file.Children.Select(f => ConvertFromIndexedFile(context, f, f.Name, vself, extractedFile))
.ToImmutableList();
return vself;
}
2021-01-05 22:09:32 +00:00
2021-10-23 16:51:17 +00:00
internal IndexedVirtualFile ToIndexedVirtualFile()
{
return new IndexedVirtualFile
{
2021-10-23 16:51:17 +00:00
Hash = Hash,
ImageState = ImageState,
Name = Name,
Children = Children.Select(c => c.ToIndexedVirtualFile()).ToList(),
Size = Size
};
}
2021-10-23 16:51:17 +00:00
public static async Task<VirtualFile> Analyze(Context context, VirtualFile? parent,
IStreamFactory extractedFile,
IPath relPath, CancellationToken token, int depth = 0)
{
Hash hash;
using (var job = await context.Limiter.Begin("Hash file", 0, token))
{
2021-10-23 16:51:17 +00:00
if (extractedFile is NativeFileStreamFactory)
2020-09-18 03:27:59 +00:00
{
2021-10-23 16:51:17 +00:00
var absPath = (AbsolutePath) extractedFile.Name;
job.Size = absPath.Size();
hash = await context.HashCache.FileHashCachedAsync(absPath, token);
2020-09-18 03:27:59 +00:00
}
2021-10-23 16:51:17 +00:00
else
{
await using var hstream = await extractedFile.GetStream();
job.Size = hstream.Length;
hash = await hstream.HashingCopy(Stream.Null, token, job);
}
}
2021-10-23 16:51:17 +00:00
if (context.VfsCache.TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
return vself;
2020-09-18 03:27:59 +00:00
2021-09-27 12:42:46 +00:00
2021-10-23 16:51:17 +00:00
await using var stream = await extractedFile.GetStream();
var sig = await FileExtractor.FileExtractor.ArchiveSigs.MatchesAsync(stream);
stream.Position = 0;
2021-09-27 12:42:46 +00:00
2021-10-23 16:51:17 +00:00
var self = new VirtualFile
{
Context = context,
Name = relPath,
Parent = parent,
Size = stream.Length,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = hash
};
2021-10-23 16:51:17 +00:00
if (TextureExtensions.Contains(relPath.FileName.Extension) && await DDSSig.MatchesAsync(stream) != null)
try
{
2021-10-23 16:51:17 +00:00
using var job = await context.Limiter.Begin("Perceptual hash", self.Size, token);
self.ImageState = await ImageLoader.Load(stream);
await job.Report((int) self.Size, token);
stream.Position = 0;
}
2021-10-23 16:51:17 +00:00
catch (Exception)
{
}
2021-10-23 16:51:17 +00:00
self.FillFullPath(depth);
// Can't extract, so return
if (!sig.HasValue ||
!FileExtractor.FileExtractor.ExtractableExtensions.Contains(relPath.FileName.Extension))
{
2021-09-27 12:42:46 +00:00
await context.VfsCache.WriteToCache(self);
2021-07-05 21:26:30 +00:00
return self;
}
2021-10-23 16:51:17 +00:00
try
2020-04-24 13:56:03 +00:00
{
2021-10-23 16:51:17 +00:00
var list = await context.Extractor.GatheringExtract(extractedFile,
_ => true,
async (path, sfactory) => await Analyze(context, self, sfactory, path, token, depth + 1),
token);
2020-04-24 13:56:03 +00:00
2021-10-23 16:51:17 +00:00
self.Children = list.Values.ToImmutableList();
2020-04-24 13:56:03 +00:00
}
2021-10-23 16:51:17 +00:00
catch (EndOfStreamException)
2020-03-24 21:42:28 +00:00
{
2021-10-23 16:51:17 +00:00
return self;
}
2021-10-23 16:51:17 +00:00
catch (Exception ex)
{
2021-10-23 16:51:17 +00:00
context.Logger.LogError(ex, "Error while examining the contents of {path}", relPath.FileName);
throw;
}
2021-10-23 16:51:17 +00:00
await context.VfsCache.WriteToCache(self);
return self;
}
internal void FillFullPath()
{
var depth = 0;
var self = this;
while (self.Parent != null)
{
2021-10-23 16:51:17 +00:00
depth += 1;
self = self.Parent;
}
2021-10-23 16:51:17 +00:00
FillFullPath(depth);
}
2021-09-27 12:42:46 +00:00
2021-10-23 16:51:17 +00:00
internal void FillFullPath(int depth)
{
if (depth == 0)
{
FullPath = new FullPath((AbsolutePath) Name);
2020-03-24 12:21:19 +00:00
}
2021-10-23 16:51:17 +00:00
else
2020-03-24 12:21:19 +00:00
{
2021-10-23 16:51:17 +00:00
var paths = new RelativePath[depth];
var self = this;
for (var idx = depth; idx != 0; idx -= 1)
2020-03-24 12:21:19 +00:00
{
2021-10-23 16:51:17 +00:00
paths[idx - 1] = self.RelativeName;
self = self.Parent;
2020-03-24 12:21:19 +00:00
}
2021-09-27 12:42:46 +00:00
2021-10-23 16:51:17 +00:00
FullPath = new FullPath(self.AbsoluteName, paths);
}
2021-10-23 16:51:17 +00:00
}
2021-10-23 16:51:17 +00:00
public void Write(BinaryWriter bw)
{
bw.Write(Name.ToString() ?? string.Empty);
bw.Write(Size);
bw.Write(LastModified);
bw.Write(LastModified);
bw.Write((ulong) Hash);
bw.Write(Children.Count);
foreach (var child in Children)
child.Write(bw);
}
public static VirtualFile Read(Context context, byte[] data)
{
using var ms = new MemoryStream(data);
using var br = new BinaryReader(ms);
return Read(context, null, br);
}
private static VirtualFile Read(Context context, VirtualFile parent, BinaryReader br)
{
var vf = new VirtualFile
{
Name = br.ReadIPath(),
Size = br.ReadInt64(),
LastModified = br.ReadUInt64(),
LastAnalyzed = br.ReadUInt64(),
Hash = Hash.FromULong(br.ReadUInt64()),
Context = context,
Parent = parent,
Children = ImmutableList<VirtualFile>.Empty
};
vf.FullPath = new FullPath(vf.AbsoluteName);
var children = br.ReadInt32();
for (var i = 0; i < children; i++)
2019-11-15 13:06:34 +00:00
{
2021-10-23 16:51:17 +00:00
var child = Read(context, vf, br, (AbsolutePath) vf.Name, new RelativePath[0]);
vf.Children = vf.Children.Add(child);
}
2019-11-15 13:06:34 +00:00
2021-10-23 16:51:17 +00:00
return vf;
}
2020-03-24 12:21:19 +00:00
2021-10-23 16:51:17 +00:00
private static VirtualFile Read(Context context, VirtualFile parent, BinaryReader br, AbsolutePath top,
RelativePath[] subpaths)
{
var name = (RelativePath) br.ReadIPath();
subpaths = subpaths.Add(name);
var vf = new VirtualFile
{
Name = name,
Size = br.ReadInt64(),
LastModified = br.ReadUInt64(),
LastAnalyzed = br.ReadUInt64(),
Hash = Hash.FromULong(br.ReadUInt64()),
Context = context,
Parent = parent,
Children = ImmutableList<VirtualFile>.Empty,
FullPath = new FullPath(top, subpaths)
};
var children = br.ReadInt32();
for (var i = 0; i < children; i++)
{
var child = Read(context, vf, br, top, subpaths);
vf.Children = vf.Children.Add(child);
2019-11-15 13:06:34 +00:00
}
2021-10-23 16:51:17 +00:00
return vf;
}
public HashRelativePath MakeRelativePaths()
{
var paths = new RelativePath[FilesInFullPath.Count() - 1];
var idx = 0;
foreach (var itm in FilesInFullPath.Skip(1))
{
2021-10-23 16:51:17 +00:00
paths[idx] = (RelativePath) itm.Name;
idx += 1;
}
2021-10-23 16:51:17 +00:00
var path = new HashRelativePath(FilesInFullPath.First().Hash, paths);
return path;
}
public VirtualFile InSameFolder(RelativePath relativePath)
{
var newPath = FullPath.InSameFolder(relativePath);
return Context.Index.ByFullPath.TryGetValue(newPath, out var found) ? found : null;
}
2021-09-27 12:42:46 +00:00
}