using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.Common.CSP;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = System.IO.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.VirtualFileSystem
{
public class Context
{
public const ulong FileVersion = 0x02;
public const string Magic = "WABBAJACK VFS FILE";
private readonly string _stagingFolder = "vfs_staging";
public IndexRoot Index { get; private set; } = IndexRoot.Empty;
///
/// A stream of tuples of ("Update Title", 0.25) which represent the name of the current task
/// and the current progress.
///
public IObservable<(string, float)> ProgressUpdates => _progressUpdates;
private readonly Subject<(string, float)> _progressUpdates = new Subject<(string, float)>();
public StatusUpdateTracker UpdateTracker { get; set; } = new StatusUpdateTracker(1);
public WorkQueue Queue { get; }
public Context(WorkQueue queue)
{
Queue = queue;
}
public TemporaryDirectory GetTemporaryFolder()
{
return new TemporaryDirectory(Path.Combine(_stagingFolder, Guid.NewGuid().ToString()));
}
public IndexRoot AddRoot(string root)
{
if (!Path.IsPathRooted(root))
throw new InvalidDataException($"Path is not absolute: {root}");
var filtered = Index.AllFiles.Where(file => File.Exists(file.Name)).ToList();
var byPath = filtered.ToImmutableDictionary(f => f.Name);
var filesToIndex = Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive).ToList();
var results = Channel.Create(1024, ProgressUpdater($"Indexing {root}", filesToIndex.Count));
var allFiles= filesToIndex
.PMap(Queue, f =>
{
if (byPath.TryGetValue(f, out var found))
{
var fi = new FileInfo(f);
if (found.LastModified == fi.LastWriteTimeUtc.Ticks && found.Size == fi.Length)
return found;
}
return VirtualFile.Analyze(this, null, f, f);
});
var newIndex = IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
lock (this)
{
Index = newIndex;
}
return newIndex;
}
class Box
{
public T Value { get; set; }
}
private Func, IObservable> ProgressUpdater(string s, float totalCount)
{
if (totalCount == 0)
totalCount = 1;
var box = new Box();
return sub => sub.Select(itm =>
{
box.Value += 1;
_progressUpdates.OnNext((s, box.Value / totalCount));
return itm;
});
}
public void WriteToFile(string filename)
{
using (var fs = File.OpenWrite(filename))
using (var bw = new BinaryWriter(fs, Encoding.UTF8, true))
{
fs.SetLength(0);
bw.Write(Encoding.ASCII.GetBytes(Magic));
bw.Write(FileVersion);
bw.Write((ulong) Index.AllFiles.Count);
Index.AllFiles
.PMap(Queue, f =>
{
var ms = new MemoryStream();
f.Write(ms);
return ms;
})
.Do(ms =>
{
var size = ms.Position;
ms.Position = 0;
bw.Write((ulong) size);
ms.CopyTo(fs);
});
Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}");
}
}
public void IntegrateFromFile(string filename)
{
try
{
using (var fs = File.OpenRead(filename))
using (var br = new BinaryReader(fs, Encoding.UTF8, true))
{
var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length));
var fileVersion = br.ReadUInt64();
if (fileVersion != FileVersion || magic != magic)
throw new InvalidDataException("Bad Data Format");
var numFiles = br.ReadUInt64();
var files = Enumerable.Range(0, (int) numFiles)
.Select(idx =>
{
var size = br.ReadUInt64();
var bytes = new byte[size];
br.BaseStream.Read(bytes, 0, (int) size);
return VirtualFile.Read(this, bytes);
}).ToList();
var newIndex = Index.Integrate(files);
lock (this)
{
Index = newIndex;
}
}
}
catch (IOException)
{
if (File.Exists(filename))
File.Delete(filename);
}
}
public Action Stage(IEnumerable files)
{
var grouped = files.SelectMany(f => f.FilesInFullPath)
.Distinct()
.Where(f => f.Parent != null)
.GroupBy(f => f.Parent)
.OrderBy(f => f.Key?.NestingFactor ?? 0)
.ToList();
var paths = new List();
foreach (var group in grouped)
{
var tmpPath = Path.Combine(_stagingFolder, Guid.NewGuid().ToString());
FileExtractor.ExtractAll(Queue, group.Key.StagedPath, tmpPath);
paths.Add(tmpPath);
foreach (var file in group)
file.StagedPath = Path.Combine(tmpPath, file.Name);
}
return () =>
{
paths.Do(p =>
{
if (Directory.Exists(p))
Directory.Delete(p, true, true);
});
};
}
public List GetPortableState(IEnumerable files)
{
return files.SelectMany(f => f.FilesInFullPath)
.Distinct()
.Select(f => new PortableFile
{
Name = f.Parent != null ? f.Name : null,
Hash = f.Hash,
ParentHash = f.Parent?.Hash,
Size = f.Size
}).ToList();
}
public void IntegrateFromPortable(List state, Dictionary links)
{
var indexedState = state.GroupBy(f => f.ParentHash)
.ToDictionary(f => f.Key ?? "", f => (IEnumerable) f);
var parents = indexedState[""]
.PMap(Queue,f => VirtualFile.CreateFromPortable(this, indexedState, links, f));
var newIndex = Index.Integrate(parents);
lock (this)
{
Index = newIndex;
}
}
public async Task> StageWith(IEnumerable files)
{
return new DisposableList(Stage(files), files);
}
#region KnownFiles
private List _knownFiles = new List();
public void AddKnown(IEnumerable known)
{
_knownFiles.AddRange(known);
}
public void BackfillMissing()
{
var newFiles = _knownFiles.Where(f => f.Paths.Length == 1)
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, s => new VirtualFile()
{
Name = s.First().Paths[0],
Hash = s.First().Hash,
Context = this
});
var parentchild = new Dictionary<(VirtualFile, string), VirtualFile>();
void BackFillOne(KnownFile file)
{
var parent = newFiles[file.Paths[0]];
foreach (var path in file.Paths.Skip(1))
{
if (parentchild.TryGetValue((parent, path), out var foundParent))
{
parent = foundParent;
continue;
}
var nf = new VirtualFile();
nf.Name = path;
nf.Parent = parent;
parent.Children = parent.Children.Add(nf);
parentchild.Add((parent, path), nf);
parent = nf;
}
}
_knownFiles.Where(f => f.Paths.Length > 1).Do(BackFillOne);
var newIndex = Index.Integrate(newFiles.Values.ToList());
lock (this)
Index = newIndex;
_knownFiles = new List();
}
#endregion
}
public class KnownFile
{
public string[] Paths { get; set; }
public string Hash { get; set; }
}
public class DisposableList : List, IDisposable
{
private Action _unstage;
public DisposableList(Action unstage, IEnumerable files) : base(files)
{
_unstage = unstage;
}
public void Dispose()
{
_unstage();
}
}
public class IndexRoot
{
public static IndexRoot Empty = new IndexRoot();
public IndexRoot(ImmutableList aFiles,
ImmutableDictionary byFullPath,
ImmutableDictionary> byHash,
ImmutableDictionary byRoot,
ImmutableDictionary> byName)
{
AllFiles = aFiles;
ByFullPath = byFullPath;
ByHash = byHash;
ByRootPath = byRoot;
ByName = byName;
}
public IndexRoot()
{
AllFiles = ImmutableList.Empty;
ByFullPath = ImmutableDictionary.Empty;
ByHash = ImmutableDictionary>.Empty;
ByRootPath = ImmutableDictionary.Empty;
ByName = ImmutableDictionary>.Empty;
}
public ImmutableList AllFiles { get; }
public ImmutableDictionary ByFullPath { get; }
public ImmutableDictionary> ByHash { get; }
public ImmutableDictionary> ByName { get; set; }
public ImmutableDictionary ByRootPath { get; }
public IndexRoot Integrate(List files)
{
Utils.Log($"Integrating");
var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList();
var byFullPath = allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToImmutableDictionary(f => f.FullPath);
var byHash = allFiles.SelectMany(f => f.ThisAndAllChildren)
.Where(f => f.Hash != null)
.ToGroupedImmutableDictionary(f => f.Hash);
var byName = allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToGroupedImmutableDictionary(f => f.Name);
var byRootPath = allFiles.ToImmutableDictionary(f => f.Name);
var result = new IndexRoot(allFiles,
byFullPath,
byHash,
byRootPath,
byName);
Utils.Log($"Done integrating");
return result;
}
public VirtualFile FileForArchiveHashPath(string[] argArchiveHashPath)
{
var cur = ByHash[argArchiveHashPath[0]].First(f => f.Parent == null);
return argArchiveHashPath.Skip(1).Aggregate(cur, (current, itm) => ByName[itm].First(f => f.Parent == current));
}
}
public class TemporaryDirectory : IDisposable
{
public TemporaryDirectory(string name)
{
FullName = name;
}
public string FullName { get; }
public void Dispose()
{
Directory.Delete(FullName, true, true);
}
}
}