diff --git a/VirtualFileSystem.Test/Program.cs b/VirtualFileSystem.Test/Program.cs index c36cfbca..55e62483 100644 --- a/VirtualFileSystem.Test/Program.cs +++ b/VirtualFileSystem.Test/Program.cs @@ -11,10 +11,11 @@ namespace VirtualFileSystem.Test { static void Main(string[] args) { + Utils.SetLoggerFn(s => Console.WriteLine(s)); + Utils.SetStatusFn((s, i) => Console.WriteLine(s)); WorkQueue.Init((a, b, c) => { return; }, (a, b) => { return; }); - var vfs = new VirtualFileSystem(); - vfs.AddRoot(@"D:\MO2 Instances\Mod Organizer 2", s => Console.WriteLine(s)); + VirtualFileSystem.VFS.AddRoot(@"D:\MO2 Instances\Mod Organizer 2"); } } } diff --git a/VirtualFileSystem/VirtualFileSystem.cs b/VirtualFileSystem/VirtualFileSystem.cs index 4689da27..932f6bca 100644 --- a/VirtualFileSystem/VirtualFileSystem.cs +++ b/VirtualFileSystem/VirtualFileSystem.cs @@ -4,24 +4,117 @@ using Newtonsoft.Json; using SevenZipExtractor; using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.IO; using System.Linq; using System.Reflection; -using System.Text; -using System.Threading.Tasks; using Wabbajack.Common; namespace VirtualFileSystem { public class VirtualFileSystem { - private Dictionary _files = new Dictionary(); - internal string _stagedRoot; - public VirtualFileSystem() + internal static string _stagedRoot; + public static VirtualFileSystem VFS; + private Dictionary _files = new Dictionary(); + + + public static string RootFolder { get; } + public Dictionary> HashIndex { get; private set; } + + static VirtualFileSystem() { - _stagedRoot = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "vfs_staged_files"); + VFS = new VirtualFileSystem(); + RootFolder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + _stagedRoot = Path.Combine(RootFolder, "vfs_staged_files"); + if (Directory.Exists(_stagedRoot)) + Directory.Delete(_stagedRoot, true); + Directory.CreateDirectory(_stagedRoot); + + } + + public VirtualFileSystem () + { + LoadFromDisk(); + } + + private void LoadFromDisk() + { + Utils.Log("Loading VFS Cache"); + if (!File.Exists("vfs_cache.bson")) return; + _files = "vfs_cache.bson".FromBSON>(root_is_array:true).ToDictionary(f => f.FullPath); + CleanDB(); + } + + public void SyncToDisk() + { + lock(this) + { + _files.Values.OfType().ToBSON("vfs_cache.bson"); + } + } + + + public void Purge(VirtualFile f) + { + var path = f.FullPath + "|"; + lock (this) + { + _files.Values + .Where(v => v.FullPath.StartsWith(path) || v.FullPath == f.FullPath) + .ToList() + .Do(r => { + _files.Remove(r.FullPath); + }); + } + } + + public void Add(VirtualFile f) + { + lock (this) + { + if (_files.ContainsKey(f.FullPath)) + Purge(f); + _files.Add(f.FullPath, f); + } + } + + public VirtualFile Lookup(string f) + { + lock (this) + { + if (_files.TryGetValue(f, out var found)) + return found; + return null; + } + } + + /// + /// Remove any orphaned files in the DB. + /// + private void CleanDB() + { + Utils.Log("Cleaning VFS cache"); + lock (this) + { + _files.Values + .Where(f => + { + if (f.IsConcrete) + return !File.Exists(f.StagedPath); + while (f.ParentPath != null) + { + if (Lookup(f.ParentPath) == null) + return true; + f = Lookup(f.ParentPath); + } + return false; + }) + .ToList() + .Do(f => _files.Remove(f.FullPath)); + } } /// @@ -29,42 +122,50 @@ namespace VirtualFileSystem /// and every archive examined. /// /// - public void AddRoot(string path, Action status) + public void AddRoot(string path) { - IndexPath(path, status); + IndexPath(path); + RefreshIndexes(); } - private void SyncToDisk() + private void RefreshIndexes() { - lock (this) + Utils.Log("Building Hash Index"); + lock(this) { - _files.Values.ToList().ToJSON("vfs_cache.json"); + HashIndex = _files.Values + .GroupBy(f => f.Hash) + .ToDictionary(f => f.Key, f => (IEnumerable)f); } } - private void IndexPath(string path, Action status) + private void IndexPath(string path) { Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories) .PMap(f => UpdateFile(f)); + SyncToDisk(); } private void UpdateFile(string f) { TOP: - Console.WriteLine(f); var lv = Lookup(f); if (lv == null) { - lv = new VirtualFile(this) + Utils.Log($"Analyzing {0}"); + + lv = new VirtualFile() { Paths = new string[] { f } }; - this[f] = lv; + lv.Analyze(); + Add(lv); if (lv.IsArchive) { UpdateArchive(lv); } + // Upsert after extraction incase extraction fails } if (lv.IsOutdated) { @@ -80,11 +181,11 @@ namespace VirtualFileSystem var new_path = new string[f.Paths.Length + 1]; f.Paths.CopyTo(new_path, 0); new_path[f.Paths.Length] = e; - var nf = new VirtualFile(this) + var nf = new VirtualFile() { Paths = new_path, }; - this[nf.FullPath] = nf; + Add(nf); return nf; }).ToList(); @@ -93,12 +194,14 @@ namespace VirtualFileSystem // Analyze them new_files.Do(file => file.Analyze()); // Recurse into any archives in this archive - new_files.Where(file => file.IsArchive).Do(file => UpdateArchive(f)); + new_files.Where(file => file.IsArchive).Do(file => UpdateArchive(file)); // Unstage the file new_files.Where(file => file.IsStaged).Do(file => file.Unstage()); + f.FinishedIndexing = true; SyncToDisk(); + Utils.Log($"{_files.Count} docs in VFS cache"); } private void Stage(IEnumerable files) @@ -121,29 +224,12 @@ namespace VirtualFileSystem } } - internal VirtualFile Lookup(string path) - { - lock(this) - { - if (_files.TryGetValue(path, out VirtualFile value)) - return value; - return null; - } - } - public VirtualFile this[string path] { get { return Lookup(path); } - set - { - lock(this) - { - _files[path] = value; - } - } } internal List GetArchiveEntryNames(VirtualFile file) @@ -182,53 +268,34 @@ namespace VirtualFileSystem } } - - - - /// - /// Remove all cached data for this file and if it is a top level archive, any sub-files. - /// - /// - internal void Purge(VirtualFile file) - { - lock(this) - { - // Remove the file - _files.Remove(file.FullPath); - - // If required, remove sub-files - if (file.IsArchive) - { - string prefix = file.FullPath + "|"; - _files.Where(f => f.Key.StartsWith(prefix)).ToList().Do(f => _files.Remove(f.Key)); - } - } - } } - [JsonObject(MemberSerialization.OptIn)] + + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public class VirtualFile { [JsonProperty] - public string[] Paths; + public string[] Paths { get; set; } [JsonProperty] - public string Hash; + public string Hash { get; set; } [JsonProperty] - public long Size; + public long Size { get; set; } [JsonProperty] - public DateTime LastModifiedUTC; + public ulong LastModified { get; set; } + + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? FinishedIndexing { get; set; } + private string _fullPath; - private VirtualFileSystem _vfs; - public VirtualFile(VirtualFileSystem vfs) + public VirtualFile() { - _vfs = vfs; } - [JsonIgnore] private string _stagedPath; + public string FullPath { get @@ -247,6 +314,7 @@ namespace VirtualFileSystem } } + /// @@ -257,7 +325,7 @@ namespace VirtualFileSystem get { if (Paths.Length == 0) return null; - return _vfs[Paths[0]]; + return VirtualFileSystem.VFS[Paths[0]]; } } @@ -265,8 +333,8 @@ namespace VirtualFileSystem { get { - if (Paths.Length == 0) return null; - return _vfs[String.Join("|", Paths.Take(Paths.Length - 1))]; + if (ParentPath == null) return null; + return VirtualFileSystem.VFS.Lookup(ParentPath); } } @@ -279,7 +347,7 @@ namespace VirtualFileSystem _isArchive = FileExtractor.CanExtract(Extension); return (bool)_isArchive; } - } + } public bool IsStaged { @@ -319,7 +387,7 @@ namespace VirtualFileSystem var fio = new FileInfo(StagedPath); Size = fio.Length; Hash = Utils.FileSHA256(StagedPath); - LastModifiedUTC = fio.LastWriteTimeUtc; + LastModified = fio.LastWriteTime.ToMilliseconds(); } @@ -337,7 +405,7 @@ namespace VirtualFileSystem internal string GenerateStagedName() { - _stagedPath = Path.Combine(_vfs._stagedRoot, Guid.NewGuid().ToString() + Path.GetExtension(Paths.Last())); + _stagedPath = Path.Combine(VirtualFileSystem._stagedRoot, Guid.NewGuid().ToString() + Path.GetExtension(Paths.Last())); return _stagedPath; } @@ -359,13 +427,26 @@ namespace VirtualFileSystem if (IsStaged) { var fi = new FileInfo(StagedPath); - if (fi.LastWriteTimeUtc != LastModifiedUTC || fi.Length != Size) + if (fi.LastWriteTime.ToMilliseconds() != LastModified || fi.Length != Size) return true; + if (IsArchive) + if (!FinishedIndexing ?? true) + return true; } return false; } } + + private string _parentPath; + public string ParentPath + { + get { + if (_parentPath == null && !IsConcrete) + _parentPath = String.Join("|", Paths.Take(Paths.Length - 1)); + return _parentPath; + } + } } diff --git a/VirtualFileSystem/VirtualFileSystem.csproj b/VirtualFileSystem/VirtualFileSystem.csproj index b5072a59..973e28c4 100644 --- a/VirtualFileSystem/VirtualFileSystem.csproj +++ b/VirtualFileSystem/VirtualFileSystem.csproj @@ -35,13 +35,13 @@ ..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll - - ..\packages\LiteDB.4.1.4\lib\net40\LiteDB.dll - ..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll + + ..\packages\System.Collections.Immutable.1.5.0\lib\netstandard2.0\System.Collections.Immutable.dll + diff --git a/VirtualFileSystem/packages.config b/VirtualFileSystem/packages.config index 42269a6f..50dc981e 100644 --- a/VirtualFileSystem/packages.config +++ b/VirtualFileSystem/packages.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 2db6ae2c..d29b9e40 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -1,6 +1,7 @@ using ICSharpCode.SharpZipLib.BZip2; using IniParser; using Newtonsoft.Json; +using Newtonsoft.Json.Bson; using System; using System.Collections.Generic; using System.Diagnostics; @@ -16,7 +17,28 @@ namespace Wabbajack.Common { public static class Utils { + private static Action _loggerFn; + private static Action _statusFn; + public static void SetLoggerFn(Action f) + { + _loggerFn = f; + } + + public static void SetStatusFn(Action f) + { + _statusFn = f; + } + + public static void Log(string msg) + { + _loggerFn?.Invoke(msg); + } + + public static void Status(string msg, int progress = 0) + { + _statusFn?.Invoke(msg, progress); + } /// @@ -89,6 +111,22 @@ namespace Wabbajack.Common File.WriteAllText(filename, JsonConvert.SerializeObject(obj, Formatting.Indented, new JsonSerializerSettings() {TypeNameHandling = TypeNameHandling.Auto})); } + public static void ToBSON(this T obj, string filename) + { + using(var fo = File.OpenWrite(filename)) + using(var br = new BsonDataWriter(fo)) + { + fo.SetLength(0); + var serializer = JsonSerializer.Create(new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto }); + serializer.Serialize(br, obj); + } + } + + public static ulong ToMilliseconds(this DateTime date) + { + return (ulong)(date - new DateTime(1970, 1, 1)).TotalMilliseconds; + } + public static string ToJSON(this T obj) { return JsonConvert.SerializeObject(obj, Formatting.Indented, new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto }); @@ -99,6 +137,17 @@ namespace Wabbajack.Common return JsonConvert.DeserializeObject(File.ReadAllText(filename), new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto }); } + public static T FromBSON(this string filename, bool root_is_array = false) + { + using (var fo = File.OpenRead(filename)) + using (var br = new BsonDataReader(fo, readRootValueAsArray: root_is_array, DateTimeKind.Local)) + { + var serializer = JsonSerializer.Create(new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto }); + return serializer.Deserialize(br); + } + + } + public static T FromJSONString(this string data) { return JsonConvert.DeserializeObject(data, new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto }); diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 666ace66..b93efb26 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -62,6 +62,9 @@ ..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll + + ..\packages\Newtonsoft.Json.Bson.1.0.2\lib\net45\Newtonsoft.Json.Bson.dll + diff --git a/Wabbajack.Common/packages.config b/Wabbajack.Common/packages.config index 3f87e68c..1d6931e8 100644 --- a/Wabbajack.Common/packages.config +++ b/Wabbajack.Common/packages.config @@ -3,6 +3,7 @@ + \ No newline at end of file