From 672cf49f471b008f030ed871d908eeec35c9c98d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 12 Aug 2019 16:20:45 -0600 Subject: [PATCH 1/7] Switch back to a lz4 compressor that works --- Compression.BSA/BSABuilder.cs | 13 ++++--------- Compression.BSA/BSAReader.cs | 8 +++----- Compression.BSA/Compression.BSA.csproj | 22 ++++++++++++++++++++-- Compression.BSA/Utils.cs | 8 +------- Compression.BSA/packages.config | 8 +++++++- 5 files changed, 35 insertions(+), 24 deletions(-) diff --git a/Compression.BSA/BSABuilder.cs b/Compression.BSA/BSABuilder.cs index 09f3f3e4..d8460b65 100644 --- a/Compression.BSA/BSABuilder.cs +++ b/Compression.BSA/BSABuilder.cs @@ -1,10 +1,10 @@ -using lz4; +using K4os.Compression.LZ4; +using K4os.Compression.LZ4.Streams; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; -using System.Threading.Tasks; namespace Compression.BSA { @@ -327,13 +327,8 @@ namespace Compression.BSA if (_bsa.HeaderType == VersionType.SSE) { var r = new MemoryStream(); - - using (var f = LZ4Stream.CreateCompressor(r, LZ4StreamMode.Write, LZ4FrameBlockMode.Independent, LZ4FrameBlockSize.Max4MB, LZ4FrameChecksumMode.Content, - highCompression: true, leaveInnerStreamOpen: true)) - { - - new MemoryStream(_rawData).CopyTo(f); - } + using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings() { CompressionLevel = LZ4Level.L10_OPT})) + (new MemoryStream(_rawData)).CopyTo(w); _rawData = r.ToArray(); diff --git a/Compression.BSA/BSAReader.cs b/Compression.BSA/BSAReader.cs index cf7b3097..865b72be 100644 --- a/Compression.BSA/BSAReader.cs +++ b/Compression.BSA/BSAReader.cs @@ -3,7 +3,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Text; -using lz4; +using K4os.Compression.LZ4.Streams; namespace Compression.BSA { @@ -362,10 +362,8 @@ namespace Compression.BSA file_size -= 4; if (_bsa.HeaderType == VersionType.SSE) { - using (var dc = LZ4Stream.CreateDecompressor(output, LZ4StreamMode.Write, true)) - { - rdr.BaseStream.CopyToLimit(dc, file_size); - } + var r = LZ4Stream.Decode(rdr.BaseStream); + r.CopyTo(output); } else { diff --git a/Compression.BSA/Compression.BSA.csproj b/Compression.BSA/Compression.BSA.csproj index 7a7ecddb..69c3e49d 100644 --- a/Compression.BSA/Compression.BSA.csproj +++ b/Compression.BSA/Compression.BSA.csproj @@ -50,12 +50,30 @@ MinimumRecommendedRules.ruleset - - ..\packages\IonKiwi.lz4.net.1.0.12\lib\net472\lz4.AnyCPU.loader.dll + + ..\packages\K4os.Compression.LZ4.1.1.11\lib\net46\K4os.Compression.LZ4.dll + + + ..\packages\K4os.Compression.LZ4.Streams.1.1.11\lib\net46\K4os.Compression.LZ4.Streams.dll + + + ..\packages\K4os.Hash.xxHash.1.0.6\lib\net46\K4os.Hash.xxHash.dll + + ..\packages\System.Buffers.4.4.0\lib\netstandard2.0\System.Buffers.dll + + + ..\packages\System.Memory.4.5.3\lib\netstandard2.0\System.Memory.dll + + + ..\packages\System.Numerics.Vectors.4.4.0\lib\net46\System.Numerics.Vectors.dll + + + ..\packages\System.Runtime.CompilerServices.Unsafe.4.5.2\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll + diff --git a/Compression.BSA/Utils.cs b/Compression.BSA/Utils.cs index 6272c174..31117c22 100644 --- a/Compression.BSA/Utils.cs +++ b/Compression.BSA/Utils.cs @@ -1,5 +1,4 @@ -using lz4.AnyCPU.loader; -using System; +using System; using System.Collections.Generic; using System.IO; using System.Text; @@ -10,11 +9,6 @@ namespace Compression.BSA { private static Encoding Windows1251 = Encoding.GetEncoding(1251); - static Utils () - { - LZ4Loader.DisableVCRuntimeDetection = true; - } - public static string ReadStringLen(this BinaryReader rdr) { var len = rdr.ReadByte(); diff --git a/Compression.BSA/packages.config b/Compression.BSA/packages.config index 2b4a3d68..0fb520a3 100644 --- a/Compression.BSA/packages.config +++ b/Compression.BSA/packages.config @@ -1,4 +1,10 @@  - + + + + + + + \ No newline at end of file From 1a7836ec2af366c32f8fd6b258d7f0fd8b646d44 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 14 Aug 2019 22:30:37 -0600 Subject: [PATCH 2/7] started work on VFS should simplify and greatly speed up really complex modlists --- VirtualFileSystem.Test/App.config | 6 + VirtualFileSystem.Test/Program.cs | 20 + .../Properties/AssemblyInfo.cs | 36 ++ .../VirtualFileSystem.Test.csproj | 63 +++ VirtualFileSystem/Properties/AssemblyInfo.cs | 36 ++ VirtualFileSystem/VirtualFileSystem.cs | 372 ++++++++++++++++++ VirtualFileSystem/VirtualFileSystem.csproj | 76 ++++ VirtualFileSystem/packages.config | 6 + Wabbajack.Common/Consts.cs | 1 + Wabbajack.Common/Data.cs | 13 + Wabbajack.Common/FileExtractor.cs | 24 +- Wabbajack.Common/HashCache.cs | 127 ++++++ Wabbajack.Common/Utils.cs | 18 + Wabbajack.Common/Wabbajack.Common.csproj | 1 + Wabbajack.sln | 28 ++ Wabbajack/Compiler.cs | 161 ++++++-- 16 files changed, 958 insertions(+), 30 deletions(-) create mode 100644 VirtualFileSystem.Test/App.config create mode 100644 VirtualFileSystem.Test/Program.cs create mode 100644 VirtualFileSystem.Test/Properties/AssemblyInfo.cs create mode 100644 VirtualFileSystem.Test/VirtualFileSystem.Test.csproj create mode 100644 VirtualFileSystem/Properties/AssemblyInfo.cs create mode 100644 VirtualFileSystem/VirtualFileSystem.cs create mode 100644 VirtualFileSystem/VirtualFileSystem.csproj create mode 100644 VirtualFileSystem/packages.config create mode 100644 Wabbajack.Common/HashCache.cs diff --git a/VirtualFileSystem.Test/App.config b/VirtualFileSystem.Test/App.config new file mode 100644 index 00000000..56efbc7b --- /dev/null +++ b/VirtualFileSystem.Test/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/VirtualFileSystem.Test/Program.cs b/VirtualFileSystem.Test/Program.cs new file mode 100644 index 00000000..c36cfbca --- /dev/null +++ b/VirtualFileSystem.Test/Program.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace VirtualFileSystem.Test +{ + class Program + { + static void Main(string[] args) + { + WorkQueue.Init((a, b, c) => { return; }, + (a, b) => { return; }); + var vfs = new VirtualFileSystem(); + vfs.AddRoot(@"D:\MO2 Instances\Mod Organizer 2", s => Console.WriteLine(s)); + } + } +} diff --git a/VirtualFileSystem.Test/Properties/AssemblyInfo.cs b/VirtualFileSystem.Test/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..7746488b --- /dev/null +++ b/VirtualFileSystem.Test/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("VirtualFileSystem.Test")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("VirtualFileSystem.Test")] +[assembly: AssemblyCopyright("Copyright © 2019")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("a2913dfe-18ff-468b-a6c1-55f7c0cc0ce8")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/VirtualFileSystem.Test/VirtualFileSystem.Test.csproj b/VirtualFileSystem.Test/VirtualFileSystem.Test.csproj new file mode 100644 index 00000000..218b1aae --- /dev/null +++ b/VirtualFileSystem.Test/VirtualFileSystem.Test.csproj @@ -0,0 +1,63 @@ + + + + + Debug + AnyCPU + {A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8} + Exe + VirtualFileSystem.Test + VirtualFileSystem.Test + v4.7.2 + 512 + true + true + + + x64 + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + {5128b489-bc28-4f66-9f0b-b4565af36cbc} + VirtualFileSystem + + + {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5} + Wabbajack.Common + + + + \ No newline at end of file diff --git a/VirtualFileSystem/Properties/AssemblyInfo.cs b/VirtualFileSystem/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..3693b24e --- /dev/null +++ b/VirtualFileSystem/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("VirtualFileSystem")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("VirtualFileSystem")] +[assembly: AssemblyCopyright("Copyright © 2019")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("5128b489-bc28-4f66-9f0b-b4565af36cbc")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/VirtualFileSystem/VirtualFileSystem.cs b/VirtualFileSystem/VirtualFileSystem.cs new file mode 100644 index 00000000..4689da27 --- /dev/null +++ b/VirtualFileSystem/VirtualFileSystem.cs @@ -0,0 +1,372 @@ +using Compression.BSA; +using ICSharpCode.SharpZipLib.Zip; +using Newtonsoft.Json; +using SevenZipExtractor; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace VirtualFileSystem +{ + public class VirtualFileSystem + { + private Dictionary _files = new Dictionary(); + internal string _stagedRoot; + + public VirtualFileSystem() + { + _stagedRoot = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "vfs_staged_files"); + Directory.CreateDirectory(_stagedRoot); + } + + /// + /// Adds the root path to the filesystem. This may take quite some time as every file in the folder will be hashed, + /// and every archive examined. + /// + /// + public void AddRoot(string path, Action status) + { + IndexPath(path, status); + } + + private void SyncToDisk() + { + lock (this) + { + _files.Values.ToList().ToJSON("vfs_cache.json"); + } + } + + private void IndexPath(string path, Action status) + { + Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories) + .PMap(f => UpdateFile(f)); + } + + private void UpdateFile(string f) + { + TOP: + Console.WriteLine(f); + var lv = Lookup(f); + if (lv == null) + { + lv = new VirtualFile(this) + { + Paths = new string[] { f } + }; + this[f] = lv; + lv.Analyze(); + if (lv.IsArchive) + { + UpdateArchive(lv); + } + } + if (lv.IsOutdated) + { + Purge(lv); + goto TOP; + } + } + + private void UpdateArchive(VirtualFile f) + { + var entries = GetArchiveEntryNames(f); + var new_files = entries.Select(e => { + var new_path = new string[f.Paths.Length + 1]; + f.Paths.CopyTo(new_path, 0); + new_path[f.Paths.Length] = e; + var nf = new VirtualFile(this) + { + Paths = new_path, + }; + this[nf.FullPath] = nf; + return nf; + }).ToList(); + + // Stage the files in the archive + Stage(new_files); + // Analyze them + new_files.Do(file => file.Analyze()); + // Recurse into any archives in this archive + new_files.Where(file => file.IsArchive).Do(file => UpdateArchive(f)); + // Unstage the file + new_files.Where(file => file.IsStaged).Do(file => file.Unstage()); + + SyncToDisk(); + + } + + private void Stage(IEnumerable files) + { + var grouped = files.GroupBy(f => f.ParentArchive) + .OrderBy(f => f.Key == null ? 0 : f.Key.Paths.Length) + .ToList(); + + foreach (var group in grouped) + { + var indexed = group.ToDictionary(e => e.Paths[group.Key.Paths.Length]); + FileExtractor.Extract(group.Key.StagedPath, e => + { + if (indexed.TryGetValue(e.Name, out var file)) + { + return File.OpenWrite(file.GenerateStagedName()); + } + return null; + }); + } + } + + internal VirtualFile Lookup(string path) + { + lock(this) + { + if (_files.TryGetValue(path, out VirtualFile value)) + return value; + return null; + } + } + + public VirtualFile this[string path] + { + get + { + return Lookup(path); + } + set + { + lock(this) + { + _files[path] = value; + } + } + } + + internal List GetArchiveEntryNames(VirtualFile file) + { + if (!file.IsStaged) + throw new InvalidDataException("File is not staged"); + + if (file.Extension == ".bsa") { + using (var ar = new BSAReader(file.StagedPath)) + { + return ar.Files.Select(f => f.Path).ToList(); + } + } + if (file.Extension == ".zip") + { + using (var s = new ZipFile(File.OpenRead(file.StagedPath))) + { + s.IsStreamOwner = true; + s.UseZip64 = UseZip64.On; + + if (s.OfType().FirstOrDefault(e => !e.CanDecompress) == null) + { + return s.OfType() + .Where(f => f.IsFile) + .Select(f => f.Name.Replace('/', '\\')) + .ToList(); + } + } + } + + using (var e = new ArchiveFile(file.StagedPath)) + { + return e.Entries + .Where(f => !f.IsFolder) + .Select(f => f.FileName).ToList(); + } + + } + + + + /// + /// Remove all cached data for this file and if it is a top level archive, any sub-files. + /// + /// + internal void Purge(VirtualFile file) + { + lock(this) + { + // Remove the file + _files.Remove(file.FullPath); + + // If required, remove sub-files + if (file.IsArchive) + { + string prefix = file.FullPath + "|"; + _files.Where(f => f.Key.StartsWith(prefix)).ToList().Do(f => _files.Remove(f.Key)); + } + } + } + } + + [JsonObject(MemberSerialization.OptIn)] + public class VirtualFile + { + [JsonProperty] + public string[] Paths; + [JsonProperty] + public string Hash; + [JsonProperty] + public long Size; + [JsonProperty] + public DateTime LastModifiedUTC; + + private string _fullPath; + private VirtualFileSystem _vfs; + + public VirtualFile(VirtualFileSystem vfs) + { + _vfs = vfs; + } + + [JsonIgnore] + private string _stagedPath; + + public string FullPath + { + get + { + if (_fullPath != null) return _fullPath; + _fullPath = String.Join("|", Paths); + return _fullPath; + } + } + + public string Extension + { + get + { + return Path.GetExtension(Paths.Last()); + } + } + + + + /// + /// If this file is in an archive, return the Archive File, otherwise return null. + /// + public VirtualFile TopLevelArchive + { + get + { + if (Paths.Length == 0) return null; + return _vfs[Paths[0]]; + } + } + + public VirtualFile ParentArchive + { + get + { + if (Paths.Length == 0) return null; + return _vfs[String.Join("|", Paths.Take(Paths.Length - 1))]; + } + } + + private bool? _isArchive; + public bool IsArchive + { + get + { + if (_isArchive == null) + _isArchive = FileExtractor.CanExtract(Extension); + return (bool)_isArchive; + } + } + + public bool IsStaged + { + get + { + if (IsConcrete) return true; + return _stagedPath != null; + } + } + + public string StagedPath + { + get + { + if (!IsStaged) + throw new InvalidDataException("File is not staged"); + if (IsConcrete) return Paths[0]; + return _stagedPath; + } + } + + public FileStream OpenRead() + { + if (!IsStaged) + throw new InvalidDataException("File is not staged, cannot open"); + return File.OpenRead(_stagedPath); + } + + /// + /// Calulate the file's SHA, size and last modified + /// + internal void Analyze() + { + if (!IsStaged) + throw new InvalidDataException("Cannot analzye a unstaged file"); + + var fio = new FileInfo(StagedPath); + Size = fio.Length; + Hash = Utils.FileSHA256(StagedPath); + LastModifiedUTC = fio.LastWriteTimeUtc; + } + + + /// + /// Delete the temoporary file associated with this file + /// + internal void Unstage() + { + if (IsStaged && !IsConcrete) + { + File.Delete(_stagedPath); + _stagedPath = null; + } + } + + internal string GenerateStagedName() + { + _stagedPath = Path.Combine(_vfs._stagedRoot, Guid.NewGuid().ToString() + Path.GetExtension(Paths.Last())); + return _stagedPath; + } + + /// + /// Returns true if this file always exists on-disk, and doesn't need to be staged. + /// + public bool IsConcrete + { + get + { + return Paths.Length == 1; + } + } + + public bool IsOutdated + { + get + { + if (IsStaged) + { + var fi = new FileInfo(StagedPath); + if (fi.LastWriteTimeUtc != LastModifiedUTC || fi.Length != Size) + return true; + } + return false; + } + + } + } + + +} diff --git a/VirtualFileSystem/VirtualFileSystem.csproj b/VirtualFileSystem/VirtualFileSystem.csproj new file mode 100644 index 00000000..b5072a59 --- /dev/null +++ b/VirtualFileSystem/VirtualFileSystem.csproj @@ -0,0 +1,76 @@ + + + + + Debug + AnyCPU + {5128B489-BC28-4F66-9F0B-B4565AF36CBC} + Library + Properties + VirtualFileSystem + VirtualFileSystem + v4.7.2 + 512 + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + x64 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll + + + ..\packages\LiteDB.4.1.4\lib\net40\LiteDB.dll + + + ..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll + + + + + + + + + + + + + + + + + + {ff5d892f-8ff4-44fc-8f7f-cd58f307ad1b} + Compression.BSA + + + {8aa97f58-5044-4bba-b8d9-a74b6947a660} + SevenZipExtractor + + + {b3f3fb6e-b9eb-4f49-9875-d78578bc7ae5} + Wabbajack.Common + + + + + + + \ No newline at end of file diff --git a/VirtualFileSystem/packages.config b/VirtualFileSystem/packages.config new file mode 100644 index 00000000..42269a6f --- /dev/null +++ b/VirtualFileSystem/packages.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 47fcaf62..bf2fc02e 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -30,5 +30,6 @@ namespace Wabbajack.Common public static string WABBAJACK_INCLUDE = "WABBAJACK_INCLUDE"; public static String AppName = "Wabbajack"; + public static string HashCacheName = "Wabbajack.hash_cache"; } } diff --git a/Wabbajack.Common/Data.cs b/Wabbajack.Common/Data.cs index 0548d128..6c16310a 100644 --- a/Wabbajack.Common/Data.cs +++ b/Wabbajack.Common/Data.cs @@ -33,6 +33,11 @@ namespace Wabbajack.Common v.To = Path; return v; } + + public void LoadHashFromCache(HashCache cache) + { + _hash = cache.HashFile(AbsolutePath); + } } public class ModList @@ -174,6 +179,14 @@ namespace Wabbajack.Common public List Headers; } + /// + /// A URL that cannot be downloaded automatically and has to be downloaded by hand + /// + public class ManualURLArchive : Archive + { + public string URL; + } + /// /// An archive that requires additional HTTP headers. /// diff --git a/Wabbajack.Common/FileExtractor.cs b/Wabbajack.Common/FileExtractor.cs index ea80cdd1..011e2d7f 100644 --- a/Wabbajack.Common/FileExtractor.cs +++ b/Wabbajack.Common/FileExtractor.cs @@ -108,11 +108,22 @@ namespace Wabbajack.Common } } + /// + /// Returns true if the given extension type can be extracted + /// + /// + /// + public static bool CanExtract(string v) + { + return Consts.SupportedArchives.Contains(v) || v == ".bsa"; + } public static void DeepExtract(string file, IEnumerable files, Func fnc, bool leave_open = false, int depth = 1) { // Files we need to extract at this level - var files_for_level = files.Where(f => f.ArchiveHashPath.Length == depth).ToDictionary(e => e.From); + var files_for_level = files.Where(f => f.ArchiveHashPath.Length == depth) + .GroupBy(e => e.From) + .ToDictionary(e => e.Key); // Archives we need to extract at this level var archives_for_level = files.Where(f => f.ArchiveHashPath.Length > depth) .GroupBy(f => f.ArchiveHashPath[depth]) @@ -127,12 +138,21 @@ namespace Wabbajack.Common if (files_for_level.TryGetValue(e.Name, out var fe)) { - a = fnc(fe, e); + foreach (var inner_fe in fe) + { + var str = fnc(inner_fe, e); + if (str == null) continue; + a = new SplittingStream(a, false, fnc(inner_fe, e), leave_open); + } } if (archives_for_level.TryGetValue(e.Name, out var archive)) { var name = Path.GetTempFileName() + Path.GetExtension(e.Name); + if (disk_archives.ContainsKey(e.Name)) + { + + } disk_archives.Add(e.Name, name); b = File.OpenWrite(name); } diff --git a/Wabbajack.Common/HashCache.cs b/Wabbajack.Common/HashCache.cs new file mode 100644 index 00000000..62a21562 --- /dev/null +++ b/Wabbajack.Common/HashCache.cs @@ -0,0 +1,127 @@ +using Compression.BSA; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Wabbajack.Common +{ + + public class HashCache : IDisposable + { + public class Entry + { + public string name; + public string hash; + public long size; + public DateTime last_modified; + } + + public class BSA + { + public string full_path; + public string hash; + public long size; + public DateTime last_modified; + public Dictionary entries; + } + + private ConcurrentDictionary _hashes = new ConcurrentDictionary(); + private ConcurrentDictionary _bsas = new ConcurrentDictionary(); + private bool disposed; + + public class DB + { + public List entries; + public List bsas; + } + + public HashCache() + { + if (Consts.HashCacheName.FileExists()) + { + var json = Consts.HashCacheName.FromJSON(); + _hashes = new ConcurrentDictionary(json.entries.Select(e => new KeyValuePair(e.name, e))); + _bsas = new ConcurrentDictionary(json.bsas.Select(e => new KeyValuePair(e.full_path, e))); + } + } + + public string HashFile(string filename) + { + TOP: + var result = _hashes.GetOrAdd(filename, + s => + { + var fi = new FileInfo(filename); + return new Entry + { + name = filename, + hash = Utils.FileSHA256(filename), + size = fi.Length, + last_modified = fi.LastWriteTimeUtc + }; + }); + + var info = new FileInfo(filename); + if (info.LastWriteTimeUtc != result.last_modified || info.Length != result.size) + { + _hashes.TryRemove(filename, out Entry v); + goto TOP; + } + return result.hash; + } + + public void Dispose() + { + if (disposed) return; + new DB + { + entries = _hashes.Values.ToList(), + bsas = _bsas.Values.ToList() + }.ToJSON(Consts.HashCacheName); + disposed = true; + _hashes = null; + _bsas = null; + } + + public List<(string, string)> HashBSA(string absolutePath, Action status) + { + TOP: + var finfo = new FileInfo(absolutePath); + if (_bsas.TryGetValue(absolutePath, out BSA ar)) + { + if (ar.last_modified == finfo.LastWriteTimeUtc && ar.size == finfo.Length) + return ar.entries.Select(kv => (kv.Key, kv.Value)).ToList(); + + _bsas.TryRemove(absolutePath, out BSA value); + } + + var bsa = new BSA() + { + full_path = absolutePath, + size = finfo.Length, + last_modified = finfo.LastAccessTimeUtc, + }; + + var entries = new ConcurrentBag<(string, string)>(); + status($"Hashing BSA: {absolutePath}"); + + using (var a = new BSAReader(absolutePath)) + { + a.Files.PMap(entry => + { + status($"Hashing BSA: {absolutePath} - {entry.Path}"); + var data = entry.GetData(); + entries.Add((entry.Path, data.SHA256())); + }); + } + bsa.entries = entries.ToDictionary(e => e.Item1, e => e.Item2); + _bsas.TryAdd(absolutePath, bsa); + + goto TOP; + } + } +} diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 2e9658a4..2db6ae2c 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -269,6 +269,24 @@ namespace Wabbajack.Common File.WriteAllText($"{DateTime.Now.ToString("yyyyMMddTHHmmss_crash_log.txt")}", ExceptionToString(e)); } + public static IEnumerable DistinctBy(this IEnumerable vs, Func select) + { + HashSet set = new HashSet(); + foreach (var v in vs) { + var key = select(v); + if (set.Contains(key)) continue; + yield return v; + } + + } + + public static T Last(this T[] a) + { + if (a == null || a.Length == 0) + throw new InvalidDataException("null or empty array"); + return a[a.Length - 1]; + } + public static V GetOrDefault(this IDictionary dict, K key) { if (dict.TryGetValue(key, out V v)) return v; diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 5083f1d0..666ace66 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -78,6 +78,7 @@ + diff --git a/Wabbajack.sln b/Wabbajack.sln index 68464d5c..c79e480f 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -20,6 +20,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution RECIPES.md = RECIPES.md EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VirtualFileSystem", "VirtualFileSystem\VirtualFileSystem.csproj", "{5128B489-BC28-4F66-9F0B-B4565AF36CBC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VirtualFileSystem.Test", "VirtualFileSystem.Test\VirtualFileSystem.Test.csproj", "{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug (no commandargs)|Any CPU = Debug (no commandargs)|Any CPU @@ -90,6 +94,30 @@ Global {BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Release|Any CPU.Build.0 = Release|Any CPU {BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Release|x64.ActiveCfg = Release|x64 {BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Release|x64.Build.0 = Release|x64 + {5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU + {5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU + {5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU + {5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU + {5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|x64.ActiveCfg = Debug|Any CPU + {5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|x64.Build.0 = Debug|Any CPU + {5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|Any CPU.Build.0 = Release|Any CPU + {5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|x64.ActiveCfg = Release|Any CPU + {5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|x64.Build.0 = Release|Any CPU + {A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU + {A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU + {A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU + {A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU + {A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|x64.ActiveCfg = Debug|Any CPU + {A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|x64.Build.0 = Debug|Any CPU + {A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|Any CPU.Build.0 = Release|Any CPU + {A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|x64.ActiveCfg = Release|Any CPU + {A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Wabbajack/Compiler.cs b/Wabbajack/Compiler.cs index 13ac5736..8b9b1046 100644 --- a/Wabbajack/Compiler.cs +++ b/Wabbajack/Compiler.cs @@ -63,6 +63,15 @@ namespace Wabbajack public List IndexedFiles { get; private set; } + public class IndexedFileMatch + { + public IndexedArchive Archive; + public IndexedArchiveEntry Entry; + public DateTime LastModified; + } + + public Dictionary> DirectMatchIndex; + public void Info(string msg, params object[] args) { if (args.Length > 0) @@ -249,6 +258,23 @@ namespace Wabbajack Info("Searching for mod files"); AllFiles = mo2_files.Concat(game_files).ToList(); + Info("Hashing files"); + + HashCache cache; + + using (cache = new HashCache()) + { + AllFiles.PMap(f => { + Status($"Hashing {f.Path}"); + try + { + f.LoadHashFromCache(cache); + } + catch (IOException ex) { } + return f; + }); + } + Info("Found {0} files to build into mod list", AllFiles.Count); ExtraFiles = new ConcurrentBag(); @@ -265,11 +291,15 @@ namespace Wabbajack .Where(f => f.Item2 != null) .ToDictionary(f => f.Item1, f => f.Item2); - var stack = MakeStack(); + cache = new HashCache(); + + var stack = MakeStack(cache); Info("Running Compilation Stack"); var results = AllFiles.PMap(f => RunStack(stack, f)).ToList(); + cache.Dispose(); + // Add the extra files that were generated by the stack Info($"Adding {ExtraFiles.Count} that were generated by the stack"); results = results.Concat(ExtraFiles).ToList(); @@ -356,14 +386,16 @@ namespace Wabbajack private void BuildArchivePatches(string archive_sha, IEnumerable group, Dictionary absolute_paths) { var archive = IndexedArchives.First(a => a.Hash == archive_sha); - var paths = group.Select(g => g.FullPath).ToHashSet(); + var paths = group.Select(g => g.FullPath) + .ToHashSet(); var streams = new Dictionary(); Status($"Extracting {paths.Count} patch files from {archive.Name}"); // First we fetch the source files from the input archive - FileExtractor.DeepExtract(archive.AbsolutePath, group, (fe, entry) => + FileExtractor.DeepExtract(archive.AbsolutePath, group.DistinctBy(f => f.FullPath), (fe, entry) => { if (!paths.Contains(fe.FullPath)) return null; + if (streams.ContainsKey(fe.FullPath)) return null; var result = new MemoryStream(); streams.Add(fe.FullPath, result); @@ -520,6 +552,13 @@ namespace Wabbajack } result = tmp; } + else if (general.manualURL != null) + { + result = new ManualURLArchive() + { + URL = general.manualURL.ToString() + }; + } else { Error("No way to handle archive {0} but it's required by the modpack", found.Name); @@ -540,10 +579,12 @@ namespace Wabbajack private Directive RunStack(IEnumerable> stack, RawSourceFile source) { Status("Compiling {0}", source.Path); - return (from f in stack - let result = f(source) - where result != null - select result).First(); + foreach (var f in stack) + { + var result = f(source); + if (result != null) return result; + } + throw new InvalidDataException("Data fell out of the compilation stack"); } @@ -553,7 +594,7 @@ namespace Wabbajack /// result included into the pack /// /// - private IEnumerable> MakeStack() + private IEnumerable> MakeStack(HashCache cache) { Info("Generating compilation stack"); return new List>() @@ -562,6 +603,7 @@ namespace Wabbajack IgnoreStartsWith("downloads\\"), IgnoreStartsWith("webcache\\"), IgnoreStartsWith("overwrite\\"), + IgnorePathContains("temporary_logs"), IgnoreEndsWith(".pyc"), IgnoreEndsWith(".log"), IgnoreOtherProfiles(), @@ -575,10 +617,10 @@ namespace Wabbajack IgnoreRegex(Consts.GameFolderFilesDir + "\\\\.*\\.bsa"), IncludeModIniData(), DirectMatch(), - IncludeTaggedFiles(), - DeconstructBSAs(), // Deconstruct BSAs before building patches so we don't generate massive patch files + DeconstructBSAs(cache), // Deconstruct BSAs before building patches so we don't generate massive patch files IncludePatches(), IncludeDummyESPs(), + IncludeTaggedFiles(), // If we have no match at this point for a game folder file, skip them, we can't do anything about them @@ -597,6 +639,22 @@ namespace Wabbajack }; } + private Func IgnorePathContains(string v) + { + v = $"\\{v.Trim('\\')}\\"; + var reason = $"Ignored because path contains {v}"; + return source => + { + if (source.Path.Contains(v)) + { + var result = source.EvolveTo(); + result.Reason = reason; + return result; + } + return null; + }; + } + /// /// If a user includes WABBAJACK_INCLUDE directly in the notes or comments of a mod, the contents of that @@ -670,8 +728,17 @@ namespace Wabbajack /// all of the files. /// /// - private Func DeconstructBSAs() + private Func DeconstructBSAs(HashCache cache) { + var include_directly = ModInis.Where(kv => { + var general = kv.Value.General; + if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE)) + return true; + if (general.comments != null && general.comments.Contains(Consts.WABBAJACK_INCLUDE)) + return true; + return false; + }).Select(kv => $"mods\\{kv.Key}\\"); + var microstack = new List>() { DirectMatch(), @@ -679,11 +746,32 @@ namespace Wabbajack DropAll() }; + var microstack_with_include = new List>() + { + DirectMatch(), + IncludePatches(), + IncludeALL() + }; + + return source => { if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path))) return null; - var hashed = HashBSA(source.AbsolutePath); + bool default_include = false; + if (source.Path.StartsWith("mods")) + { + foreach (var modpath in include_directly) + { + if (source.Path.StartsWith(modpath)) + { + default_include = true; + break; + } + } + } + + var hashed = cache.HashBSA(source.AbsolutePath, s => Status(s)); var source_files = hashed.Select(e => new RawSourceFile() { Hash = e.Item2, @@ -691,8 +779,9 @@ namespace Wabbajack AbsolutePath = e.Item1 }); + var stack = default_include ? microstack_with_include : microstack; - var matches = source_files.Select(e => RunStack(microstack, e)); + var matches = source_files.PMap(e => RunStack(stack, e)); var id = Guid.NewGuid().ToString(); @@ -724,12 +813,22 @@ namespace Wabbajack }; } + private Func IncludeALL() + { + return source => + { + var inline = source.EvolveTo(); + inline.SourceData = File.ReadAllBytes(source.AbsolutePath).ToBase64(); + return inline; + }; + } + /// /// Given a BSA on disk, index it and return a dictionary of SHA256 -> filename /// /// /// - private List<(string, string)> HashBSA(string absolutePath) + private List<(string, string)> HashBSA(HashCache cache, string absolutePath) { Status($"Hashing BSA: {absolutePath}"); var results = new List<(string, string)>(); @@ -909,31 +1008,37 @@ namespace Wabbajack { var archive_shas = IndexedArchives.GroupBy(e => e.Hash) .ToDictionary(e => e.Key); + if (DirectMatchIndex == null) + { - var indexed = (from entry in IndexedFiles - select new { archive = archive_shas[entry.HashPath[0]].First(), - entry = entry }) - .GroupBy(e => e.entry.Hash) - .ToDictionary(e => e.Key); - - + DirectMatchIndex = IndexedFiles.PMap(entry => { + var archive = archive_shas[entry.HashPath[0]].First(); + return new IndexedFileMatch + { + Archive = archive, + Entry = entry, + LastModified = new FileInfo(archive.AbsolutePath).LastAccessTimeUtc + }; + }) + .OrderByDescending(e => e.LastModified) + .GroupBy(e => e.Entry.Hash) + .ToDictionary(e => e.Key, e => e.AsEnumerable()); + } return source => { - if (indexed.TryGetValue(source.Hash, out var found)) + if (DirectMatchIndex.TryGetValue(source.Hash, out var found)) { var result = source.EvolveTo(); - var match = found.Where(f => Path.GetFileName(f.entry.Path) == Path.GetFileName(source.Path)) - .OrderByDescending(f => new FileInfo(f.archive.AbsolutePath).LastWriteTime) + var match = found.Where(f => Path.GetFileName(f.Entry.Path) == Path.GetFileName(source.Path)) .FirstOrDefault(); if (match == null) - match = found.OrderByDescending(f => new FileInfo(f.archive.AbsolutePath).LastWriteTime) - .FirstOrDefault(); + match = found.FirstOrDefault(); - result.ArchiveHashPath = match.entry.HashPath; - result.From = match.entry.Path; + result.ArchiveHashPath = match.Entry.HashPath; + result.From = match.Entry.Path; return result; } return null; From fa8f038f7057ac5961022a1bbd6dc71e364fff1a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 19 Aug 2019 16:08:34 -0600 Subject: [PATCH 3/7] new VFS impl, should make maintaining Wabbajack a bit simpler --- VirtualFileSystem.Test/Program.cs | 5 +- VirtualFileSystem/VirtualFileSystem.cs | 227 ++++++++++++++------- VirtualFileSystem/VirtualFileSystem.csproj | 6 +- VirtualFileSystem/packages.config | 2 +- Wabbajack.Common/Utils.cs | 49 +++++ Wabbajack.Common/Wabbajack.Common.csproj | 3 + Wabbajack.Common/packages.config | 1 + 7 files changed, 214 insertions(+), 79 deletions(-) diff --git a/VirtualFileSystem.Test/Program.cs b/VirtualFileSystem.Test/Program.cs index c36cfbca..55e62483 100644 --- a/VirtualFileSystem.Test/Program.cs +++ b/VirtualFileSystem.Test/Program.cs @@ -11,10 +11,11 @@ namespace VirtualFileSystem.Test { static void Main(string[] args) { + Utils.SetLoggerFn(s => Console.WriteLine(s)); + Utils.SetStatusFn((s, i) => Console.WriteLine(s)); WorkQueue.Init((a, b, c) => { return; }, (a, b) => { return; }); - var vfs = new VirtualFileSystem(); - vfs.AddRoot(@"D:\MO2 Instances\Mod Organizer 2", s => Console.WriteLine(s)); + VirtualFileSystem.VFS.AddRoot(@"D:\MO2 Instances\Mod Organizer 2"); } } } diff --git a/VirtualFileSystem/VirtualFileSystem.cs b/VirtualFileSystem/VirtualFileSystem.cs index 4689da27..932f6bca 100644 --- a/VirtualFileSystem/VirtualFileSystem.cs +++ b/VirtualFileSystem/VirtualFileSystem.cs @@ -4,24 +4,117 @@ using Newtonsoft.Json; using SevenZipExtractor; using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.IO; using System.Linq; using System.Reflection; -using System.Text; -using System.Threading.Tasks; using Wabbajack.Common; namespace VirtualFileSystem { public class VirtualFileSystem { - private Dictionary _files = new Dictionary(); - internal string _stagedRoot; - public VirtualFileSystem() + internal static string _stagedRoot; + public static VirtualFileSystem VFS; + private Dictionary _files = new Dictionary(); + + + public static string RootFolder { get; } + public Dictionary> HashIndex { get; private set; } + + static VirtualFileSystem() { - _stagedRoot = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "vfs_staged_files"); + VFS = new VirtualFileSystem(); + RootFolder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + _stagedRoot = Path.Combine(RootFolder, "vfs_staged_files"); + if (Directory.Exists(_stagedRoot)) + Directory.Delete(_stagedRoot, true); + Directory.CreateDirectory(_stagedRoot); + + } + + public VirtualFileSystem () + { + LoadFromDisk(); + } + + private void LoadFromDisk() + { + Utils.Log("Loading VFS Cache"); + if (!File.Exists("vfs_cache.bson")) return; + _files = "vfs_cache.bson".FromBSON>(root_is_array:true).ToDictionary(f => f.FullPath); + CleanDB(); + } + + public void SyncToDisk() + { + lock(this) + { + _files.Values.OfType().ToBSON("vfs_cache.bson"); + } + } + + + public void Purge(VirtualFile f) + { + var path = f.FullPath + "|"; + lock (this) + { + _files.Values + .Where(v => v.FullPath.StartsWith(path) || v.FullPath == f.FullPath) + .ToList() + .Do(r => { + _files.Remove(r.FullPath); + }); + } + } + + public void Add(VirtualFile f) + { + lock (this) + { + if (_files.ContainsKey(f.FullPath)) + Purge(f); + _files.Add(f.FullPath, f); + } + } + + public VirtualFile Lookup(string f) + { + lock (this) + { + if (_files.TryGetValue(f, out var found)) + return found; + return null; + } + } + + /// + /// Remove any orphaned files in the DB. + /// + private void CleanDB() + { + Utils.Log("Cleaning VFS cache"); + lock (this) + { + _files.Values + .Where(f => + { + if (f.IsConcrete) + return !File.Exists(f.StagedPath); + while (f.ParentPath != null) + { + if (Lookup(f.ParentPath) == null) + return true; + f = Lookup(f.ParentPath); + } + return false; + }) + .ToList() + .Do(f => _files.Remove(f.FullPath)); + } } /// @@ -29,42 +122,50 @@ namespace VirtualFileSystem /// and every archive examined. /// /// - public void AddRoot(string path, Action status) + public void AddRoot(string path) { - IndexPath(path, status); + IndexPath(path); + RefreshIndexes(); } - private void SyncToDisk() + private void RefreshIndexes() { - lock (this) + Utils.Log("Building Hash Index"); + lock(this) { - _files.Values.ToList().ToJSON("vfs_cache.json"); + HashIndex = _files.Values + .GroupBy(f => f.Hash) + .ToDictionary(f => f.Key, f => (IEnumerable)f); } } - private void IndexPath(string path, Action status) + private void IndexPath(string path) { Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories) .PMap(f => UpdateFile(f)); + SyncToDisk(); } private void UpdateFile(string f) { TOP: - Console.WriteLine(f); var lv = Lookup(f); if (lv == null) { - lv = new VirtualFile(this) + Utils.Log($"Analyzing {0}"); + + lv = new VirtualFile() { Paths = new string[] { f } }; - this[f] = lv; + lv.Analyze(); + Add(lv); if (lv.IsArchive) { UpdateArchive(lv); } + // Upsert after extraction incase extraction fails } if (lv.IsOutdated) { @@ -80,11 +181,11 @@ namespace VirtualFileSystem var new_path = new string[f.Paths.Length + 1]; f.Paths.CopyTo(new_path, 0); new_path[f.Paths.Length] = e; - var nf = new VirtualFile(this) + var nf = new VirtualFile() { Paths = new_path, }; - this[nf.FullPath] = nf; + Add(nf); return nf; }).ToList(); @@ -93,12 +194,14 @@ namespace VirtualFileSystem // Analyze them new_files.Do(file => file.Analyze()); // Recurse into any archives in this archive - new_files.Where(file => file.IsArchive).Do(file => UpdateArchive(f)); + new_files.Where(file => file.IsArchive).Do(file => UpdateArchive(file)); // Unstage the file new_files.Where(file => file.IsStaged).Do(file => file.Unstage()); + f.FinishedIndexing = true; SyncToDisk(); + Utils.Log($"{_files.Count} docs in VFS cache"); } private void Stage(IEnumerable files) @@ -121,29 +224,12 @@ namespace VirtualFileSystem } } - internal VirtualFile Lookup(string path) - { - lock(this) - { - if (_files.TryGetValue(path, out VirtualFile value)) - return value; - return null; - } - } - public VirtualFile this[string path] { get { return Lookup(path); } - set - { - lock(this) - { - _files[path] = value; - } - } } internal List GetArchiveEntryNames(VirtualFile file) @@ -182,53 +268,34 @@ namespace VirtualFileSystem } } - - - - /// - /// Remove all cached data for this file and if it is a top level archive, any sub-files. - /// - /// - internal void Purge(VirtualFile file) - { - lock(this) - { - // Remove the file - _files.Remove(file.FullPath); - - // If required, remove sub-files - if (file.IsArchive) - { - string prefix = file.FullPath + "|"; - _files.Where(f => f.Key.StartsWith(prefix)).ToList().Do(f => _files.Remove(f.Key)); - } - } - } } - [JsonObject(MemberSerialization.OptIn)] + + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public class VirtualFile { [JsonProperty] - public string[] Paths; + public string[] Paths { get; set; } [JsonProperty] - public string Hash; + public string Hash { get; set; } [JsonProperty] - public long Size; + public long Size { get; set; } [JsonProperty] - public DateTime LastModifiedUTC; + public ulong LastModified { get; set; } + + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? FinishedIndexing { get; set; } + private string _fullPath; - private VirtualFileSystem _vfs; - public VirtualFile(VirtualFileSystem vfs) + public VirtualFile() { - _vfs = vfs; } - [JsonIgnore] private string _stagedPath; + public string FullPath { get @@ -247,6 +314,7 @@ namespace VirtualFileSystem } } + /// @@ -257,7 +325,7 @@ namespace VirtualFileSystem get { if (Paths.Length == 0) return null; - return _vfs[Paths[0]]; + return VirtualFileSystem.VFS[Paths[0]]; } } @@ -265,8 +333,8 @@ namespace VirtualFileSystem { get { - if (Paths.Length == 0) return null; - return _vfs[String.Join("|", Paths.Take(Paths.Length - 1))]; + if (ParentPath == null) return null; + return VirtualFileSystem.VFS.Lookup(ParentPath); } } @@ -279,7 +347,7 @@ namespace VirtualFileSystem _isArchive = FileExtractor.CanExtract(Extension); return (bool)_isArchive; } - } + } public bool IsStaged { @@ -319,7 +387,7 @@ namespace VirtualFileSystem var fio = new FileInfo(StagedPath); Size = fio.Length; Hash = Utils.FileSHA256(StagedPath); - LastModifiedUTC = fio.LastWriteTimeUtc; + LastModified = fio.LastWriteTime.ToMilliseconds(); } @@ -337,7 +405,7 @@ namespace VirtualFileSystem internal string GenerateStagedName() { - _stagedPath = Path.Combine(_vfs._stagedRoot, Guid.NewGuid().ToString() + Path.GetExtension(Paths.Last())); + _stagedPath = Path.Combine(VirtualFileSystem._stagedRoot, Guid.NewGuid().ToString() + Path.GetExtension(Paths.Last())); return _stagedPath; } @@ -359,13 +427,26 @@ namespace VirtualFileSystem if (IsStaged) { var fi = new FileInfo(StagedPath); - if (fi.LastWriteTimeUtc != LastModifiedUTC || fi.Length != Size) + if (fi.LastWriteTime.ToMilliseconds() != LastModified || fi.Length != Size) return true; + if (IsArchive) + if (!FinishedIndexing ?? true) + return true; } return false; } } + + private string _parentPath; + public string ParentPath + { + get { + if (_parentPath == null && !IsConcrete) + _parentPath = String.Join("|", Paths.Take(Paths.Length - 1)); + return _parentPath; + } + } } diff --git a/VirtualFileSystem/VirtualFileSystem.csproj b/VirtualFileSystem/VirtualFileSystem.csproj index b5072a59..973e28c4 100644 --- a/VirtualFileSystem/VirtualFileSystem.csproj +++ b/VirtualFileSystem/VirtualFileSystem.csproj @@ -35,13 +35,13 @@ ..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll - - ..\packages\LiteDB.4.1.4\lib\net40\LiteDB.dll - ..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll + + ..\packages\System.Collections.Immutable.1.5.0\lib\netstandard2.0\System.Collections.Immutable.dll + diff --git a/VirtualFileSystem/packages.config b/VirtualFileSystem/packages.config index 42269a6f..50dc981e 100644 --- a/VirtualFileSystem/packages.config +++ b/VirtualFileSystem/packages.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 2db6ae2c..d29b9e40 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -1,6 +1,7 @@ using ICSharpCode.SharpZipLib.BZip2; using IniParser; using Newtonsoft.Json; +using Newtonsoft.Json.Bson; using System; using System.Collections.Generic; using System.Diagnostics; @@ -16,7 +17,28 @@ namespace Wabbajack.Common { public static class Utils { + private static Action _loggerFn; + private static Action _statusFn; + public static void SetLoggerFn(Action f) + { + _loggerFn = f; + } + + public static void SetStatusFn(Action f) + { + _statusFn = f; + } + + public static void Log(string msg) + { + _loggerFn?.Invoke(msg); + } + + public static void Status(string msg, int progress = 0) + { + _statusFn?.Invoke(msg, progress); + } /// @@ -89,6 +111,22 @@ namespace Wabbajack.Common File.WriteAllText(filename, JsonConvert.SerializeObject(obj, Formatting.Indented, new JsonSerializerSettings() {TypeNameHandling = TypeNameHandling.Auto})); } + public static void ToBSON(this T obj, string filename) + { + using(var fo = File.OpenWrite(filename)) + using(var br = new BsonDataWriter(fo)) + { + fo.SetLength(0); + var serializer = JsonSerializer.Create(new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto }); + serializer.Serialize(br, obj); + } + } + + public static ulong ToMilliseconds(this DateTime date) + { + return (ulong)(date - new DateTime(1970, 1, 1)).TotalMilliseconds; + } + public static string ToJSON(this T obj) { return JsonConvert.SerializeObject(obj, Formatting.Indented, new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto }); @@ -99,6 +137,17 @@ namespace Wabbajack.Common return JsonConvert.DeserializeObject(File.ReadAllText(filename), new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto }); } + public static T FromBSON(this string filename, bool root_is_array = false) + { + using (var fo = File.OpenRead(filename)) + using (var br = new BsonDataReader(fo, readRootValueAsArray: root_is_array, DateTimeKind.Local)) + { + var serializer = JsonSerializer.Create(new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto }); + return serializer.Deserialize(br); + } + + } + public static T FromJSONString(this string data) { return JsonConvert.DeserializeObject(data, new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto }); diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 666ace66..b93efb26 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -62,6 +62,9 @@ ..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll + + ..\packages\Newtonsoft.Json.Bson.1.0.2\lib\net45\Newtonsoft.Json.Bson.dll + diff --git a/Wabbajack.Common/packages.config b/Wabbajack.Common/packages.config index 3f87e68c..1d6931e8 100644 --- a/Wabbajack.Common/packages.config +++ b/Wabbajack.Common/packages.config @@ -3,6 +3,7 @@ + \ No newline at end of file From c343a74359e9741539dc41e4a87f76c03f274894 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 19 Aug 2019 22:57:08 -0600 Subject: [PATCH 4/7] massive refactoring, it compiles, but it's surely broken --- VirtualFileSystem.Test/Program.cs | 6 +- VirtualFileSystem/VirtualFileSystem.cs | 136 ++++++++- Wabbajack.Common/FileExtractor.cs | 4 + Wabbajack.Common/HashCache.cs | 127 --------- Wabbajack.Common/Wabbajack.Common.csproj | 2 - Wabbajack/AppState.cs | 1 - Wabbajack/Compiler.cs | 337 ++++++----------------- {Wabbajack.Common => Wabbajack}/Data.cs | 65 ++--- Wabbajack/Installer.cs | 65 ++++- Wabbajack/MainWindow.xaml.cs | 4 +- Wabbajack/Wabbajack.csproj | 5 + 11 files changed, 295 insertions(+), 457 deletions(-) delete mode 100644 Wabbajack.Common/HashCache.cs rename {Wabbajack.Common => Wabbajack}/Data.cs (81%) diff --git a/VirtualFileSystem.Test/Program.cs b/VirtualFileSystem.Test/Program.cs index 55e62483..c85b3c9a 100644 --- a/VirtualFileSystem.Test/Program.cs +++ b/VirtualFileSystem.Test/Program.cs @@ -1,8 +1,4 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Wabbajack.Common; namespace VirtualFileSystem.Test @@ -15,7 +11,7 @@ namespace VirtualFileSystem.Test Utils.SetStatusFn((s, i) => Console.WriteLine(s)); WorkQueue.Init((a, b, c) => { return; }, (a, b) => { return; }); - VirtualFileSystem.VFS.AddRoot(@"D:\MO2 Instances\Mod Organizer 2"); + VFS.VirtualFileSystem.VFS.AddRoot(@"D:\MO2 Instances\Mod Organizer 2"); } } } diff --git a/VirtualFileSystem/VirtualFileSystem.cs b/VirtualFileSystem/VirtualFileSystem.cs index 932f6bca..ec4ee82a 100644 --- a/VirtualFileSystem/VirtualFileSystem.cs +++ b/VirtualFileSystem/VirtualFileSystem.cs @@ -10,7 +10,7 @@ using System.Linq; using System.Reflection; using Wabbajack.Common; -namespace VirtualFileSystem +namespace VFS { public class VirtualFileSystem { @@ -18,7 +18,7 @@ namespace VirtualFileSystem internal static string _stagedRoot; public static VirtualFileSystem VFS; private Dictionary _files = new Dictionary(); - + private bool _disableDiskCache; public static string RootFolder { get; } public Dictionary> HashIndex { get; private set; } @@ -26,13 +26,12 @@ namespace VirtualFileSystem static VirtualFileSystem() { VFS = new VirtualFileSystem(); - RootFolder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + RootFolder = ".\\"; _stagedRoot = Path.Combine(RootFolder, "vfs_staged_files"); if (Directory.Exists(_stagedRoot)) Directory.Delete(_stagedRoot, true); Directory.CreateDirectory(_stagedRoot); - } public VirtualFileSystem () @@ -42,20 +41,41 @@ namespace VirtualFileSystem private void LoadFromDisk() { - Utils.Log("Loading VFS Cache"); - if (!File.Exists("vfs_cache.bson")) return; - _files = "vfs_cache.bson".FromBSON>(root_is_array:true).ToDictionary(f => f.FullPath); - CleanDB(); + try + { + Utils.Log("Loading VFS Cache"); + if (!File.Exists("vfs_cache.bson")) return; + _files = "vfs_cache.bson".FromBSON>(root_is_array: true).ToDictionary(f => f.FullPath); + CleanDB(); + } + catch(Exception ex) + { + Utils.Log($"Purging cache due to {ex}"); + File.Delete("vfs_cache.bson"); + _files.Clear(); + } } public void SyncToDisk() { + if (!_disableDiskCache) lock(this) { _files.Values.OfType().ToBSON("vfs_cache.bson"); } } + public IList FilesInArchive(VirtualFile f) + { + var path = f.FullPath + "|"; + lock (this) + { + return _files.Values + .Where(v => v.FullPath.StartsWith(path)) + .ToList(); + } + } + public void Purge(VirtualFile f) { @@ -117,6 +137,36 @@ namespace VirtualFileSystem } } + public void BackfillMissing() + { + lock(this) + { + _files.Values + .Select(f => f.ParentPath) + .Where(s => !_files.ContainsKey(s)) + .ToHashSet() + .Do(s => + { + AddKnown(new VirtualFile() { Paths = s.Split('|') }); + }); + } + } + + /// + /// Add a known file to the index, bit of a hack as we won't assume that all the fields for the archive are filled in. + /// you will need to manually update the SHA hash when you are done adding files, by calling `RefreshIndexes` + /// + /// + public void AddKnown(VirtualFile virtualFile) + { + lock(this) + { + // We don't know enough about these files to be able to store them in the disk cache + _disableDiskCache = true; + _files[virtualFile.FullPath] = virtualFile; + } + } + /// /// Adds the root path to the filesystem. This may take quite some time as every file in the folder will be hashed, /// and every archive examined. @@ -128,7 +178,7 @@ namespace VirtualFileSystem RefreshIndexes(); } - private void RefreshIndexes() + public void RefreshIndexes() { Utils.Log("Building Hash Index"); lock(this) @@ -141,8 +191,9 @@ namespace VirtualFileSystem private void IndexPath(string path) { - Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories) - .PMap(f => UpdateFile(f)); + var file_list = Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories).ToList(); + Utils.Log($"Updating the cache for {file_list.Count} files"); + file_list.PMap(f => UpdateFile(f)); SyncToDisk(); } @@ -152,7 +203,7 @@ namespace VirtualFileSystem var lv = Lookup(f); if (lv == null) { - Utils.Log($"Analyzing {0}"); + Utils.Status($"Analyzing {f}"); lv = new VirtualFile() { @@ -200,11 +251,9 @@ namespace VirtualFileSystem f.FinishedIndexing = true; SyncToDisk(); - - Utils.Log($"{_files.Count} docs in VFS cache"); } - private void Stage(IEnumerable files) + public void Stage(IEnumerable files) { var grouped = files.GroupBy(f => f.ParentArchive) .OrderBy(f => f.Key == null ? 0 : f.Key.Paths.Length) @@ -224,6 +273,13 @@ namespace VirtualFileSystem } } + + + public StagingGroup StageWith(IEnumerable files) + { + return new StagingGroup(files); + } + public VirtualFile this[string path] { get @@ -268,8 +324,36 @@ namespace VirtualFileSystem } } + + /// + /// Given a path that starts with a HASH, return the Virtual file referenced + /// + /// + /// + public VirtualFile FileForArchiveHashPath(string[] archiveHashPath) + { + var archive = HashIndex[archiveHashPath[0]].Where(a => a.IsArchive).OrderByDescending(a => a.LastModified).First(); + string fullPath = HashIndex[archiveHashPath[0]] + "|" + String.Join("|", archiveHashPath.Skip(1)); + return Lookup(fullPath); + } } + public class StagingGroup : List, IDisposable + { + public StagingGroup(IEnumerable files) : base(files) + { + } + + public void Dispose() + { + this.Do(f => f.Unstage()); + } + + internal void Stage() + { + VirtualFileSystem.VFS.Stage(this); + } + } [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public class VirtualFile @@ -367,6 +451,12 @@ namespace VirtualFileSystem if (IsConcrete) return Paths[0]; return _stagedPath; } + set + { + if (IsStaged && value != null) + throw new InvalidDataException("Can't change the path of a already staged file"); + _stagedPath = value; + } } public FileStream OpenRead() @@ -405,6 +495,7 @@ namespace VirtualFileSystem internal string GenerateStagedName() { + if (_stagedPath != null) return _stagedPath; _stagedPath = Path.Combine(VirtualFileSystem._stagedRoot, Guid.NewGuid().ToString() + Path.GetExtension(Paths.Last())); return _stagedPath; } @@ -447,6 +538,21 @@ namespace VirtualFileSystem return _parentPath; } } + + public IEnumerable FileInArchive + { + get + { + return VirtualFileSystem.VFS.FilesInArchive(this); + } + } + + public string[] MakeRelativePaths() + { + var path_copy = (string[])Paths.Clone(); + path_copy[0] = VirtualFileSystem.VFS.Lookup(Paths[0]).Hash; + return path_copy; + } } diff --git a/Wabbajack.Common/FileExtractor.cs b/Wabbajack.Common/FileExtractor.cs index 011e2d7f..6b3e5469 100644 --- a/Wabbajack.Common/FileExtractor.cs +++ b/Wabbajack.Common/FileExtractor.cs @@ -118,6 +118,8 @@ namespace Wabbajack.Common return Consts.SupportedArchives.Contains(v) || v == ".bsa"; } + // Probably replace this with VFS? + /* public static void DeepExtract(string file, IEnumerable files, Func fnc, bool leave_open = false, int depth = 1) { // Files we need to extract at this level @@ -168,6 +170,8 @@ namespace Wabbajack.Common DeepExtract(archive.Value, archives_for_level[archive.Key], fnc, leave_open, depth + 1); File.Delete(archive.Value); } + } + */ } } diff --git a/Wabbajack.Common/HashCache.cs b/Wabbajack.Common/HashCache.cs deleted file mode 100644 index 62a21562..00000000 --- a/Wabbajack.Common/HashCache.cs +++ /dev/null @@ -1,127 +0,0 @@ -using Compression.BSA; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Wabbajack.Common -{ - - public class HashCache : IDisposable - { - public class Entry - { - public string name; - public string hash; - public long size; - public DateTime last_modified; - } - - public class BSA - { - public string full_path; - public string hash; - public long size; - public DateTime last_modified; - public Dictionary entries; - } - - private ConcurrentDictionary _hashes = new ConcurrentDictionary(); - private ConcurrentDictionary _bsas = new ConcurrentDictionary(); - private bool disposed; - - public class DB - { - public List entries; - public List bsas; - } - - public HashCache() - { - if (Consts.HashCacheName.FileExists()) - { - var json = Consts.HashCacheName.FromJSON(); - _hashes = new ConcurrentDictionary(json.entries.Select(e => new KeyValuePair(e.name, e))); - _bsas = new ConcurrentDictionary(json.bsas.Select(e => new KeyValuePair(e.full_path, e))); - } - } - - public string HashFile(string filename) - { - TOP: - var result = _hashes.GetOrAdd(filename, - s => - { - var fi = new FileInfo(filename); - return new Entry - { - name = filename, - hash = Utils.FileSHA256(filename), - size = fi.Length, - last_modified = fi.LastWriteTimeUtc - }; - }); - - var info = new FileInfo(filename); - if (info.LastWriteTimeUtc != result.last_modified || info.Length != result.size) - { - _hashes.TryRemove(filename, out Entry v); - goto TOP; - } - return result.hash; - } - - public void Dispose() - { - if (disposed) return; - new DB - { - entries = _hashes.Values.ToList(), - bsas = _bsas.Values.ToList() - }.ToJSON(Consts.HashCacheName); - disposed = true; - _hashes = null; - _bsas = null; - } - - public List<(string, string)> HashBSA(string absolutePath, Action status) - { - TOP: - var finfo = new FileInfo(absolutePath); - if (_bsas.TryGetValue(absolutePath, out BSA ar)) - { - if (ar.last_modified == finfo.LastWriteTimeUtc && ar.size == finfo.Length) - return ar.entries.Select(kv => (kv.Key, kv.Value)).ToList(); - - _bsas.TryRemove(absolutePath, out BSA value); - } - - var bsa = new BSA() - { - full_path = absolutePath, - size = finfo.Length, - last_modified = finfo.LastAccessTimeUtc, - }; - - var entries = new ConcurrentBag<(string, string)>(); - status($"Hashing BSA: {absolutePath}"); - - using (var a = new BSAReader(absolutePath)) - { - a.Files.PMap(entry => - { - status($"Hashing BSA: {absolutePath} - {entry.Path}"); - var data = entry.GetData(); - entries.Add((entry.Path, data.SHA256())); - }); - } - bsa.entries = entries.ToDictionary(e => e.Item1, e => e.Item2); - _bsas.TryAdd(absolutePath, bsa); - - goto TOP; - } - } -} diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index b93efb26..22e41be8 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -78,10 +78,8 @@ - - diff --git a/Wabbajack/AppState.cs b/Wabbajack/AppState.cs index 76f9a743..9ccd4769 100644 --- a/Wabbajack/AppState.cs +++ b/Wabbajack/AppState.cs @@ -312,7 +312,6 @@ namespace Wabbajack { try { - compiler.LoadArchives(); compiler.Compile(); } catch (Exception ex) diff --git a/Wabbajack/Compiler.cs b/Wabbajack/Compiler.cs index 8b9b1046..78cf118d 100644 --- a/Wabbajack/Compiler.cs +++ b/Wabbajack/Compiler.cs @@ -1,6 +1,5 @@ using Compression.BSA; using Newtonsoft.Json; -using SharpCompress.Archives; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -10,10 +9,10 @@ using System.Reflection; using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; -using System.Threading.Tasks; using System.Web; using Wabbajack.Common; using static Wabbajack.NexusAPI; +using VFS; namespace Wabbajack { @@ -59,9 +58,17 @@ namespace Wabbajack public ConcurrentBag ExtraFiles { get; private set; } public Dictionary ModInis { get; private set; } - public List IndexedArchives; + public VirtualFileSystem VFS + { + get + { + return VirtualFileSystem.VFS; + } - public List IndexedFiles { get; private set; } + } + + public List IndexedArchives { get; private set; } + public Dictionary> IndexedFiles { get; private set; } public class IndexedFileMatch { @@ -103,85 +110,21 @@ namespace Wabbajack GamePath = ((string)MO2Ini.General.gamePath).Replace("\\\\", "\\"); } - - - public void LoadArchives() - { - IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder) - .Where(file => Consts.SupportedArchives.Contains(Path.GetExtension(file))) - .PMap(file => LoadArchive(file)); - IndexedFiles = FlattenFiles(IndexedArchives); - Info($"Found {IndexedFiles.Count} files in archives"); - } - - private List FlattenFiles(IEnumerable archives) - { - return archives.PMap(e => FlattenArchiveEntries(e, null, new string[0])) - .SelectMany(e => e) - .ToList(); - } - - private IEnumerable FlattenArchiveEntries(IndexedArchiveCache archive, string name, string[] path) - { - var new_path = new string[path.Length + 1]; - Array.Copy(path, 0, new_path, 0, path.Length); - new_path[path.Length] = path.Length == 0 ? archive.Hash : name; - - foreach (var e in archive.Entries) - { - yield return new IndexedArchiveEntry() - { - Path = e.Path, - Size = e.Size, - Hash = e.Hash, - HashPath = new_path - }; - } - if (archive.InnerArchives != null) { - foreach (var inner in archive.InnerArchives) - { - foreach (var entry in FlattenArchiveEntries(inner.Value, inner.Key, new_path)) - { - yield return entry; - } - } - } - - } - - - private const int ARCHIVE_CONTENTS_VERSION = 1; private IndexedArchive LoadArchive(string file) { - TOP: - string metaname = file + ".archive_contents"; + var info = new IndexedArchive(); + info.File = VFS.Lookup(file); - if (metaname.FileExists() && new FileInfo(metaname).LastWriteTime >= new FileInfo(file).LastWriteTime) + info.Name = Path.GetFileName(file); + + var ini_name = file + ".meta"; + if (ini_name.FileExists()) { - Status("Loading Archive Index for {0}", Path.GetFileName(file)); - var info = metaname.FromJSON(); - if (info.Version != ARCHIVE_CONTENTS_VERSION) - { - File.Delete(metaname); - goto TOP; - } - - info.Name = Path.GetFileName(file); - info.AbsolutePath = file; - - - var ini_name = file + ".meta"; - if (ini_name.FileExists()) - { - info.IniData = ini_name.LoadIniFile(); - info.Meta = File.ReadAllText(ini_name); - } - - return info; + info.IniData = ini_name.LoadIniFile(); + info.Meta = File.ReadAllText(ini_name); } - IndexArchive(file).ToJSON(metaname); - goto TOP; + return info; } private bool IsArchiveFile(string name) @@ -192,89 +135,39 @@ namespace Wabbajack return false; } - private IndexedArchiveCache IndexArchive(string file) - { - Status("Indexing {0}", Path.GetFileName(file)); - var streams = new Dictionary(); - var inner_archives = new Dictionary(); - FileExtractor.Extract(file, entry => - { - Stream inner; - if (IsArchiveFile(entry.Name)) - { - var name = Path.GetTempFileName() + Path.GetExtension(entry.Name); - inner_archives.Add(entry.Name, name); - inner = File.OpenWrite(name); - } - else - { - inner = Stream.Null; - } - var sha = new SHA256Managed(); - var os = new CryptoStream(inner, sha, CryptoStreamMode.Write); - streams.Add(entry.Name, (sha, (long)entry.Size)); - return os; - }); - - var indexed = new IndexedArchiveCache(); - indexed.Version = ARCHIVE_CONTENTS_VERSION; - indexed.Hash = file.FileSHA256(); - indexed.Entries = streams.Select(entry => - { - return new IndexedEntry() - { - Hash = entry.Value.Item1.Hash.ToBase64(), - Size = (long)entry.Value.Item2, - Path = entry.Key - }; - }).ToList(); - - streams.Do(e => e.Value.Item1.Dispose()); - - if (inner_archives.Count > 0) - { - var result = inner_archives.Select(archive => - { - return (archive.Key, IndexArchive(archive.Value)); - }).ToDictionary(e => e.Key, e => e.Item2); - indexed.InnerArchives = result; - - inner_archives.Do(e => File.Delete(e.Value)); - } - - return indexed; - } - public void Compile() { + VFS.AddRoot(MO2Folder); + VFS.AddRoot(GamePath); + var mo2_files = Directory.EnumerateFiles(MO2Folder, "*", SearchOption.AllDirectories) .Where(p => p.FileExists()) - .Select(p => new RawSourceFile() { Path = p.RelativeTo(MO2Folder), AbsolutePath = p }); + .Select(p => new RawSourceFile(VFS.Lookup(p))); var game_files = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories) .Where(p => p.FileExists()) - .Select(p => new RawSourceFile() { Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)), AbsolutePath = p }); + .Select(p => new RawSourceFile(VFS.Lookup(p)) { Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath))}); + + IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder) + .Where(f => Consts.SupportedArchives.Contains(Path.GetExtension(f))) + .Where(f => File.Exists(f + ".meta")) + .Select(f => new IndexedArchive() + { + File = VFS.Lookup(f), + IniData = (f + ".meta").LoadIniFile(), + Meta = File.ReadAllText(f + ".meta") + }) + .ToList(); + + IndexedFiles = IndexedArchives.SelectMany(f => VFS.FilesInArchive(f.File)) + .OrderByDescending(f => f.TopLevelArchive.LastModified) + .GroupBy(f => f.Hash) + .ToDictionary(f => f.Key, f => f.AsEnumerable()); Info("Searching for mod files"); + AllFiles = mo2_files.Concat(game_files).ToList(); - Info("Hashing files"); - - HashCache cache; - - using (cache = new HashCache()) - { - AllFiles.PMap(f => { - Status($"Hashing {f.Path}"); - try - { - f.LoadHashFromCache(cache); - } - catch (IOException ex) { } - return f; - }); - } - Info("Found {0} files to build into mod list", AllFiles.Count); ExtraFiles = new ConcurrentBag(); @@ -291,15 +184,11 @@ namespace Wabbajack .Where(f => f.Item2 != null) .ToDictionary(f => f.Item1, f => f.Item2); - cache = new HashCache(); - - var stack = MakeStack(cache); + var stack = MakeStack(); Info("Running Compilation Stack"); var results = AllFiles.PMap(f => RunStack(stack, f)).ToList(); - cache.Dispose(); - // Add the extra files that were generated by the stack Info($"Adding {ExtraFiles.Count} that were generated by the stack"); results = results.Concat(ExtraFiles).ToList(); @@ -356,7 +245,6 @@ namespace Wabbajack private void ResetMembers() { AllFiles = null; - IndexedArchives = null; InstallDirectives = null; SelectedArchives = null; ExtraFiles = null; @@ -385,42 +273,25 @@ namespace Wabbajack private void BuildArchivePatches(string archive_sha, IEnumerable group, Dictionary absolute_paths) { - var archive = IndexedArchives.First(a => a.Hash == archive_sha); - var paths = group.Select(g => g.FullPath) - .ToHashSet(); - var streams = new Dictionary(); - Status($"Extracting {paths.Count} patch files from {archive.Name}"); - // First we fetch the source files from the input archive - - FileExtractor.DeepExtract(archive.AbsolutePath, group.DistinctBy(f => f.FullPath), (fe, entry) => + var archive = VFS.HashIndex[archive_sha]; + using (var files = VFS.StageWith(group.Select(g => VFS.FileForArchiveHashPath(g.ArchiveHashPath)))) { - if (!paths.Contains(fe.FullPath)) return null; - if (streams.ContainsKey(fe.FullPath)) return null; - - var result = new MemoryStream(); - streams.Add(fe.FullPath, result); - return result; - - }, false); - - var extracted = streams.ToDictionary(k => k.Key, v => v.Value.ToArray()); - // Now Create the patches - Status("Building Patches for {0}", archive.Name); - Info("Building Patches for {0}", archive.Name); - group.PMap(entry => - { - Info("Patching {0}", entry.To); - var ss = extracted[entry.FullPath]; - using (var origin = new MemoryStream(ss)) - using (var output = new MemoryStream()) + var by_path = files.GroupBy(f => string.Join("|", f.Paths.Skip(1))).ToDictionary(f => f.Key, f => f.First()); + // Now Create the patches + group.PMap(entry => { - var a = origin.ReadAll(); - var b = LoadDataForTo(entry.To, absolute_paths); - BSDiff.Create(a, b, output); - entry.Patch = output.ToArray().ToBase64(); - Info($"Patch size {entry.Patch.Length} for {entry.To}"); - } - }); + Info("Patching {0}", entry.To); + using (var origin = by_path[string.Join("|", entry.ArchiveHashPath.Skip(1))].OpenRead()) + using (var output = new MemoryStream()) + { + var a = origin.ReadAll(); + var b = LoadDataForTo(entry.To, absolute_paths); + BSDiff.Create(a, b, output); + entry.Patch = output.ToArray().ToBase64(); + Info($"Patch size {entry.Patch.Length} for {entry.To}"); + } + }); + } } @@ -448,17 +319,20 @@ namespace Wabbajack private void GatherArchives() { Info($"Building a list of archives based on the files required"); - var archives = IndexedArchives.GroupBy(a => a.Hash).ToDictionary(k => k.Key, k => k.First()); var shas = InstallDirectives.OfType() .Select(a => a.ArchiveHashPath[0]) .Distinct(); + var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified) + .GroupBy(f => f.File.Hash) + .ToDictionary(f => f.Key, f => f.First()); + SelectedArchives = shas.PMap(sha => ResolveArchive(sha, archives)); } - private Archive ResolveArchive(string sha, Dictionary archives) + private Archive ResolveArchive(string sha, IDictionary archives) { if (archives.TryGetValue(sha, out var found)) { @@ -566,7 +440,7 @@ namespace Wabbajack } result.Name = found.Name; - result.Hash = found.Hash; + result.Hash = found.File.Hash; result.Meta = found.Meta; return result; @@ -594,7 +468,7 @@ namespace Wabbajack /// result included into the pack /// /// - private IEnumerable> MakeStack(HashCache cache) + private IEnumerable> MakeStack() { Info("Generating compilation stack"); return new List>() @@ -617,7 +491,7 @@ namespace Wabbajack IgnoreRegex(Consts.GameFolderFilesDir + "\\\\.*\\.bsa"), IncludeModIniData(), DirectMatch(), - DeconstructBSAs(cache), // Deconstruct BSAs before building patches so we don't generate massive patch files + DeconstructBSAs(), // Deconstruct BSAs before building patches so we don't generate massive patch files IncludePatches(), IncludeDummyESPs(), IncludeTaggedFiles(), @@ -728,7 +602,7 @@ namespace Wabbajack /// all of the files. /// /// - private Func DeconstructBSAs(HashCache cache) + private Func DeconstructBSAs() { var include_directly = ModInis.Where(kv => { var general = kv.Value.General; @@ -771,17 +645,11 @@ namespace Wabbajack } } - var hashed = cache.HashBSA(source.AbsolutePath, s => Status(s)); - - var source_files = hashed.Select(e => new RawSourceFile() { - Hash = e.Item2, - Path = e.Item1, - AbsolutePath = e.Item1 - }); + var source_files = source.File.FileInArchive; var stack = default_include ? microstack_with_include : microstack; - var matches = source_files.PMap(e => RunStack(stack, e)); + var matches = source_files.PMap(e => RunStack(stack, new RawSourceFile(e))); var id = Guid.NewGuid().ToString(); @@ -823,28 +691,6 @@ namespace Wabbajack }; } - /// - /// Given a BSA on disk, index it and return a dictionary of SHA256 -> filename - /// - /// - /// - private List<(string, string)> HashBSA(HashCache cache, string absolutePath) - { - Status($"Hashing BSA: {absolutePath}"); - var results = new List<(string, string)>(); - using (var a = new BSAReader(absolutePath)) - { - a.Files.PMap(entry => - { - Status($"Hashing BSA: {absolutePath} - {entry.Path}"); - - var data = entry.GetData(); - results.Add((entry.Path, data.SHA256())); - }); - } - return results; - } - private Func IgnoreDisabledMods() { var disabled_mods = File.ReadAllLines(Path.Combine(MO2ProfileDir, "modlist.txt")) @@ -865,23 +711,20 @@ namespace Wabbajack private Func IncludePatches() { - var archive_shas = IndexedArchives.GroupBy(e => e.Hash) - .ToDictionary(e => e.Key); - var indexed = (from entry in IndexedFiles - select new { archive = archive_shas[entry.HashPath[0]].First(), - entry = entry }) - .GroupBy(e => Path.GetFileName(e.entry.Path).ToLower()) - .ToDictionary(e => e.Key); + var indexed = IndexedFiles.Values + .SelectMany(f => f) + .GroupBy(f => Path.GetFileName(f.Paths.Last())) + .ToDictionary(f => f.Key); return source => { if (indexed.TryGetValue(Path.GetFileName(source.Path.ToLower()), out var value)) { + // TODO: Improve this var found = value.First(); var e = source.EvolveTo(); - e.From = found.entry.Path; - e.ArchiveHashPath = found.entry.HashPath; + e.ArchiveHashPath = found.MakeRelativePaths(); e.To = source.Path; return e; } @@ -1006,39 +849,21 @@ namespace Wabbajack private Func DirectMatch() { - var archive_shas = IndexedArchives.GroupBy(e => e.Hash) - .ToDictionary(e => e.Key); - if (DirectMatchIndex == null) - { - - DirectMatchIndex = IndexedFiles.PMap(entry => { - var archive = archive_shas[entry.HashPath[0]].First(); - return new IndexedFileMatch - { - Archive = archive, - Entry = entry, - LastModified = new FileInfo(archive.AbsolutePath).LastAccessTimeUtc - }; - }) - .OrderByDescending(e => e.LastModified) - .GroupBy(e => e.Entry.Hash) - .ToDictionary(e => e.Key, e => e.AsEnumerable()); - } return source => { - if (DirectMatchIndex.TryGetValue(source.Hash, out var found)) + if (IndexedFiles.TryGetValue(source.Hash, out var found)) { var result = source.EvolveTo(); - var match = found.Where(f => Path.GetFileName(f.Entry.Path) == Path.GetFileName(source.Path)) + var match = found.Where(f => Path.GetFileName(f.Paths[0]) == Path.GetFileName(source.Path)) .FirstOrDefault(); if (match == null) match = found.FirstOrDefault(); - result.ArchiveHashPath = match.Entry.HashPath; - result.From = match.Entry.Path; + result.ArchiveHashPath = match.MakeRelativePaths(); + return result; } return null; diff --git a/Wabbajack.Common/Data.cs b/Wabbajack/Data.cs similarity index 81% rename from Wabbajack.Common/Data.cs rename to Wabbajack/Data.cs index 6c16310a..087d8bf5 100644 --- a/Wabbajack.Common/Data.cs +++ b/Wabbajack/Data.cs @@ -4,27 +4,36 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; +using VFS; -namespace Wabbajack.Common +namespace Wabbajack { - public class RawSourceFile + public class RawSourceFile { - public string AbsolutePath; + public RawSourceFile(VirtualFile file) + { + File = file; + } + + public string AbsolutePath + { + get + { + return File.StagedPath; + } + } + public string Path; - private string _hash; + public VirtualFile File { get; private set; } + public string Hash { get { - if (_hash != null) return _hash; - _hash = AbsolutePath.FileSHA256(); - return _hash; - } - set - { - _hash = value; + return File.Hash; } + } public T EvolveTo() where T : Directive, new() @@ -33,11 +42,6 @@ namespace Wabbajack.Common v.To = Path; return v; } - - public void LoadHashFromCache(HashCache cache) - { - _hash = cache.HashFile(AbsolutePath); - } } public class ModList @@ -98,10 +102,9 @@ namespace Wabbajack.Common /// MurMur3 hash of the archive this file comes from /// public string[] ArchiveHashPath; - /// - /// The relative path of the file in the archive - /// - public string From; + + [JsonIgnore] + public VirtualFile FromFile; private string _fullPath = null; [JsonIgnore] @@ -110,9 +113,7 @@ namespace Wabbajack.Common get { if (_fullPath == null) { - var path = ArchiveHashPath.ToList(); - path.Add(From); - _fullPath = String.Join("|", path); + _fullPath = String.Join("|", ArchiveHashPath); } return _fullPath; } @@ -216,26 +217,12 @@ namespace Wabbajack.Common { } - /// - /// The indexed contents of an archive - /// - public class IndexedArchiveCache - { - public string Hash; - public int Version; - public List Entries; - - [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] - public Dictionary InnerArchives; - } - - public class IndexedArchive : IndexedArchiveCache + public class IndexedArchive { public dynamic IniData; public string Name; public string Meta; - public string AbsolutePath; - public List HashPath; + public VirtualFile File { get; internal set; } } /// diff --git a/Wabbajack/Installer.cs b/Wabbajack/Installer.cs index 5378f02a..c0cb6f62 100644 --- a/Wabbajack/Installer.cs +++ b/Wabbajack/Installer.cs @@ -9,12 +9,21 @@ using System.Reflection; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; +using VFS; using Wabbajack.Common; namespace Wabbajack { public class Installer { + public VirtualFileSystem VFS + { + get + { + return VirtualFileSystem.VFS; + } + } + public Installer(ModList mod_list, string output_folder, Action log_fn) { Outputfolder = output_folder; @@ -88,6 +97,9 @@ namespace Wabbajack Error("Cannot continue, was unable to download one or more archives"); } } + + PrimeVFS(); + BuildFolderStructure(); InstallArchives(); InstallIncludedFiles(); @@ -96,6 +108,37 @@ namespace Wabbajack Info("Installation complete! You may exit the program."); } + + /// + /// We don't want to make the installer index all the archives, that's just a waste of time, so instead + /// we'll pass just enough information to VFS to let it know about the files we have. + /// + private void PrimeVFS() + { + HashedArchives.Do(a => VFS.AddKnown(new VirtualFile() + { + Paths = new string[] { a.Key }, + Hash = a.Value + })); + VFS.RefreshIndexes(); + + + ModList.Directives + .OfType() + .Do(f => + { + var updated_path = new string[f.ArchiveHashPath.Length + 1]; + f.ArchiveHashPath.CopyTo(updated_path, 0); + updated_path[0] = VFS.HashIndex[updated_path[0]].Where(e => e.IsConcrete).First().FullPath; + VFS.AddKnown(new VirtualFile() { Paths = updated_path }); + }); + + VFS.BackfillMissing(); + + + + } + private void BuildBSAs() { var bsas = ModList.Directives.OfType().ToList(); @@ -187,18 +230,18 @@ namespace Wabbajack var files = grouping.GroupBy(e => e.FullPath) .ToDictionary(e => e.Key); - FileExtractor.DeepExtract(absolutePath, files.Select(f => f.Value.First()), - (fe, entry) => + + var vfiles = files.Select(g => { - if (files.TryGetValue(fe.FullPath, out var directives)) - { - var directive = directives.First(); - var absolute = Path.Combine(Outputfolder, directive.To); - if (absolute.FileExists()) File.Delete(absolute); - return File.OpenWrite(absolute); - } - return null; - }); + var first_file = g.Value.First(); + var file = VFS.FileForArchiveHashPath(first_file.ArchiveHashPath); + file.StagedPath = first_file.To; + return file; + }).ToList(); + + VFS.Stage(vfiles); + + vfiles.Do(f => f.StagedPath = null); Status("Copying duplicated files for {0}", archive.Name); diff --git a/Wabbajack/MainWindow.xaml.cs b/Wabbajack/MainWindow.xaml.cs index d4ac27a6..566d91f0 100644 --- a/Wabbajack/MainWindow.xaml.cs +++ b/Wabbajack/MainWindow.xaml.cs @@ -48,6 +48,9 @@ namespace Wabbajack WorkQueue.Init((id, msg, progress) => context.SetProgress(id, msg, progress), (max, current) => context.SetQueueSize(max, current)); + Utils.SetLoggerFn(s => context.LogMsg(s)); + Utils.SetStatusFn((msg, progress) => WorkQueue.Report(msg, progress)); + if (DebugMode) @@ -60,7 +63,6 @@ namespace Wabbajack context.ModListName = compiler.MO2Profile; context.Mode = "Building"; - compiler.LoadArchives(); compiler.Compile(); var modlist = compiler.ModList.ToJSON(); diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 945b7318..00229220 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -139,6 +139,7 @@ MSBuild:Compile Designer + @@ -200,6 +201,10 @@ {8aa97f58-5044-4bba-b8d9-a74b6947a660} SevenZipExtractor + + {5128b489-bc28-4f66-9f0b-b4565af36cbc} + VirtualFileSystem + {b3f3fb6e-b9eb-4f49-9875-d78578bc7ae5} Wabbajack.Common From 14bb629b8acfc18697a52ee06b8859451b8787a3 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 20 Aug 2019 15:44:32 -0600 Subject: [PATCH 5/7] cut memory usange in half using string interning, most of the Basic Fixes pack compiles --- VirtualFileSystem/VirtualFileSystem.cs | 24 +++++++++++++++++------- Wabbajack/AppState.cs | 4 ++++ Wabbajack/Compiler.cs | 11 +++++++++-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/VirtualFileSystem/VirtualFileSystem.cs b/VirtualFileSystem/VirtualFileSystem.cs index ec4ee82a..f54f9931 100644 --- a/VirtualFileSystem/VirtualFileSystem.cs +++ b/VirtualFileSystem/VirtualFileSystem.cs @@ -68,12 +68,9 @@ namespace VFS public IList FilesInArchive(VirtualFile f) { var path = f.FullPath + "|"; - lock (this) - { - return _files.Values - .Where(v => v.FullPath.StartsWith(path)) - .ToList(); - } + return _files.Values + .Where(v => v.FullPath.StartsWith(path)) + .ToList(); } @@ -358,8 +355,21 @@ namespace VFS [JsonObject(MemberSerialization = MemberSerialization.OptIn)] public class VirtualFile { + public string[] _paths; [JsonProperty] - public string[] Paths { get; set; } + public string[] Paths + { + get + { + return _paths; + } + set + { + for (int idx = 0; idx < value.Length; idx += 1) + value[idx] = String.Intern(value[idx]); + _paths = value; + } + } [JsonProperty] public string Hash { get; set; } [JsonProperty] diff --git a/Wabbajack/AppState.cs b/Wabbajack/AppState.cs index 9ccd4769..347a8519 100644 --- a/Wabbajack/AppState.cs +++ b/Wabbajack/AppState.cs @@ -130,6 +130,7 @@ namespace Wabbajack public AppState(Dispatcher d, String mode) { + _startTime = DateTime.Now; ArchiveFile.SetupLibrary(); LogFile = Assembly.GetExecutingAssembly().Location + ".log"; @@ -186,6 +187,7 @@ namespace Wabbajack public void LogMsg(string msg) { + msg = $"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}"; dispatcher.Invoke(() => Log.Add(msg)); lock (dispatcher) { File.AppendAllText(LogFile, msg + "\r\n"); @@ -264,6 +266,8 @@ namespace Wabbajack } private ICommand _begin; + private DateTime _startTime; + public ICommand Begin { get diff --git a/Wabbajack/Compiler.cs b/Wabbajack/Compiler.cs index 78cf118d..2851d74a 100644 --- a/Wabbajack/Compiler.cs +++ b/Wabbajack/Compiler.cs @@ -137,17 +137,21 @@ namespace Wabbajack public void Compile() { + Info($"Indexing {MO2Folder}"); VFS.AddRoot(MO2Folder); + Info($"Indexing {GamePath}"); VFS.AddRoot(GamePath); var mo2_files = Directory.EnumerateFiles(MO2Folder, "*", SearchOption.AllDirectories) .Where(p => p.FileExists()) - .Select(p => new RawSourceFile(VFS.Lookup(p))); + .Select(p => new RawSourceFile(VFS.Lookup(p)) { Path = p.RelativeTo(MO2Folder)}); var game_files = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories) .Where(p => p.FileExists()) .Select(p => new RawSourceFile(VFS.Lookup(p)) { Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath))}); + + Info($"Indexing Archives"); IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder) .Where(f => Consts.SupportedArchives.Contains(Path.GetExtension(f))) .Where(f => File.Exists(f + ".meta")) @@ -159,7 +163,10 @@ namespace Wabbajack }) .ToList(); - IndexedFiles = IndexedArchives.SelectMany(f => VFS.FilesInArchive(f.File)) + Info($"Indexing Files"); + IndexedFiles = IndexedArchives.PMap(f => { Status($"Finding files in {Path.GetFileName(f.File.FullPath)}"); + return VFS.FilesInArchive(f.File); }) + .SelectMany(fs => fs) .OrderByDescending(f => f.TopLevelArchive.LastModified) .GroupBy(f => f.Hash) .ToDictionary(f => f.Key, f => f.AsEnumerable()); From e24d29e93df9359ce47c580c436ca62f59e3073c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 20 Aug 2019 16:03:18 -0600 Subject: [PATCH 6/7] Boom! Basic Fixes.exe compiles with the new VFS compiler --- VirtualFileSystem/VirtualFileSystem.cs | 6 ++++-- Wabbajack/Compiler.cs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/VirtualFileSystem/VirtualFileSystem.cs b/VirtualFileSystem/VirtualFileSystem.cs index f54f9931..8a3532a4 100644 --- a/VirtualFileSystem/VirtualFileSystem.cs +++ b/VirtualFileSystem/VirtualFileSystem.cs @@ -274,7 +274,9 @@ namespace VFS public StagingGroup StageWith(IEnumerable files) { - return new StagingGroup(files); + var grp = new StagingGroup(files); + grp.Stage(); + return grp; } public VirtualFile this[string path] @@ -330,7 +332,7 @@ namespace VFS public VirtualFile FileForArchiveHashPath(string[] archiveHashPath) { var archive = HashIndex[archiveHashPath[0]].Where(a => a.IsArchive).OrderByDescending(a => a.LastModified).First(); - string fullPath = HashIndex[archiveHashPath[0]] + "|" + String.Join("|", archiveHashPath.Skip(1)); + string fullPath = archive.FullPath + "|" + String.Join("|", archiveHashPath.Skip(1)); return Lookup(fullPath); } } diff --git a/Wabbajack/Compiler.cs b/Wabbajack/Compiler.cs index 2851d74a..64ecc962 100644 --- a/Wabbajack/Compiler.cs +++ b/Wabbajack/Compiler.cs @@ -720,7 +720,7 @@ namespace Wabbajack { var indexed = IndexedFiles.Values .SelectMany(f => f) - .GroupBy(f => Path.GetFileName(f.Paths.Last())) + .GroupBy(f => Path.GetFileName(f.Paths.Last()).ToLower()) .ToDictionary(f => f.Key); return source => From b91e7348b7f2d6637261efe5bd5dc8976e19525b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 20 Aug 2019 16:37:55 -0600 Subject: [PATCH 7/7] Can install Basic Fixes.exe with the new VFS backend --- VirtualFileSystem/VirtualFileSystem.cs | 16 +++++++++++++++- Wabbajack/Installer.cs | 13 ++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/VirtualFileSystem/VirtualFileSystem.cs b/VirtualFileSystem/VirtualFileSystem.cs index 8a3532a4..477bdadf 100644 --- a/VirtualFileSystem/VirtualFileSystem.cs +++ b/VirtualFileSystem/VirtualFileSystem.cs @@ -140,6 +140,7 @@ namespace VFS { _files.Values .Select(f => f.ParentPath) + .Where(s => s != null) .Where(s => !_files.ContainsKey(s)) .ToHashSet() .Do(s => @@ -252,7 +253,10 @@ namespace VFS public void Stage(IEnumerable files) { - var grouped = files.GroupBy(f => f.ParentArchive) + var grouped = files.SelectMany(f => f.FilesInPath) + .Distinct() + .Where(f => f.ParentArchive != null) + .GroupBy(f => f.ParentArchive) .OrderBy(f => f.Key == null ? 0 : f.Key.Paths.Length) .ToList(); @@ -565,6 +569,16 @@ namespace VFS path_copy[0] = VirtualFileSystem.VFS.Lookup(Paths[0]).Hash; return path_copy; } + + public IEnumerable FilesInPath + { + get { + return Enumerable.Range(1, Paths.Length) + .Select(i => Paths.Take(i)) + .Select(path => VirtualFileSystem.VFS.Lookup(String.Join("|", path))); + + } + } } diff --git a/Wabbajack/Installer.cs b/Wabbajack/Installer.cs index c0cb6f62..73952e8a 100644 --- a/Wabbajack/Installer.cs +++ b/Wabbajack/Installer.cs @@ -117,8 +117,8 @@ namespace Wabbajack { HashedArchives.Do(a => VFS.AddKnown(new VirtualFile() { - Paths = new string[] { a.Key }, - Hash = a.Value + Paths = new string[] { a.Value }, + Hash = a.Key })); VFS.RefreshIndexes(); @@ -127,7 +127,7 @@ namespace Wabbajack .OfType() .Do(f => { - var updated_path = new string[f.ArchiveHashPath.Length + 1]; + var updated_path = new string[f.ArchiveHashPath.Length]; f.ArchiveHashPath.CopyTo(updated_path, 0); updated_path[0] = VFS.HashIndex[updated_path[0]].Where(e => e.IsConcrete).First().FullPath; VFS.AddKnown(new VirtualFile() { Paths = updated_path }); @@ -174,8 +174,11 @@ namespace Wabbajack } }); - Info($"Removing temp folder {Consts.BSACreationDir}"); - Directory.Delete(Path.Combine(Outputfolder, Consts.BSACreationDir), true); + if (Directory.Exists(Consts.BSACreationDir)) + { + Info($"Removing temp folder {Consts.BSACreationDir}"); + Directory.Delete(Path.Combine(Outputfolder, Consts.BSACreationDir), true); + } }