Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

started work on VFS should simplify and greatly speed up really complex modlists

Browse Source
This commit is contained in:
Timothy Baldridge 2019-08-14 22:30:37 -06:00
parent 672cf49f47
commit 1a7836ec2a
16 changed files with 958 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
VirtualFileSystem.Test/App.config Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2" />
</startup>
</configuration>

20
VirtualFileSystem.Test/Program.cs Normal file
View File

@ -0,0 +1,20 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace VirtualFileSystem.Test
{
class Program
{
static void Main(string[] args)
{
WorkQueue.Init((a, b, c) => { return; },
(a, b) => { return; });
var vfs = new VirtualFileSystem();
vfs.AddRoot(@"D:\MO2 Instances\Mod Organizer 2", s => Console.WriteLine(s));
}
}
}

36
VirtualFileSystem.Test/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("VirtualFileSystem.Test")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("VirtualFileSystem.Test")]
[assembly: AssemblyCopyright("Copyright © 2019")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("a2913dfe-18ff-468b-a6c1-55f7c0cc0ce8")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

63
VirtualFileSystem.Test/VirtualFileSystem.Test.csproj Normal file
View File

@ -0,0 +1,63 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}</ProjectGuid>
<OutputType>Exe</OutputType>
<RootNamespace>VirtualFileSystem.Test</RootNamespace>
<AssemblyName>VirtualFileSystem.Test</AssemblyName>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<Deterministic>true</Deterministic>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>x64</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VirtualFileSystem\VirtualFileSystem.csproj">
<Project>{5128b489-bc28-4f66-9f0b-b4565af36cbc}</Project>
<Name>VirtualFileSystem</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
<Project>{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}</Project>
<Name>Wabbajack.Common</Name>
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>

36
VirtualFileSystem/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("VirtualFileSystem")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("VirtualFileSystem")]
[assembly: AssemblyCopyright("Copyright © 2019")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("5128b489-bc28-4f66-9f0b-b4565af36cbc")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

372
VirtualFileSystem/VirtualFileSystem.cs Normal file
View File

@ -0,0 +1,372 @@
using Compression.BSA;
using ICSharpCode.SharpZipLib.Zip;
using Newtonsoft.Json;
using SevenZipExtractor;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace VirtualFileSystem
{
public class VirtualFileSystem
{
private Dictionary<string, VirtualFile> _files = new Dictionary<string, VirtualFile>();
internal string _stagedRoot;
public VirtualFileSystem()
{
_stagedRoot = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "vfs_staged_files");
Directory.CreateDirectory(_stagedRoot);
}
/// <summary>
/// Adds the root path to the filesystem. This may take quite some time as every file in the folder will be hashed,
/// and every archive examined.
/// </summary>
/// <param name="path"></param>
public void AddRoot(string path, Action<string> status)
{
IndexPath(path, status);
}
private void SyncToDisk()
{
lock (this)
{
_files.Values.ToList().ToJSON("vfs_cache.json");
}
}
private void IndexPath(string path, Action<string> status)
{
Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories)
.PMap(f => UpdateFile(f));
}
private void UpdateFile(string f)
{
TOP:
Console.WriteLine(f);
var lv = Lookup(f);
if (lv == null)
{
lv = new VirtualFile(this)
{
Paths = new string[] { f }
};
this[f] = lv;
lv.Analyze();
if (lv.IsArchive)
{
UpdateArchive(lv);
}
}
if (lv.IsOutdated)
{
Purge(lv);
goto TOP;
}
}
private void UpdateArchive(VirtualFile f)
{
var entries = GetArchiveEntryNames(f);
var new_files = entries.Select(e => {
var new_path = new string[f.Paths.Length + 1];
f.Paths.CopyTo(new_path, 0);
new_path[f.Paths.Length] = e;
var nf = new VirtualFile(this)
{
Paths = new_path,
};
this[nf.FullPath] = nf;
return nf;
}).ToList();
// Stage the files in the archive
Stage(new_files);
// Analyze them
new_files.Do(file => file.Analyze());
// Recurse into any archives in this archive
new_files.Where(file => file.IsArchive).Do(file => UpdateArchive(f));
// Unstage the file
new_files.Where(file => file.IsStaged).Do(file => file.Unstage());
SyncToDisk();
}
private void Stage(IEnumerable<VirtualFile> files)
{
var grouped = files.GroupBy(f => f.ParentArchive)
.OrderBy(f => f.Key == null ? 0 : f.Key.Paths.Length)
.ToList();
foreach (var group in grouped)
{
var indexed = group.ToDictionary(e => e.Paths[group.Key.Paths.Length]);
FileExtractor.Extract(group.Key.StagedPath, e =>
{
if (indexed.TryGetValue(e.Name, out var file))
{
return File.OpenWrite(file.GenerateStagedName());
}
return null;
});
}
}
internal VirtualFile Lookup(string path)
{
lock(this)
{
if (_files.TryGetValue(path, out VirtualFile value))
return value;
return null;
}
}
public VirtualFile this[string path]
{
get
{
return Lookup(path);
}
set
{
lock(this)
{
_files[path] = value;
}
}
}
internal List<string> GetArchiveEntryNames(VirtualFile file)
{
if (!file.IsStaged)
throw new InvalidDataException("File is not staged");
if (file.Extension == ".bsa") {
using (var ar = new BSAReader(file.StagedPath))
{
return ar.Files.Select(f => f.Path).ToList();
}
}
if (file.Extension == ".zip")
{
using (var s = new ZipFile(File.OpenRead(file.StagedPath)))
{
s.IsStreamOwner = true;
s.UseZip64 = UseZip64.On;
if (s.OfType<ZipEntry>().FirstOrDefault(e => !e.CanDecompress) == null)
{
return s.OfType<ZipEntry>()
.Where(f => f.IsFile)
.Select(f => f.Name.Replace('/', '\\'))
.ToList();
}
}
}
using (var e = new ArchiveFile(file.StagedPath))
{
return e.Entries
.Where(f => !f.IsFolder)
.Select(f => f.FileName).ToList();
}
}
/// <summary>
/// Remove all cached data for this file and if it is a top level archive, any sub-files.
/// </summary>
/// <param name="file"></param>
internal void Purge(VirtualFile file)
{
lock(this)
{
// Remove the file
_files.Remove(file.FullPath);
// If required, remove sub-files
if (file.IsArchive)
{
string prefix = file.FullPath + "|";
_files.Where(f => f.Key.StartsWith(prefix)).ToList().Do(f => _files.Remove(f.Key));
}
}
}
}
[JsonObject(MemberSerialization.OptIn)]
public class VirtualFile
{
[JsonProperty]
public string[] Paths;
[JsonProperty]
public string Hash;
[JsonProperty]
public long Size;
[JsonProperty]
public DateTime LastModifiedUTC;
private string _fullPath;
private VirtualFileSystem _vfs;
public VirtualFile(VirtualFileSystem vfs)
{
_vfs = vfs;
}
[JsonIgnore]
private string _stagedPath;
public string FullPath
{
get
{
if (_fullPath != null) return _fullPath;
_fullPath = String.Join("|", Paths);
return _fullPath;
}
}
public string Extension
{
get
{
return Path.GetExtension(Paths.Last());
}
}
/// <summary>
/// If this file is in an archive, return the Archive File, otherwise return null.
/// </summary>
public VirtualFile TopLevelArchive
{
get
{
if (Paths.Length == 0) return null;
return _vfs[Paths[0]];
}
}
public VirtualFile ParentArchive
{
get
{
if (Paths.Length == 0) return null;
return _vfs[String.Join("|", Paths.Take(Paths.Length - 1))];
}
}
private bool? _isArchive;
public bool IsArchive
{
get
{
if (_isArchive == null)
_isArchive = FileExtractor.CanExtract(Extension);
return (bool)_isArchive;
}
}
public bool IsStaged
{
get
{
if (IsConcrete) return true;
return _stagedPath != null;
}
}
public string StagedPath
{
get
{
if (!IsStaged)
throw new InvalidDataException("File is not staged");
if (IsConcrete) return Paths[0];
return _stagedPath;
}
}
public FileStream OpenRead()
{
if (!IsStaged)
throw new InvalidDataException("File is not staged, cannot open");
return File.OpenRead(_stagedPath);
}
/// <summary>
/// Calulate the file's SHA, size and last modified
/// </summary>
internal void Analyze()
{
if (!IsStaged)
throw new InvalidDataException("Cannot analzye a unstaged file");
var fio = new FileInfo(StagedPath);
Size = fio.Length;
Hash = Utils.FileSHA256(StagedPath);
LastModifiedUTC = fio.LastWriteTimeUtc;
}
/// <summary>
/// Delete the temoporary file associated with this file
/// </summary>
internal void Unstage()
{
if (IsStaged && !IsConcrete)
{
File.Delete(_stagedPath);
_stagedPath = null;
}
}
internal string GenerateStagedName()
{
_stagedPath = Path.Combine(_vfs._stagedRoot, Guid.NewGuid().ToString() + Path.GetExtension(Paths.Last()));
return _stagedPath;
}
/// <summary>
/// Returns true if this file always exists on-disk, and doesn't need to be staged.
/// </summary>
public bool IsConcrete
{
get
{
return Paths.Length == 1;
}
}
public bool IsOutdated
{
get
{
if (IsStaged)
{
var fi = new FileInfo(StagedPath);
if (fi.LastWriteTimeUtc != LastModifiedUTC || fi.Length != Size)
return true;
}
return false;
}
}
}
}

76
VirtualFileSystem/VirtualFileSystem.csproj Normal file
View File

@ -0,0 +1,76 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{5128B489-BC28-4F66-9F0B-B4565AF36CBC}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>VirtualFileSystem</RootNamespace>
<AssemblyName>VirtualFileSystem</AssemblyName>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<Deterministic>true</Deterministic>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<PlatformTarget>x64</PlatformTarget>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="ICSharpCode.SharpZipLib, Version=1.2.0.246, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
<HintPath>..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll</HintPath>
</Reference>
<Reference Include="LiteDB, Version=4.1.4.0, Culture=neutral, PublicKeyToken=4ee40123013c9f27, processorArchitecture=MSIL">
<HintPath>..\packages\LiteDB.4.1.4\lib\net40\LiteDB.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.IO.Compression.FileSystem" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="VirtualFileSystem.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Compression.BSA\Compression.BSA.csproj">
<Project>{ff5d892f-8ff4-44fc-8f7f-cd58f307ad1b}</Project>
<Name>Compression.BSA</Name>
</ProjectReference>
<ProjectReference Include="..\SevenZipExtractor\SevenZipExtractor.csproj">
<Project>{8aa97f58-5044-4bba-b8d9-a74b6947a660}</Project>
<Name>SevenZipExtractor</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
<Project>{b3f3fb6e-b9eb-4f49-9875-d78578bc7ae5}</Project>
<Name>Wabbajack.Common</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>

6
VirtualFileSystem/packages.config Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="LiteDB" version="4.1.4" targetFramework="net472" />
<package id="Newtonsoft.Json" version="12.0.2" targetFramework="net472" />
<package id="SharpZipLib" version="1.2.0" targetFramework="net472" />
</packages>

1
Wabbajack.Common/Consts.cs
View File

@ -30,5 +30,6 @@ namespace Wabbajack.Common
public static string WABBAJACK_INCLUDE = "WABBAJACK_INCLUDE";
public static String AppName = "Wabbajack";
public static string HashCacheName = "Wabbajack.hash_cache";
}
}

13
Wabbajack.Common/Data.cs
View File

@ -33,6 +33,11 @@ namespace Wabbajack.Common
v.To = Path;
return v;
}
public void LoadHashFromCache(HashCache cache)
{
_hash = cache.HashFile(AbsolutePath);
}
}
public class ModList
@ -174,6 +179,14 @@ namespace Wabbajack.Common
public List<string> Headers;
}
/// <summary>
/// A URL that cannot be downloaded automatically and has to be downloaded by hand
/// </summary>
public class ManualURLArchive : Archive
{
public string URL;
}
/// <summary>
/// An archive that requires additional HTTP headers.
/// </summary>

24
Wabbajack.Common/FileExtractor.cs
View File

@ -108,11 +108,22 @@ namespace Wabbajack.Common
}
}
/// <summary>
/// Returns true if the given extension type can be extracted
/// </summary>
/// <param name="v"></param>
/// <returns></returns>
public static bool CanExtract(string v)
{
return Consts.SupportedArchives.Contains(v) || v == ".bsa";
}
public static void DeepExtract(string file, IEnumerable<FromArchive> files, Func<FromArchive, Entry, Stream> fnc, bool leave_open = false, int depth = 1)
{
// Files we need to extract at this level
var files_for_level = files.Where(f => f.ArchiveHashPath.Length == depth).ToDictionary(e => e.From);
var files_for_level = files.Where(f => f.ArchiveHashPath.Length == depth)
.GroupBy(e => e.From)
.ToDictionary(e => e.Key);
// Archives we need to extract at this level
var archives_for_level = files.Where(f => f.ArchiveHashPath.Length > depth)
.GroupBy(f => f.ArchiveHashPath[depth])
@ -127,12 +138,21 @@ namespace Wabbajack.Common
if (files_for_level.TryGetValue(e.Name, out var fe))
{
a = fnc(fe, e);
foreach (var inner_fe in fe)
{
var str = fnc(inner_fe, e);
if (str == null) continue;
a = new SplittingStream(a, false, fnc(inner_fe, e), leave_open);
}
}
if (archives_for_level.TryGetValue(e.Name, out var archive))
{
var name = Path.GetTempFileName() + Path.GetExtension(e.Name);
if (disk_archives.ContainsKey(e.Name))
{
}
disk_archives.Add(e.Name, name);
b = File.OpenWrite(name);
}

127
Wabbajack.Common/HashCache.cs Normal file
View File

@ -0,0 +1,127 @@
using Compression.BSA;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Common
{
public class HashCache : IDisposable
{
public class Entry
{
public string name;
public string hash;
public long size;
public DateTime last_modified;
}
public class BSA
{
public string full_path;
public string hash;
public long size;
public DateTime last_modified;
public Dictionary<string, string> entries;
}
private ConcurrentDictionary<string, Entry> _hashes = new ConcurrentDictionary<string, Entry>();
private ConcurrentDictionary<string, BSA> _bsas = new ConcurrentDictionary<string, BSA>();
private bool disposed;
public class DB
{
public List<Entry> entries;
public List<BSA> bsas;
}
public HashCache()
{
if (Consts.HashCacheName.FileExists())
{
var json = Consts.HashCacheName.FromJSON<DB>();
_hashes = new ConcurrentDictionary<string, Entry>(json.entries.Select(e => new KeyValuePair<string, Entry>(e.name, e)));
_bsas = new ConcurrentDictionary<string, BSA>(json.bsas.Select(e => new KeyValuePair<string, BSA>(e.full_path, e)));
}
}
public string HashFile(string filename)
{
TOP:
var result = _hashes.GetOrAdd(filename,
s =>
{
var fi = new FileInfo(filename);
return new Entry
{
name = filename,
hash = Utils.FileSHA256(filename),
size = fi.Length,
last_modified = fi.LastWriteTimeUtc
};
});
var info = new FileInfo(filename);
if (info.LastWriteTimeUtc != result.last_modified || info.Length != result.size)
{
_hashes.TryRemove(filename, out Entry v);
goto TOP;
}
return result.hash;
}
public void Dispose()
{
if (disposed) return;
new DB
{
entries = _hashes.Values.ToList(),
bsas = _bsas.Values.ToList()
}.ToJSON(Consts.HashCacheName);
disposed = true;
_hashes = null;
_bsas = null;
}
public List<(string, string)> HashBSA(string absolutePath, Action<string> status)
{
TOP:
var finfo = new FileInfo(absolutePath);
if (_bsas.TryGetValue(absolutePath, out BSA ar))
{
if (ar.last_modified == finfo.LastWriteTimeUtc && ar.size == finfo.Length)
return ar.entries.Select(kv => (kv.Key, kv.Value)).ToList();
_bsas.TryRemove(absolutePath, out BSA value);
}
var bsa = new BSA()
{
full_path = absolutePath,
size = finfo.Length,
last_modified = finfo.LastAccessTimeUtc,
};
var entries = new ConcurrentBag<(string, string)>();
status($"Hashing BSA: {absolutePath}");
using (var a = new BSAReader(absolutePath))
{
a.Files.PMap(entry =>
{
status($"Hashing BSA: {absolutePath} - {entry.Path}");
var data = entry.GetData();
entries.Add((entry.Path, data.SHA256()));
});
}
bsa.entries = entries.ToDictionary(e => e.Item1, e => e.Item2);
_bsas.TryAdd(absolutePath, bsa);
goto TOP;
}
}
}

18
Wabbajack.Common/Utils.cs
View File

@ -269,6 +269,24 @@ namespace Wabbajack.Common
File.WriteAllText($"{DateTime.Now.ToString("yyyyMMddTHHmmss_crash_log.txt")}", ExceptionToString(e));
}
public static IEnumerable<T> DistinctBy<T, V>(this IEnumerable<T> vs, Func<T, V> select)
{
HashSet<V> set = new HashSet<V>();
foreach (var v in vs) {
var key = select(v);
if (set.Contains(key)) continue;
yield return v;
}
}
public static T Last<T>(this T[] a)
{
if (a == null || a.Length == 0)
throw new InvalidDataException("null or empty array");
return a[a.Length - 1];
}
public static V GetOrDefault<K, V>(this IDictionary<K, V> dict, K key)
{
if (dict.TryGetValue(key, out V v)) return v;

1
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -78,6 +78,7 @@
<Compile Include="Data.cs" />
<Compile Include="DynamicIniData.cs" />
<Compile Include="FileExtractor.cs" />
<Compile Include="HashCache.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="SplittingStream.cs" />
<Compile Include="Utils.cs" />

28
Wabbajack.sln
View File

@ -20,6 +20,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
RECIPES.md = RECIPES.md
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VirtualFileSystem", "VirtualFileSystem\VirtualFileSystem.csproj", "{5128B489-BC28-4F66-9F0B-B4565AF36CBC}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VirtualFileSystem.Test", "VirtualFileSystem.Test\VirtualFileSystem.Test.csproj", "{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug (no commandargs)|Any CPU = Debug (no commandargs)|Any CPU
@ -90,6 +94,30 @@ Global
{BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Release|Any CPU.Build.0 = Release|Any CPU
{BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Release|x64.ActiveCfg = Release|x64
{BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Release|x64.Build.0 = Release|x64
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|x64.ActiveCfg = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|x64.Build.0 = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|Any CPU.Build.0 = Release|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|x64.ActiveCfg = Release|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|x64.Build.0 = Release|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|x64.ActiveCfg = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|x64.Build.0 = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|Any CPU.Build.0 = Release|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|x64.ActiveCfg = Release|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

161
Wabbajack/Compiler.cs
View File

@ -63,6 +63,15 @@ namespace Wabbajack
public List<IndexedArchiveEntry> IndexedFiles { get; private set; }
public class IndexedFileMatch
{
public IndexedArchive Archive;
public IndexedArchiveEntry Entry;
public DateTime LastModified;
}
public Dictionary<string, IEnumerable<IndexedFileMatch>> DirectMatchIndex;
public void Info(string msg, params object[] args)
{
if (args.Length > 0)
@ -249,6 +258,23 @@ namespace Wabbajack
Info("Searching for mod files");
AllFiles = mo2_files.Concat(game_files).ToList();
Info("Hashing files");
HashCache cache;
using (cache = new HashCache())
{
AllFiles.PMap(f => {
Status($"Hashing {f.Path}");
try
{
f.LoadHashFromCache(cache);
}
catch (IOException ex) { }
return f;
});
}
Info("Found {0} files to build into mod list", AllFiles.Count);
ExtraFiles = new ConcurrentBag<Directive>();
@ -265,11 +291,15 @@ namespace Wabbajack
.Where(f => f.Item2 != null)
.ToDictionary(f => f.Item1, f => f.Item2);
var stack = MakeStack();
cache = new HashCache();
var stack = MakeStack(cache);
Info("Running Compilation Stack");
var results = AllFiles.PMap(f => RunStack(stack, f)).ToList();
cache.Dispose();
// Add the extra files that were generated by the stack
Info($"Adding {ExtraFiles.Count} that were generated by the stack");
results = results.Concat(ExtraFiles).ToList();
@ -356,14 +386,16 @@ namespace Wabbajack
private void BuildArchivePatches(string archive_sha, IEnumerable<PatchedFromArchive> group, Dictionary<string, string> absolute_paths)
{
var archive = IndexedArchives.First(a => a.Hash == archive_sha);
var paths = group.Select(g => g.FullPath).ToHashSet();
var paths = group.Select(g => g.FullPath)
.ToHashSet();
var streams = new Dictionary<string, MemoryStream>();
Status($"Extracting {paths.Count} patch files from {archive.Name}");
// First we fetch the source files from the input archive
FileExtractor.DeepExtract(archive.AbsolutePath, group, (fe, entry) =>
FileExtractor.DeepExtract(archive.AbsolutePath, group.DistinctBy(f => f.FullPath), (fe, entry) =>
{
if (!paths.Contains(fe.FullPath)) return null;
if (streams.ContainsKey(fe.FullPath)) return null;
var result = new MemoryStream();
streams.Add(fe.FullPath, result);
@ -520,6 +552,13 @@ namespace Wabbajack
}
result = tmp;
}
else if (general.manualURL != null)
{
result = new ManualURLArchive()
{
URL = general.manualURL.ToString()
};
}
else
{
Error("No way to handle archive {0} but it's required by the modpack", found.Name);
@ -540,10 +579,12 @@ namespace Wabbajack
private Directive RunStack(IEnumerable<Func<RawSourceFile, Directive>> stack, RawSourceFile source)
{
Status("Compiling {0}", source.Path);
return (from f in stack
let result = f(source)
where result != null
select result).First();
foreach (var f in stack)
{
var result = f(source);
if (result != null) return result;
}
throw new InvalidDataException("Data fell out of the compilation stack");
}
@ -553,7 +594,7 @@ namespace Wabbajack
/// result included into the pack
/// </summary>
/// <returns></returns>
private IEnumerable<Func<RawSourceFile, Directive>> MakeStack()
private IEnumerable<Func<RawSourceFile, Directive>> MakeStack(HashCache cache)
{
Info("Generating compilation stack");
return new List<Func<RawSourceFile, Directive>>()
@ -562,6 +603,7 @@ namespace Wabbajack
IgnoreStartsWith("downloads\\"),
IgnoreStartsWith("webcache\\"),
IgnoreStartsWith("overwrite\\"),
IgnorePathContains("temporary_logs"),
IgnoreEndsWith(".pyc"),
IgnoreEndsWith(".log"),
IgnoreOtherProfiles(),
@ -575,10 +617,10 @@ namespace Wabbajack
IgnoreRegex(Consts.GameFolderFilesDir + "\\\\.*\\.bsa"),
IncludeModIniData(),
DirectMatch(),
IncludeTaggedFiles(),
DeconstructBSAs(), // Deconstruct BSAs before building patches so we don't generate massive patch files
DeconstructBSAs(cache), // Deconstruct BSAs before building patches so we don't generate massive patch files
IncludePatches(),
IncludeDummyESPs(),
IncludeTaggedFiles(),
// If we have no match at this point for a game folder file, skip them, we can't do anything about them
@ -597,6 +639,22 @@ namespace Wabbajack
};
}
private Func<RawSourceFile, Directive> IgnorePathContains(string v)
{
v = $"\\{v.Trim('\\')}\\";
var reason = $"Ignored because path contains {v}";
return source =>
{
if (source.Path.Contains(v))
{
var result = source.EvolveTo<IgnoredDirectly>();
result.Reason = reason;
return result;
}
return null;
};
}
/// <summary>
/// If a user includes WABBAJACK_INCLUDE directly in the notes or comments of a mod, the contents of that
@ -670,8 +728,17 @@ namespace Wabbajack
/// all of the files.
/// </summary>
/// <returns></returns>
private Func<RawSourceFile, Directive> DeconstructBSAs()
private Func<RawSourceFile, Directive> DeconstructBSAs(HashCache cache)
{
var include_directly = ModInis.Where(kv => {
var general = kv.Value.General;
if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE))
return true;
if (general.comments != null && general.comments.Contains(Consts.WABBAJACK_INCLUDE))
return true;
return false;
}).Select(kv => $"mods\\{kv.Key}\\");
var microstack = new List<Func<RawSourceFile, Directive>>()
{
DirectMatch(),
@ -679,11 +746,32 @@ namespace Wabbajack
DropAll()
};
var microstack_with_include = new List<Func<RawSourceFile, Directive>>()
{
DirectMatch(),
IncludePatches(),
IncludeALL()
};
return source =>
{
if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path))) return null;
var hashed = HashBSA(source.AbsolutePath);
bool default_include = false;
if (source.Path.StartsWith("mods"))
{
foreach (var modpath in include_directly)
{
if (source.Path.StartsWith(modpath))
{
default_include = true;
break;
}
}
}
var hashed = cache.HashBSA(source.AbsolutePath, s => Status(s));
var source_files = hashed.Select(e => new RawSourceFile() {
Hash = e.Item2,
@ -691,8 +779,9 @@ namespace Wabbajack
AbsolutePath = e.Item1
});
var stack = default_include ? microstack_with_include : microstack;
var matches = source_files.Select(e => RunStack(microstack, e));
var matches = source_files.PMap(e => RunStack(stack, e));
var id = Guid.NewGuid().ToString();
@ -724,12 +813,22 @@ namespace Wabbajack
};
}
private Func<RawSourceFile, Directive> IncludeALL()
{
return source =>
{
var inline = source.EvolveTo<InlineFile>();
inline.SourceData = File.ReadAllBytes(source.AbsolutePath).ToBase64();
return inline;
};
}
/// <summary>
/// Given a BSA on disk, index it and return a dictionary of SHA256 -> filename
/// </summary>
/// <param name="absolutePath"></param>
/// <returns></returns>
private List<(string, string)> HashBSA(string absolutePath)
private List<(string, string)> HashBSA(HashCache cache, string absolutePath)
{
Status($"Hashing BSA: {absolutePath}");
var results = new List<(string, string)>();
@ -909,31 +1008,37 @@ namespace Wabbajack
{
var archive_shas = IndexedArchives.GroupBy(e => e.Hash)
.ToDictionary(e => e.Key);
if (DirectMatchIndex == null)
{
var indexed = (from entry in IndexedFiles
select new { archive = archive_shas[entry.HashPath[0]].First(),
entry = entry })
.GroupBy(e => e.entry.Hash)
.ToDictionary(e => e.Key);
DirectMatchIndex = IndexedFiles.PMap(entry => {
var archive = archive_shas[entry.HashPath[0]].First();
return new IndexedFileMatch
{
Archive = archive,
Entry = entry,
LastModified = new FileInfo(archive.AbsolutePath).LastAccessTimeUtc
};
})
.OrderByDescending(e => e.LastModified)
.GroupBy(e => e.Entry.Hash)
.ToDictionary(e => e.Key, e => e.AsEnumerable());
}
return source =>
{
if (indexed.TryGetValue(source.Hash, out var found))
if (DirectMatchIndex.TryGetValue(source.Hash, out var found))
{
var result = source.EvolveTo<FromArchive>();
var match = found.Where(f => Path.GetFileName(f.entry.Path) == Path.GetFileName(source.Path))
.OrderByDescending(f => new FileInfo(f.archive.AbsolutePath).LastWriteTime)
var match = found.Where(f => Path.GetFileName(f.Entry.Path) == Path.GetFileName(source.Path))
.FirstOrDefault();
if (match == null)
match = found.OrderByDescending(f => new FileInfo(f.archive.AbsolutePath).LastWriteTime)
.FirstOrDefault();
match = found.FirstOrDefault();
result.ArchiveHashPath = match.entry.HashPath;
result.From = match.entry.Path;
result.ArchiveHashPath = match.Entry.HashPath;
result.From = match.Entry.Path;
return result;
}
return null;