Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #12 from halgari/vfs-rework

Browse Source 
VFS rework
This commit is contained in:
Timothy Baldridge 2019-08-20 16:41:18 -06:00 committed by GitHub
commit e2817ef949
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 1252 additions and 300 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Compression.BSA/BSABuilder.cs
View File

@ -1,10 +1,10 @@
using lz4;
using K4os.Compression.LZ4;
using K4os.Compression.LZ4.Streams;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Compression.BSA
{
@ -327,13 +327,8 @@ namespace Compression.BSA
if (_bsa.HeaderType == VersionType.SSE)
{
var r = new MemoryStream();
using (var f = LZ4Stream.CreateCompressor(r, LZ4StreamMode.Write, LZ4FrameBlockMode.Independent, LZ4FrameBlockSize.Max4MB, LZ4FrameChecksumMode.Content,
highCompression: true, leaveInnerStreamOpen: true))
{
new MemoryStream(_rawData).CopyTo(f);
}
using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings() { CompressionLevel = LZ4Level.L10_OPT}))
(new MemoryStream(_rawData)).CopyTo(w);
_rawData = r.ToArray();

8
Compression.BSA/BSAReader.cs
View File

@ -3,7 +3,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using lz4;
using K4os.Compression.LZ4.Streams;
namespace Compression.BSA
{
@ -362,10 +362,8 @@ namespace Compression.BSA
file_size -= 4;
if (_bsa.HeaderType == VersionType.SSE)
{
using (var dc = LZ4Stream.CreateDecompressor(output, LZ4StreamMode.Write, true))
{
rdr.BaseStream.CopyToLimit(dc, file_size);
}
var r = LZ4Stream.Decode(rdr.BaseStream);
r.CopyTo(output);
}
else
{

22
Compression.BSA/Compression.BSA.csproj
View File

@ -50,12 +50,30 @@
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<ItemGroup>
<Reference Include="lz4.AnyCPU.loader, Version=1.0.12.0, Culture=neutral, PublicKeyToken=7aa3c636ef56b77f, processorArchitecture=MSIL">
<HintPath>..\packages\IonKiwi.lz4.net.1.0.12\lib\net472\lz4.AnyCPU.loader.dll</HintPath>
<Reference Include="K4os.Compression.LZ4, Version=1.1.11.0, Culture=neutral, PublicKeyToken=2186fa9121ef231d, processorArchitecture=MSIL">
<HintPath>..\packages\K4os.Compression.LZ4.1.1.11\lib\net46\K4os.Compression.LZ4.dll</HintPath>
</Reference>
<Reference Include="K4os.Compression.LZ4.Streams, Version=1.1.11.0, Culture=neutral, PublicKeyToken=2186fa9121ef231d, processorArchitecture=MSIL">
<HintPath>..\packages\K4os.Compression.LZ4.Streams.1.1.11\lib\net46\K4os.Compression.LZ4.Streams.dll</HintPath>
</Reference>
<Reference Include="K4os.Hash.xxHash, Version=1.0.6.0, Culture=neutral, PublicKeyToken=32cd54395057cec3, processorArchitecture=MSIL">
<HintPath>..\packages\K4os.Hash.xxHash.1.0.6\lib\net46\K4os.Hash.xxHash.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Buffers, Version=4.0.2.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\System.Buffers.4.4.0\lib\netstandard2.0\System.Buffers.dll</HintPath>
</Reference>
<Reference Include="System.Core" />
<Reference Include="System.Memory, Version=4.0.1.1, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\System.Memory.4.5.3\lib\netstandard2.0\System.Memory.dll</HintPath>
</Reference>
<Reference Include="System.Numerics" />
<Reference Include="System.Numerics.Vectors, Version=4.1.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Numerics.Vectors.4.4.0\lib\net46\System.Numerics.Vectors.dll</HintPath>
</Reference>
<Reference Include="System.Runtime.CompilerServices.Unsafe, Version=4.0.4.1, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Runtime.CompilerServices.Unsafe.4.5.2\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll</HintPath>
</Reference>
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />

8
Compression.BSA/Utils.cs
View File

@ -1,5 +1,4 @@
using lz4.AnyCPU.loader;
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
@ -10,11 +9,6 @@ namespace Compression.BSA
{
private static Encoding Windows1251 = Encoding.GetEncoding(1251);
static Utils ()
{
LZ4Loader.DisableVCRuntimeDetection = true;
}
public static string ReadStringLen(this BinaryReader rdr)
{
var len = rdr.ReadByte();

8
Compression.BSA/packages.config
View File

@ -1,4 +1,10 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="IonKiwi.lz4.net" version="1.0.12" targetFramework="net472" />
<package id="K4os.Compression.LZ4" version="1.1.11" targetFramework="net472" />
<package id="K4os.Compression.LZ4.Streams" version="1.1.11" targetFramework="net472" />
<package id="K4os.Hash.xxHash" version="1.0.6" targetFramework="net472" />
<package id="System.Buffers" version="4.4.0" targetFramework="net472" />
<package id="System.Memory" version="4.5.3" targetFramework="net472" />
<package id="System.Numerics.Vectors" version="4.4.0" targetFramework="net472" />
<package id="System.Runtime.CompilerServices.Unsafe" version="4.5.2" targetFramework="net472" />
</packages>

6
VirtualFileSystem.Test/App.config Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2" />
</startup>
</configuration>

17
VirtualFileSystem.Test/Program.cs Normal file
View File

@ -0,0 +1,17 @@
using System;
using Wabbajack.Common;
namespace VirtualFileSystem.Test
{
class Program
{
static void Main(string[] args)
{
Utils.SetLoggerFn(s => Console.WriteLine(s));
Utils.SetStatusFn((s, i) => Console.WriteLine(s));
WorkQueue.Init((a, b, c) => { return; },
(a, b) => { return; });
VFS.VirtualFileSystem.VFS.AddRoot(@"D:\MO2 Instances\Mod Organizer 2");
}
}
}

36
VirtualFileSystem.Test/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("VirtualFileSystem.Test")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("VirtualFileSystem.Test")]
[assembly: AssemblyCopyright("Copyright © 2019")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("a2913dfe-18ff-468b-a6c1-55f7c0cc0ce8")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

63
VirtualFileSystem.Test/VirtualFileSystem.Test.csproj Normal file
View File

@ -0,0 +1,63 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}</ProjectGuid>
<OutputType>Exe</OutputType>
<RootNamespace>VirtualFileSystem.Test</RootNamespace>
<AssemblyName>VirtualFileSystem.Test</AssemblyName>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<Deterministic>true</Deterministic>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>x64</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VirtualFileSystem\VirtualFileSystem.csproj">
<Project>{5128b489-bc28-4f66-9f0b-b4565af36cbc}</Project>
<Name>VirtualFileSystem</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
<Project>{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}</Project>
<Name>Wabbajack.Common</Name>
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>

36
VirtualFileSystem/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("VirtualFileSystem")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("VirtualFileSystem")]
[assembly: AssemblyCopyright("Copyright © 2019")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("5128b489-bc28-4f66-9f0b-b4565af36cbc")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

585
VirtualFileSystem/VirtualFileSystem.cs Normal file
View File

@ -0,0 +1,585 @@
using Compression.BSA;
using ICSharpCode.SharpZipLib.Zip;
using Newtonsoft.Json;
using SevenZipExtractor;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Reflection;
using Wabbajack.Common;
namespace VFS
{
public class VirtualFileSystem
{
internal static string _stagedRoot;
public static VirtualFileSystem VFS;
private Dictionary<string, VirtualFile> _files = new Dictionary<string, VirtualFile>();
private bool _disableDiskCache;
public static string RootFolder { get; }
public Dictionary<string, IEnumerable<VirtualFile>> HashIndex { get; private set; }
static VirtualFileSystem()
{
VFS = new VirtualFileSystem();
RootFolder = ".\\";
_stagedRoot = Path.Combine(RootFolder, "vfs_staged_files");
if (Directory.Exists(_stagedRoot))
Directory.Delete(_stagedRoot, true);
Directory.CreateDirectory(_stagedRoot);
}
public VirtualFileSystem ()
{
LoadFromDisk();
}
private void LoadFromDisk()
{
try
{
Utils.Log("Loading VFS Cache");
if (!File.Exists("vfs_cache.bson")) return;
_files = "vfs_cache.bson".FromBSON<IEnumerable<VirtualFile>>(root_is_array: true).ToDictionary(f => f.FullPath);
CleanDB();
}
catch(Exception ex)
{
Utils.Log($"Purging cache due to {ex}");
File.Delete("vfs_cache.bson");
_files.Clear();
}
}
public void SyncToDisk()
{
if (!_disableDiskCache)
lock(this)
{
_files.Values.OfType<VirtualFile>().ToBSON("vfs_cache.bson");
}
}
public IList<VirtualFile> FilesInArchive(VirtualFile f)
{
var path = f.FullPath + "|";
return _files.Values
.Where(v => v.FullPath.StartsWith(path))
.ToList();
}
public void Purge(VirtualFile f)
{
var path = f.FullPath + "|";
lock (this)
{
_files.Values
.Where(v => v.FullPath.StartsWith(path) || v.FullPath == f.FullPath)
.ToList()
.Do(r => {
_files.Remove(r.FullPath);
});
}
}
public void Add(VirtualFile f)
{
lock (this)
{
if (_files.ContainsKey(f.FullPath))
Purge(f);
_files.Add(f.FullPath, f);
}
}
public VirtualFile Lookup(string f)
{
lock (this)
{
if (_files.TryGetValue(f, out var found))
return found;
return null;
}
}
/// <summary>
/// Remove any orphaned files in the DB.
/// </summary>
private void CleanDB()
{
Utils.Log("Cleaning VFS cache");
lock (this)
{
_files.Values
.Where(f =>
{
if (f.IsConcrete)
return !File.Exists(f.StagedPath);
while (f.ParentPath != null)
{
if (Lookup(f.ParentPath) == null)
return true;
f = Lookup(f.ParentPath);
}
return false;
})
.ToList()
.Do(f => _files.Remove(f.FullPath));
}
}
public void BackfillMissing()
{
lock(this)
{
_files.Values
.Select(f => f.ParentPath)
.Where(s => s != null)
.Where(s => !_files.ContainsKey(s))
.ToHashSet()
.Do(s =>
{
AddKnown(new VirtualFile() { Paths = s.Split('|') });
});
}
}
/// <summary>
/// Add a known file to the index, bit of a hack as we won't assume that all the fields for the archive are filled in.
/// you will need to manually update the SHA hash when you are done adding files, by calling `RefreshIndexes`
/// </summary>
/// <param name="virtualFile"></param>
public void AddKnown(VirtualFile virtualFile)
{
lock(this)
{
// We don't know enough about these files to be able to store them in the disk cache
_disableDiskCache = true;
_files[virtualFile.FullPath] = virtualFile;
}
}
/// <summary>
/// Adds the root path to the filesystem. This may take quite some time as every file in the folder will be hashed,
/// and every archive examined.
/// </summary>
/// <param name="path"></param>
public void AddRoot(string path)
{
IndexPath(path);
RefreshIndexes();
}
public void RefreshIndexes()
{
Utils.Log("Building Hash Index");
lock(this)
{
HashIndex = _files.Values
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, f => (IEnumerable<VirtualFile>)f);
}
}
private void IndexPath(string path)
{
var file_list = Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories).ToList();
Utils.Log($"Updating the cache for {file_list.Count} files");
file_list.PMap(f => UpdateFile(f));
SyncToDisk();
}
private void UpdateFile(string f)
{
TOP:
var lv = Lookup(f);
if (lv == null)
{
Utils.Status($"Analyzing {f}");
lv = new VirtualFile()
{
Paths = new string[] { f }
};
lv.Analyze();
Add(lv);
if (lv.IsArchive)
{
UpdateArchive(lv);
}
// Upsert after extraction incase extraction fails
}
if (lv.IsOutdated)
{
Purge(lv);
goto TOP;
}
}
private void UpdateArchive(VirtualFile f)
{
var entries = GetArchiveEntryNames(f);
var new_files = entries.Select(e => {
var new_path = new string[f.Paths.Length + 1];
f.Paths.CopyTo(new_path, 0);
new_path[f.Paths.Length] = e;
var nf = new VirtualFile()
{
Paths = new_path,
};
Add(nf);
return nf;
}).ToList();
// Stage the files in the archive
Stage(new_files);
// Analyze them
new_files.Do(file => file.Analyze());
// Recurse into any archives in this archive
new_files.Where(file => file.IsArchive).Do(file => UpdateArchive(file));
// Unstage the file
new_files.Where(file => file.IsStaged).Do(file => file.Unstage());
f.FinishedIndexing = true;
SyncToDisk();
}
public void Stage(IEnumerable<VirtualFile> files)
{
var grouped = files.SelectMany(f => f.FilesInPath)
.Distinct()
.Where(f => f.ParentArchive != null)
.GroupBy(f => f.ParentArchive)
.OrderBy(f => f.Key == null ? 0 : f.Key.Paths.Length)
.ToList();
foreach (var group in grouped)
{
var indexed = group.ToDictionary(e => e.Paths[group.Key.Paths.Length]);
FileExtractor.Extract(group.Key.StagedPath, e =>
{
if (indexed.TryGetValue(e.Name, out var file))
{
return File.OpenWrite(file.GenerateStagedName());
}
return null;
});
}
}
public StagingGroup StageWith(IEnumerable<VirtualFile> files)
{
var grp = new StagingGroup(files);
grp.Stage();
return grp;
}
public VirtualFile this[string path]
{
get
{
return Lookup(path);
}
}
internal List<string> GetArchiveEntryNames(VirtualFile file)
{
if (!file.IsStaged)
throw new InvalidDataException("File is not staged");
if (file.Extension == ".bsa") {
using (var ar = new BSAReader(file.StagedPath))
{
return ar.Files.Select(f => f.Path).ToList();
}
}
if (file.Extension == ".zip")
{
using (var s = new ZipFile(File.OpenRead(file.StagedPath)))
{
s.IsStreamOwner = true;
s.UseZip64 = UseZip64.On;
if (s.OfType<ZipEntry>().FirstOrDefault(e => !e.CanDecompress) == null)
{
return s.OfType<ZipEntry>()
.Where(f => f.IsFile)
.Select(f => f.Name.Replace('/', '\\'))
.ToList();
}
}
}
using (var e = new ArchiveFile(file.StagedPath))
{
return e.Entries
.Where(f => !f.IsFolder)
.Select(f => f.FileName).ToList();
}
}
/// <summary>
/// Given a path that starts with a HASH, return the Virtual file referenced
/// </summary>
/// <param name="archiveHashPath"></param>
/// <returns></returns>
public VirtualFile FileForArchiveHashPath(string[] archiveHashPath)
{
var archive = HashIndex[archiveHashPath[0]].Where(a => a.IsArchive).OrderByDescending(a => a.LastModified).First();
string fullPath = archive.FullPath + "|" + String.Join("|", archiveHashPath.Skip(1));
return Lookup(fullPath);
}
}
public class StagingGroup : List<VirtualFile>, IDisposable
{
public StagingGroup(IEnumerable<VirtualFile> files) : base(files)
{
}
public void Dispose()
{
this.Do(f => f.Unstage());
}
internal void Stage()
{
VirtualFileSystem.VFS.Stage(this);
}
}
[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
public class VirtualFile
{
public string[] _paths;
[JsonProperty]
public string[] Paths
{
get
{
return _paths;
}
set
{
for (int idx = 0; idx < value.Length; idx += 1)
value[idx] = String.Intern(value[idx]);
_paths = value;
}
}
[JsonProperty]
public string Hash { get; set; }
[JsonProperty]
public long Size { get; set; }
[JsonProperty]
public ulong LastModified { get; set; }
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public bool? FinishedIndexing { get; set; }
private string _fullPath;
public VirtualFile()
{
}
private string _stagedPath;
public string FullPath
{
get
{
if (_fullPath != null) return _fullPath;
_fullPath = String.Join("|", Paths);
return _fullPath;
}
}
public string Extension
{
get
{
return Path.GetExtension(Paths.Last());
}
}
/// <summary>
/// If this file is in an archive, return the Archive File, otherwise return null.
/// </summary>
public VirtualFile TopLevelArchive
{
get
{
if (Paths.Length == 0) return null;
return VirtualFileSystem.VFS[Paths[0]];
}
}
public VirtualFile ParentArchive
{
get
{
if (ParentPath == null) return null;
return VirtualFileSystem.VFS.Lookup(ParentPath);
}
}
private bool? _isArchive;
public bool IsArchive
{
get
{
if (_isArchive == null)
_isArchive = FileExtractor.CanExtract(Extension);
return (bool)_isArchive;
}
}
public bool IsStaged
{
get
{
if (IsConcrete) return true;
return _stagedPath != null;
}
}
public string StagedPath
{
get
{
if (!IsStaged)
throw new InvalidDataException("File is not staged");
if (IsConcrete) return Paths[0];
return _stagedPath;
}
set
{
if (IsStaged && value != null)
throw new InvalidDataException("Can't change the path of a already staged file");
_stagedPath = value;
}
}
public FileStream OpenRead()
{
if (!IsStaged)
throw new InvalidDataException("File is not staged, cannot open");
return File.OpenRead(_stagedPath);
}
/// <summary>
/// Calulate the file's SHA, size and last modified
/// </summary>
internal void Analyze()
{
if (!IsStaged)
throw new InvalidDataException("Cannot analzye a unstaged file");
var fio = new FileInfo(StagedPath);
Size = fio.Length;
Hash = Utils.FileSHA256(StagedPath);
LastModified = fio.LastWriteTime.ToMilliseconds();
}
/// <summary>
/// Delete the temoporary file associated with this file
/// </summary>
internal void Unstage()
{
if (IsStaged && !IsConcrete)
{
File.Delete(_stagedPath);
_stagedPath = null;
}
}
internal string GenerateStagedName()
{
if (_stagedPath != null) return _stagedPath;
_stagedPath = Path.Combine(VirtualFileSystem._stagedRoot, Guid.NewGuid().ToString() + Path.GetExtension(Paths.Last()));
return _stagedPath;
}
/// <summary>
/// Returns true if this file always exists on-disk, and doesn't need to be staged.
/// </summary>
public bool IsConcrete
{
get
{
return Paths.Length == 1;
}
}
public bool IsOutdated
{
get
{
if (IsStaged)
{
var fi = new FileInfo(StagedPath);
if (fi.LastWriteTime.ToMilliseconds() != LastModified || fi.Length != Size)
return true;
if (IsArchive)
if (!FinishedIndexing ?? true)
return true;
}
return false;
}
}
private string _parentPath;
public string ParentPath
{
get {
if (_parentPath == null && !IsConcrete)
_parentPath = String.Join("|", Paths.Take(Paths.Length - 1));
return _parentPath;
}
}
public IEnumerable<VirtualFile> FileInArchive
{
get
{
return VirtualFileSystem.VFS.FilesInArchive(this);
}
}
public string[] MakeRelativePaths()
{
var path_copy = (string[])Paths.Clone();
path_copy[0] = VirtualFileSystem.VFS.Lookup(Paths[0]).Hash;
return path_copy;
}
public IEnumerable<VirtualFile> FilesInPath
{
get {
return Enumerable.Range(1, Paths.Length)
.Select(i => Paths.Take(i))
.Select(path => VirtualFileSystem.VFS.Lookup(String.Join("|", path)));
}
}
}
}

76
VirtualFileSystem/VirtualFileSystem.csproj Normal file
View File

@ -0,0 +1,76 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{5128B489-BC28-4F66-9F0B-B4565AF36CBC}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>VirtualFileSystem</RootNamespace>
<AssemblyName>VirtualFileSystem</AssemblyName>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<Deterministic>true</Deterministic>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<PlatformTarget>x64</PlatformTarget>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="ICSharpCode.SharpZipLib, Version=1.2.0.246, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
<HintPath>..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Collections.Immutable, Version=1.2.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Collections.Immutable.1.5.0\lib\netstandard2.0\System.Collections.Immutable.dll</HintPath>
</Reference>
<Reference Include="System.Core" />
<Reference Include="System.IO.Compression.FileSystem" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="VirtualFileSystem.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Compression.BSA\Compression.BSA.csproj">
<Project>{ff5d892f-8ff4-44fc-8f7f-cd58f307ad1b}</Project>
<Name>Compression.BSA</Name>
</ProjectReference>
<ProjectReference Include="..\SevenZipExtractor\SevenZipExtractor.csproj">
<Project>{8aa97f58-5044-4bba-b8d9-a74b6947a660}</Project>
<Name>SevenZipExtractor</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
<Project>{b3f3fb6e-b9eb-4f49-9875-d78578bc7ae5}</Project>
<Name>Wabbajack.Common</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>

6
VirtualFileSystem/packages.config Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Newtonsoft.Json" version="12.0.2" targetFramework="net472" />
<package id="SharpZipLib" version="1.2.0" targetFramework="net472" />
<package id="System.Collections.Immutable" version="1.5.0" targetFramework="net472" />
</packages>

1
Wabbajack.Common/Consts.cs
View File

@ -30,5 +30,6 @@ namespace Wabbajack.Common
public static string WABBAJACK_INCLUDE = "WABBAJACK_INCLUDE";
public static String AppName = "Wabbajack";
public static string HashCacheName = "Wabbajack.hash_cache";
}
}

28
Wabbajack.Common/FileExtractor.cs
View File

@ -108,11 +108,24 @@ namespace Wabbajack.Common
}
}
/// <summary>
/// Returns true if the given extension type can be extracted
/// </summary>
/// <param name="v"></param>
/// <returns></returns>
public static bool CanExtract(string v)
{
return Consts.SupportedArchives.Contains(v) || v == ".bsa";
}
// Probably replace this with VFS?
/*
public static void DeepExtract(string file, IEnumerable<FromArchive> files, Func<FromArchive, Entry, Stream> fnc, bool leave_open = false, int depth = 1)
{
// Files we need to extract at this level
var files_for_level = files.Where(f => f.ArchiveHashPath.Length == depth).ToDictionary(e => e.From);
var files_for_level = files.Where(f => f.ArchiveHashPath.Length == depth)
.GroupBy(e => e.From)
.ToDictionary(e => e.Key);
// Archives we need to extract at this level
var archives_for_level = files.Where(f => f.ArchiveHashPath.Length > depth)
.GroupBy(f => f.ArchiveHashPath[depth])
@ -127,12 +140,21 @@ namespace Wabbajack.Common
if (files_for_level.TryGetValue(e.Name, out var fe))
{
a = fnc(fe, e);
foreach (var inner_fe in fe)
{
var str = fnc(inner_fe, e);
if (str == null) continue;
a = new SplittingStream(a, false, fnc(inner_fe, e), leave_open);
}
}
if (archives_for_level.TryGetValue(e.Name, out var archive))
{
var name = Path.GetTempFileName() + Path.GetExtension(e.Name);
if (disk_archives.ContainsKey(e.Name))
{
}
disk_archives.Add(e.Name, name);
b = File.OpenWrite(name);
}
@ -148,6 +170,8 @@ namespace Wabbajack.Common
DeepExtract(archive.Value, archives_for_level[archive.Key], fnc, leave_open, depth + 1);
File.Delete(archive.Value);
}
}
*/
}
}

67
Wabbajack.Common/Utils.cs
View File

@ -1,6 +1,7 @@
using ICSharpCode.SharpZipLib.BZip2;
using IniParser;
using Newtonsoft.Json;
using Newtonsoft.Json.Bson;
using System;
using System.Collections.Generic;
using System.Diagnostics;
@ -16,7 +17,28 @@ namespace Wabbajack.Common
{
public static class Utils
{
private static Action<string> _loggerFn;
private static Action<string, int> _statusFn;
public static void SetLoggerFn(Action<string> f)
{
_loggerFn = f;
}
public static void SetStatusFn(Action<string, int> f)
{
_statusFn = f;
}
public static void Log(string msg)
{
_loggerFn?.Invoke(msg);
}
public static void Status(string msg, int progress = 0)
{
_statusFn?.Invoke(msg, progress);
}
/// <summary>
@ -89,6 +111,22 @@ namespace Wabbajack.Common
File.WriteAllText(filename, JsonConvert.SerializeObject(obj, Formatting.Indented, new JsonSerializerSettings() {TypeNameHandling = TypeNameHandling.Auto}));
}
public static void ToBSON<T>(this T obj, string filename)
{
using(var fo = File.OpenWrite(filename))
using(var br = new BsonDataWriter(fo))
{
fo.SetLength(0);
var serializer = JsonSerializer.Create(new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto });
serializer.Serialize(br, obj);
}
}
public static ulong ToMilliseconds(this DateTime date)
{
return (ulong)(date - new DateTime(1970, 1, 1)).TotalMilliseconds;
}
public static string ToJSON<T>(this T obj)
{
return JsonConvert.SerializeObject(obj, Formatting.Indented, new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto });
@ -99,6 +137,17 @@ namespace Wabbajack.Common
return JsonConvert.DeserializeObject<T>(File.ReadAllText(filename), new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto });
}
public static T FromBSON<T>(this string filename, bool root_is_array = false)
{
using (var fo = File.OpenRead(filename))
using (var br = new BsonDataReader(fo, readRootValueAsArray: root_is_array, DateTimeKind.Local))
{
var serializer = JsonSerializer.Create(new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto });
return serializer.Deserialize<T>(br);
}
}
public static T FromJSONString<T>(this string data)
{
return JsonConvert.DeserializeObject<T>(data, new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto });
@ -269,6 +318,24 @@ namespace Wabbajack.Common
File.WriteAllText($"{DateTime.Now.ToString("yyyyMMddTHHmmss_crash_log.txt")}", ExceptionToString(e));
}
public static IEnumerable<T> DistinctBy<T, V>(this IEnumerable<T> vs, Func<T, V> select)
{
HashSet<V> set = new HashSet<V>();
foreach (var v in vs) {
var key = select(v);
if (set.Contains(key)) continue;
yield return v;
}
}
public static T Last<T>(this T[] a)
{
if (a == null || a.Length == 0)
throw new InvalidDataException("null or empty array");
return a[a.Length - 1];
}
public static V GetOrDefault<K, V>(this IDictionary<K, V> dict, K key)
{
if (dict.TryGetValue(key, out V v)) return v;

4
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -62,6 +62,9 @@
<Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json.Bson, Version=1.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.Bson.1.0.2\lib\net45\Newtonsoft.Json.Bson.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.IO.Compression" />
@ -75,7 +78,6 @@
<ItemGroup>
<Compile Include="BSDiff.cs" />
<Compile Include="Consts.cs" />
<Compile Include="Data.cs" />
<Compile Include="DynamicIniData.cs" />
<Compile Include="FileExtractor.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />

1
Wabbajack.Common/packages.config
View File

@ -3,6 +3,7 @@
<package id="ini-parser" version="2.5.2" targetFramework="net472" />
<package id="murmurhash" version="1.0.3" targetFramework="net472" />
<package id="Newtonsoft.Json" version="12.0.2" targetFramework="net472" />
<package id="Newtonsoft.Json.Bson" version="1.0.2" targetFramework="net472" />
<package id="SharpZipLib" version="1.1.0" targetFramework="net472" />
<package id="System.Runtime.Numerics" version="4.3.0" targetFramework="net472" />
</packages>

28
Wabbajack.sln
View File

@ -20,6 +20,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
RECIPES.md = RECIPES.md
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VirtualFileSystem", "VirtualFileSystem\VirtualFileSystem.csproj", "{5128B489-BC28-4F66-9F0B-B4565AF36CBC}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VirtualFileSystem.Test", "VirtualFileSystem.Test\VirtualFileSystem.Test.csproj", "{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug (no commandargs)|Any CPU = Debug (no commandargs)|Any CPU
@ -90,6 +94,30 @@ Global
{BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Release|Any CPU.Build.0 = Release|Any CPU
{BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Release|x64.ActiveCfg = Release|x64
{BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Release|x64.Build.0 = Release|x64
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|x64.ActiveCfg = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Debug|x64.Build.0 = Debug|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|Any CPU.Build.0 = Release|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|x64.ActiveCfg = Release|Any CPU
{5128B489-BC28-4F66-9F0B-B4565AF36CBC}.Release|x64.Build.0 = Release|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|x64.ActiveCfg = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Debug|x64.Build.0 = Debug|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|Any CPU.Build.0 = Release|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|x64.ActiveCfg = Release|Any CPU
{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

5
Wabbajack/AppState.cs
View File

@ -130,6 +130,7 @@ namespace Wabbajack
public AppState(Dispatcher d, String mode)
{
_startTime = DateTime.Now;
ArchiveFile.SetupLibrary();
LogFile = Assembly.GetExecutingAssembly().Location + ".log";
@ -186,6 +187,7 @@ namespace Wabbajack
public void LogMsg(string msg)
{
msg = $"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}";
dispatcher.Invoke(() => Log.Add(msg));
lock (dispatcher) {
File.AppendAllText(LogFile, msg + "\r\n");
@ -264,6 +266,8 @@ namespace Wabbajack
}
private ICommand _begin;
private DateTime _startTime;
public ICommand Begin
{
get
@ -312,7 +316,6 @@ namespace Wabbajack
{
try
{
compiler.LoadArchives();
compiler.Compile();
}
catch (Exception ex)

385
Wabbajack/Compiler.cs
View File

@ -1,6 +1,5 @@
using Compression.BSA;
using Newtonsoft.Json;
using SharpCompress.Archives;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
@ -10,10 +9,10 @@ using System.Reflection;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web;
using Wabbajack.Common;
using static Wabbajack.NexusAPI;
using VFS;
namespace Wabbajack
{
@ -59,9 +58,26 @@ namespace Wabbajack
public ConcurrentBag<Directive> ExtraFiles { get; private set; }
public Dictionary<string, dynamic> ModInis { get; private set; }
public List<IndexedArchive> IndexedArchives;
public VirtualFileSystem VFS
{
get
{
return VirtualFileSystem.VFS;
}
public List<IndexedArchiveEntry> IndexedFiles { get; private set; }
}
public List<IndexedArchive> IndexedArchives { get; private set; }
public Dictionary<string, IEnumerable<VirtualFile>> IndexedFiles { get; private set; }
public class IndexedFileMatch
{
public IndexedArchive Archive;
public IndexedArchiveEntry Entry;
public DateTime LastModified;
}
public Dictionary<string, IEnumerable<IndexedFileMatch>> DirectMatchIndex;
public void Info(string msg, params object[] args)
{
@ -94,85 +110,21 @@ namespace Wabbajack
GamePath = ((string)MO2Ini.General.gamePath).Replace("\\\\", "\\");
}
public void LoadArchives()
{
IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder)
.Where(file => Consts.SupportedArchives.Contains(Path.GetExtension(file)))
.PMap(file => LoadArchive(file));
IndexedFiles = FlattenFiles(IndexedArchives);
Info($"Found {IndexedFiles.Count} files in archives");
}
private List<IndexedArchiveEntry> FlattenFiles(IEnumerable<IndexedArchive> archives)
{
return archives.PMap(e => FlattenArchiveEntries(e, null, new string[0]))
.SelectMany(e => e)
.ToList();
}
private IEnumerable<IndexedArchiveEntry> FlattenArchiveEntries(IndexedArchiveCache archive, string name, string[] path)
{
var new_path = new string[path.Length + 1];
Array.Copy(path, 0, new_path, 0, path.Length);
new_path[path.Length] = path.Length == 0 ? archive.Hash : name;
foreach (var e in archive.Entries)
{
yield return new IndexedArchiveEntry()
{
Path = e.Path,
Size = e.Size,
Hash = e.Hash,
HashPath = new_path
};
}
if (archive.InnerArchives != null) {
foreach (var inner in archive.InnerArchives)
{
foreach (var entry in FlattenArchiveEntries(inner.Value, inner.Key, new_path))
{
yield return entry;
}
}
}
}
private const int ARCHIVE_CONTENTS_VERSION = 1;
private IndexedArchive LoadArchive(string file)
{
TOP:
string metaname = file + ".archive_contents";
var info = new IndexedArchive();
info.File = VFS.Lookup(file);
if (metaname.FileExists() && new FileInfo(metaname).LastWriteTime >= new FileInfo(file).LastWriteTime)
info.Name = Path.GetFileName(file);
var ini_name = file + ".meta";
if (ini_name.FileExists())
{
Status("Loading Archive Index for {0}", Path.GetFileName(file));
var info = metaname.FromJSON<IndexedArchive>();
if (info.Version != ARCHIVE_CONTENTS_VERSION)
{
File.Delete(metaname);
goto TOP;
}
info.Name = Path.GetFileName(file);
info.AbsolutePath = file;
var ini_name = file + ".meta";
if (ini_name.FileExists())
{
info.IniData = ini_name.LoadIniFile();
info.Meta = File.ReadAllText(ini_name);
}
return info;
info.IniData = ini_name.LoadIniFile();
info.Meta = File.ReadAllText(ini_name);
}
IndexArchive(file).ToJSON(metaname);
goto TOP;
return info;
}
private bool IsArchiveFile(string name)
@ -183,70 +135,44 @@ namespace Wabbajack
return false;
}
private IndexedArchiveCache IndexArchive(string file)
{
Status("Indexing {0}", Path.GetFileName(file));
var streams = new Dictionary<string, (SHA256Managed, long)>();
var inner_archives = new Dictionary<string, string>();
FileExtractor.Extract(file, entry =>
{
Stream inner;
if (IsArchiveFile(entry.Name))
{
var name = Path.GetTempFileName() + Path.GetExtension(entry.Name);
inner_archives.Add(entry.Name, name);
inner = File.OpenWrite(name);
}
else
{
inner = Stream.Null;
}
var sha = new SHA256Managed();
var os = new CryptoStream(inner, sha, CryptoStreamMode.Write);
streams.Add(entry.Name, (sha, (long)entry.Size));
return os;
});
var indexed = new IndexedArchiveCache();
indexed.Version = ARCHIVE_CONTENTS_VERSION;
indexed.Hash = file.FileSHA256();
indexed.Entries = streams.Select(entry =>
{
return new IndexedEntry()
{
Hash = entry.Value.Item1.Hash.ToBase64(),
Size = (long)entry.Value.Item2,
Path = entry.Key
};
}).ToList();
streams.Do(e => e.Value.Item1.Dispose());
if (inner_archives.Count > 0)
{
var result = inner_archives.Select(archive =>
{
return (archive.Key, IndexArchive(archive.Value));
}).ToDictionary(e => e.Key, e => e.Item2);
indexed.InnerArchives = result;
inner_archives.Do(e => File.Delete(e.Value));
}
return indexed;
}
public void Compile()
{
Info($"Indexing {MO2Folder}");
VFS.AddRoot(MO2Folder);
Info($"Indexing {GamePath}");
VFS.AddRoot(GamePath);
var mo2_files = Directory.EnumerateFiles(MO2Folder, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile() { Path = p.RelativeTo(MO2Folder), AbsolutePath = p });
.Select(p => new RawSourceFile(VFS.Lookup(p)) { Path = p.RelativeTo(MO2Folder)});
var game_files = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile() { Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)), AbsolutePath = p });
.Select(p => new RawSourceFile(VFS.Lookup(p)) { Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath))});
Info($"Indexing Archives");
IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder)
.Where(f => Consts.SupportedArchives.Contains(Path.GetExtension(f)))
.Where(f => File.Exists(f + ".meta"))
.Select(f => new IndexedArchive()
{
File = VFS.Lookup(f),
IniData = (f + ".meta").LoadIniFile(),
Meta = File.ReadAllText(f + ".meta")
})
.ToList();
Info($"Indexing Files");
IndexedFiles = IndexedArchives.PMap(f => { Status($"Finding files in {Path.GetFileName(f.File.FullPath)}");
return VFS.FilesInArchive(f.File); })
.SelectMany(fs => fs)
.OrderByDescending(f => f.TopLevelArchive.LastModified)
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, f => f.AsEnumerable());
Info("Searching for mod files");
AllFiles = mo2_files.Concat(game_files).ToList();
Info("Found {0} files to build into mod list", AllFiles.Count);
@ -326,7 +252,6 @@ namespace Wabbajack
private void ResetMembers()
{
AllFiles = null;
IndexedArchives = null;
InstallDirectives = null;
SelectedArchives = null;
ExtraFiles = null;
@ -355,40 +280,25 @@ namespace Wabbajack
private void BuildArchivePatches(string archive_sha, IEnumerable<PatchedFromArchive> group, Dictionary<string, string> absolute_paths)
{
var archive = IndexedArchives.First(a => a.Hash == archive_sha);
var paths = group.Select(g => g.FullPath).ToHashSet();
var streams = new Dictionary<string, MemoryStream>();
Status($"Extracting {paths.Count} patch files from {archive.Name}");
// First we fetch the source files from the input archive
FileExtractor.DeepExtract(archive.AbsolutePath, group, (fe, entry) =>
var archive = VFS.HashIndex[archive_sha];
using (var files = VFS.StageWith(group.Select(g => VFS.FileForArchiveHashPath(g.ArchiveHashPath))))
{
if (!paths.Contains(fe.FullPath)) return null;
var result = new MemoryStream();
streams.Add(fe.FullPath, result);
return result;
}, false);
var extracted = streams.ToDictionary(k => k.Key, v => v.Value.ToArray());
// Now Create the patches
Status("Building Patches for {0}", archive.Name);
Info("Building Patches for {0}", archive.Name);
group.PMap(entry =>
{
Info("Patching {0}", entry.To);
var ss = extracted[entry.FullPath];
using (var origin = new MemoryStream(ss))
using (var output = new MemoryStream())
var by_path = files.GroupBy(f => string.Join("|", f.Paths.Skip(1))).ToDictionary(f => f.Key, f => f.First());
// Now Create the patches
group.PMap(entry =>
{
var a = origin.ReadAll();
var b = LoadDataForTo(entry.To, absolute_paths);
BSDiff.Create(a, b, output);
entry.Patch = output.ToArray().ToBase64();
Info($"Patch size {entry.Patch.Length} for {entry.To}");
}
});
Info("Patching {0}", entry.To);
using (var origin = by_path[string.Join("|", entry.ArchiveHashPath.Skip(1))].OpenRead())
using (var output = new MemoryStream())
{
var a = origin.ReadAll();
var b = LoadDataForTo(entry.To, absolute_paths);
BSDiff.Create(a, b, output);
entry.Patch = output.ToArray().ToBase64();
Info($"Patch size {entry.Patch.Length} for {entry.To}");
}
});
}
}
@ -416,17 +326,20 @@ namespace Wabbajack
private void GatherArchives()
{
Info($"Building a list of archives based on the files required");
var archives = IndexedArchives.GroupBy(a => a.Hash).ToDictionary(k => k.Key, k => k.First());
var shas = InstallDirectives.OfType<FromArchive>()
.Select(a => a.ArchiveHashPath[0])
.Distinct();
var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified)
.GroupBy(f => f.File.Hash)
.ToDictionary(f => f.Key, f => f.First());
SelectedArchives = shas.PMap(sha => ResolveArchive(sha, archives));
}
private Archive ResolveArchive(string sha, Dictionary<string, IndexedArchive> archives)
private Archive ResolveArchive(string sha, IDictionary<string, IndexedArchive> archives)
{
if (archives.TryGetValue(sha, out var found))
{
@ -520,6 +433,13 @@ namespace Wabbajack
}
result = tmp;
}
else if (general.manualURL != null)
{
result = new ManualURLArchive()
{
URL = general.manualURL.ToString()
};
}
else
{
Error("No way to handle archive {0} but it's required by the modpack", found.Name);
@ -527,7 +447,7 @@ namespace Wabbajack
}
result.Name = found.Name;
result.Hash = found.Hash;
result.Hash = found.File.Hash;
result.Meta = found.Meta;
return result;
@ -540,10 +460,12 @@ namespace Wabbajack
private Directive RunStack(IEnumerable<Func<RawSourceFile, Directive>> stack, RawSourceFile source)
{
Status("Compiling {0}", source.Path);
return (from f in stack
let result = f(source)
where result != null
select result).First();
foreach (var f in stack)
{
var result = f(source);
if (result != null) return result;
}
throw new InvalidDataException("Data fell out of the compilation stack");
}
@ -562,6 +484,7 @@ namespace Wabbajack
IgnoreStartsWith("downloads\\"),
IgnoreStartsWith("webcache\\"),
IgnoreStartsWith("overwrite\\"),
IgnorePathContains("temporary_logs"),
IgnoreEndsWith(".pyc"),
IgnoreEndsWith(".log"),
IgnoreOtherProfiles(),
@ -575,10 +498,10 @@ namespace Wabbajack
IgnoreRegex(Consts.GameFolderFilesDir + "\\\\.*\\.bsa"),
IncludeModIniData(),
DirectMatch(),
IncludeTaggedFiles(),
DeconstructBSAs(), // Deconstruct BSAs before building patches so we don't generate massive patch files
IncludePatches(),
IncludeDummyESPs(),
IncludeTaggedFiles(),
// If we have no match at this point for a game folder file, skip them, we can't do anything about them
@ -597,6 +520,22 @@ namespace Wabbajack
};
}
private Func<RawSourceFile, Directive> IgnorePathContains(string v)
{
v = $"\\{v.Trim('\\')}\\";
var reason = $"Ignored because path contains {v}";
return source =>
{
if (source.Path.Contains(v))
{
var result = source.EvolveTo<IgnoredDirectly>();
result.Reason = reason;
return result;
}
return null;
};
}
/// <summary>
/// If a user includes WABBAJACK_INCLUDE directly in the notes or comments of a mod, the contents of that
@ -672,6 +611,15 @@ namespace Wabbajack
/// <returns></returns>
private Func<RawSourceFile, Directive> DeconstructBSAs()
{
var include_directly = ModInis.Where(kv => {
var general = kv.Value.General;
if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE))
return true;
if (general.comments != null && general.comments.Contains(Consts.WABBAJACK_INCLUDE))
return true;
return false;
}).Select(kv => $"mods\\{kv.Key}\\");
var microstack = new List<Func<RawSourceFile, Directive>>()
{
DirectMatch(),
@ -679,20 +627,36 @@ namespace Wabbajack
DropAll()
};
var microstack_with_include = new List<Func<RawSourceFile, Directive>>()
{
DirectMatch(),
IncludePatches(),
IncludeALL()
};
return source =>
{
if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path))) return null;
var hashed = HashBSA(source.AbsolutePath);
bool default_include = false;
if (source.Path.StartsWith("mods"))
{
foreach (var modpath in include_directly)
{
if (source.Path.StartsWith(modpath))
{
default_include = true;
break;
}
}
}
var source_files = hashed.Select(e => new RawSourceFile() {
Hash = e.Item2,
Path = e.Item1,
AbsolutePath = e.Item1
});
var source_files = source.File.FileInArchive;
var stack = default_include ? microstack_with_include : microstack;
var matches = source_files.Select(e => RunStack(microstack, e));
var matches = source_files.PMap(e => RunStack(stack, new RawSourceFile(e)));
var id = Guid.NewGuid().ToString();
@ -724,26 +688,14 @@ namespace Wabbajack
};
}
/// <summary>
/// Given a BSA on disk, index it and return a dictionary of SHA256 -> filename
/// </summary>
/// <param name="absolutePath"></param>
/// <returns></returns>
private List<(string, string)> HashBSA(string absolutePath)
private Func<RawSourceFile, Directive> IncludeALL()
{
Status($"Hashing BSA: {absolutePath}");
var results = new List<(string, string)>();
using (var a = new BSAReader(absolutePath))
return source =>
{
a.Files.PMap(entry =>
{
Status($"Hashing BSA: {absolutePath} - {entry.Path}");
var data = entry.GetData();
results.Add((entry.Path, data.SHA256()));
});
}
return results;
var inline = source.EvolveTo<InlineFile>();
inline.SourceData = File.ReadAllBytes(source.AbsolutePath).ToBase64();
return inline;
};
}
private Func<RawSourceFile, Directive> IgnoreDisabledMods()
@ -766,23 +718,20 @@ namespace Wabbajack
private Func<RawSourceFile, Directive> IncludePatches()
{
var archive_shas = IndexedArchives.GroupBy(e => e.Hash)
.ToDictionary(e => e.Key);
var indexed = (from entry in IndexedFiles
select new { archive = archive_shas[entry.HashPath[0]].First(),
entry = entry })
.GroupBy(e => Path.GetFileName(e.entry.Path).ToLower())
.ToDictionary(e => e.Key);
var indexed = IndexedFiles.Values
.SelectMany(f => f)
.GroupBy(f => Path.GetFileName(f.Paths.Last()).ToLower())
.ToDictionary(f => f.Key);
return source =>
{
if (indexed.TryGetValue(Path.GetFileName(source.Path.ToLower()), out var value))
{
// TODO: Improve this
var found = value.First();
var e = source.EvolveTo<PatchedFromArchive>();
e.From = found.entry.Path;
e.ArchiveHashPath = found.entry.HashPath;
e.ArchiveHashPath = found.MakeRelativePaths();
e.To = source.Path;
return e;
}
@ -907,33 +856,21 @@ namespace Wabbajack
private Func<RawSourceFile, Directive> DirectMatch()
{
var archive_shas = IndexedArchives.GroupBy(e => e.Hash)
.ToDictionary(e => e.Key);
var indexed = (from entry in IndexedFiles
select new { archive = archive_shas[entry.HashPath[0]].First(),
entry = entry })
.GroupBy(e => e.entry.Hash)
.ToDictionary(e => e.Key);
return source =>
{
if (indexed.TryGetValue(source.Hash, out var found))
if (IndexedFiles.TryGetValue(source.Hash, out var found))
{
var result = source.EvolveTo<FromArchive>();
var match = found.Where(f => Path.GetFileName(f.entry.Path) == Path.GetFileName(source.Path))
.OrderByDescending(f => new FileInfo(f.archive.AbsolutePath).LastWriteTime)
var match = found.Where(f => Path.GetFileName(f.Paths[0]) == Path.GetFileName(source.Path))
.FirstOrDefault();
if (match == null)
match = found.OrderByDescending(f => new FileInfo(f.archive.AbsolutePath).LastWriteTime)
.FirstOrDefault();
match = found.FirstOrDefault();
result.ArchiveHashPath = match.MakeRelativePaths();
result.ArchiveHashPath = match.entry.HashPath;
result.From = match.entry.Path;
return result;
}
return null;

68
Wabbajack.Common/Data.cs → Wabbajack/Data.cs
View File

@ -4,27 +4,36 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using VFS;
namespace Wabbajack.Common
namespace Wabbajack
{
public class RawSourceFile
public class RawSourceFile
{
public string AbsolutePath;
public RawSourceFile(VirtualFile file)
{
File = file;
}
public string AbsolutePath
{
get
{
return File.StagedPath;
}
}
public string Path;
private string _hash;
public VirtualFile File { get; private set; }
public string Hash
{
get
{
if (_hash != null) return _hash;
_hash = AbsolutePath.FileSHA256();
return _hash;
}
set
{
_hash = value;
return File.Hash;
}
}
public T EvolveTo<T>() where T : Directive, new()
@ -93,10 +102,9 @@ namespace Wabbajack.Common
/// MurMur3 hash of the archive this file comes from
/// </summary>
public string[] ArchiveHashPath;
/// <summary>
/// The relative path of the file in the archive
/// </summary>
public string From;
[JsonIgnore]
public VirtualFile FromFile;
private string _fullPath = null;
[JsonIgnore]
@ -105,9 +113,7 @@ namespace Wabbajack.Common
get
{
if (_fullPath == null) {
var path = ArchiveHashPath.ToList();
path.Add(From);
_fullPath = String.Join("|", path);
_fullPath = String.Join("|", ArchiveHashPath);
}
return _fullPath;
}
@ -174,6 +180,14 @@ namespace Wabbajack.Common
public List<string> Headers;
}
/// <summary>
/// A URL that cannot be downloaded automatically and has to be downloaded by hand
/// </summary>
public class ManualURLArchive : Archive
{
public string URL;
}
/// <summary>
/// An archive that requires additional HTTP headers.
/// </summary>
@ -203,26 +217,12 @@ namespace Wabbajack.Common
{
}
/// <summary>
/// The indexed contents of an archive
/// </summary>
public class IndexedArchiveCache
{
public string Hash;
public int Version;
public List<IndexedEntry> Entries;
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public Dictionary<string, IndexedArchiveCache> InnerArchives;
}
public class IndexedArchive : IndexedArchiveCache
public class IndexedArchive
{
public dynamic IniData;
public string Name;
public string Meta;
public string AbsolutePath;
public List<string> HashPath;
public VirtualFile File { get; internal set; }
}
/// <summary>

72
Wabbajack/Installer.cs
View File

@ -9,12 +9,21 @@ using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using VFS;
using Wabbajack.Common;
namespace Wabbajack
{
public class Installer
{
public VirtualFileSystem VFS
{
get
{
return VirtualFileSystem.VFS;
}
}
public Installer(ModList mod_list, string output_folder, Action<string> log_fn)
{
Outputfolder = output_folder;
@ -88,6 +97,9 @@ namespace Wabbajack
Error("Cannot continue, was unable to download one or more archives");
}
}
PrimeVFS();
BuildFolderStructure();
InstallArchives();
InstallIncludedFiles();
@ -96,6 +108,37 @@ namespace Wabbajack
Info("Installation complete! You may exit the program.");
}
/// <summary>
/// We don't want to make the installer index all the archives, that's just a waste of time, so instead
/// we'll pass just enough information to VFS to let it know about the files we have.
/// </summary>
private void PrimeVFS()
{
HashedArchives.Do(a => VFS.AddKnown(new VirtualFile()
{
Paths = new string[] { a.Value },
Hash = a.Key
}));
VFS.RefreshIndexes();
ModList.Directives
.OfType<FromArchive>()
.Do(f =>
{
var updated_path = new string[f.ArchiveHashPath.Length];
f.ArchiveHashPath.CopyTo(updated_path, 0);
updated_path[0] = VFS.HashIndex[updated_path[0]].Where(e => e.IsConcrete).First().FullPath;
VFS.AddKnown(new VirtualFile() { Paths = updated_path });
});
VFS.BackfillMissing();
}
private void BuildBSAs()
{
var bsas = ModList.Directives.OfType<CreateBSA>().ToList();
@ -131,8 +174,11 @@ namespace Wabbajack
}
});
Info($"Removing temp folder {Consts.BSACreationDir}");
Directory.Delete(Path.Combine(Outputfolder, Consts.BSACreationDir), true);
if (Directory.Exists(Consts.BSACreationDir))
{
Info($"Removing temp folder {Consts.BSACreationDir}");
Directory.Delete(Path.Combine(Outputfolder, Consts.BSACreationDir), true);
}
}
@ -187,18 +233,18 @@ namespace Wabbajack
var files = grouping.GroupBy(e => e.FullPath)
.ToDictionary(e => e.Key);
FileExtractor.DeepExtract(absolutePath, files.Select(f => f.Value.First()),
(fe, entry) =>
var vfiles = files.Select(g =>
{
if (files.TryGetValue(fe.FullPath, out var directives))
{
var directive = directives.First();
var absolute = Path.Combine(Outputfolder, directive.To);
if (absolute.FileExists()) File.Delete(absolute);
return File.OpenWrite(absolute);
}
return null;
});
var first_file = g.Value.First();
var file = VFS.FileForArchiveHashPath(first_file.ArchiveHashPath);
file.StagedPath = first_file.To;
return file;
}).ToList();
VFS.Stage(vfiles);
vfiles.Do(f => f.StagedPath = null);
Status("Copying duplicated files for {0}", archive.Name);

4
Wabbajack/MainWindow.xaml.cs
View File

@ -48,6 +48,9 @@ namespace Wabbajack
WorkQueue.Init((id, msg, progress) => context.SetProgress(id, msg, progress),
(max, current) => context.SetQueueSize(max, current));
Utils.SetLoggerFn(s => context.LogMsg(s));
Utils.SetStatusFn((msg, progress) => WorkQueue.Report(msg, progress));
if (DebugMode)
@ -60,7 +63,6 @@ namespace Wabbajack
context.ModListName = compiler.MO2Profile;
context.Mode = "Building";
compiler.LoadArchives();
compiler.Compile();
var modlist = compiler.ModList.ToJSON();

5
Wabbajack/Wabbajack.csproj
View File

@ -139,6 +139,7 @@
<Generator>MSBuild:Compile</Generator>
<SubType>Designer</SubType>
</ApplicationDefinition>
<Compile Include="Data.cs" />
<Compile Include="LambdaCommand.cs" />
<Compile Include="Themes\LeftMarginMultiplierConverter.cs" />
<Compile Include="Themes\TreeViewItemExtensions.cs" />
@ -200,6 +201,10 @@
<Project>{8aa97f58-5044-4bba-b8d9-a74b6947a660}</Project>
<Name>SevenZipExtractor</Name>
</ProjectReference>
<ProjectReference Include="..\VirtualFileSystem\VirtualFileSystem.csproj">
<Project>{5128b489-bc28-4f66-9f0b-b4565af36cbc}</Project>
<Name>VirtualFileSystem</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
<Project>{b3f3fb6e-b9eb-4f49-9875-d78578bc7ae5}</Project>
<Name>Wabbajack.Common</Name>