Merge branch 'master' into patch-2

This commit is contained in:
erri120 2019-09-13 13:41:14 +00:00 committed by GitHub
commit 36203f9223
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 427 additions and 290 deletions

View File

@ -7,6 +7,13 @@
* Fixed a integer overflow resulting in a crash in very large BSA reading
* Fix a bug in BSA string encoding
* Add human friendly filesizes to the download header and file info sections in the Install Report
* Improve compilation times by caching BSDiff patches
* Detect when VFS root folders don't exist
* Only reauth against the Nexus every 3 days (instead of 12 hours)
* Optimize executable patching by switching to .NET serialization and LZ4 compression
* Ignore some files Wabbajack creates
* Improve compilation times by reworking file indexing algorithm
* Store patch files in byte format instead of base64 strings
#### Version 0.9.1 - 9/5/2019
* Fixed a bug where having only one profile selected would result in no profiles being selected

View File

@ -146,6 +146,7 @@ namespace Compression.BSA
{
_fileName = filename;
}
public BSAReader(Stream stream)

View File

@ -272,6 +272,7 @@ namespace VFS
/// <param name="path"></param>
public void AddRoot(string path)
{
if (!Directory.Exists(path)) return;
IndexPath(path);
RefreshIndexes();
}
@ -471,6 +472,11 @@ namespace VFS
string fullPath = archive.FullPath + "|" + String.Join("|", archiveHashPath.Skip(1));
return Lookup(fullPath);
}
public IDictionary<VirtualFile, IEnumerable<VirtualFile>> GroupedByArchive()
{
return _files.Values.GroupBy(f => f.TopLevelArchive).ToDictionary(f => f.Key, f => (IEnumerable<VirtualFile>)f);
}
}
public class StagingGroup : List<VirtualFile>, IDisposable
@ -615,7 +621,7 @@ namespace VFS
}
/// <summary>
/// Calulate the file's SHA, size and last modified
/// Calculate the file's SHA, size and last modified
/// </summary>
internal void Analyze()
{

View File

@ -62,6 +62,8 @@ namespace Wabbajack.Common
}
private static void ExtractAllWithBSA(string source, string dest)
{
try
{
using (var arch = new BSAReader(source))
{
@ -84,6 +86,12 @@ namespace Wabbajack.Common
});
}
}
catch (Exception ex)
{
Utils.Log($"While Extracting {source}");
throw ex;
}
}
private static void ExtractAllWith7Zip(string source, string dest)
{

View File

@ -312,7 +312,6 @@ namespace Wabbajack.Common
f(i);
return false;
});
return;
}
public static void DoProgress<T>(this IEnumerable<T> coll, String msg, Action<T> f)
@ -417,5 +416,44 @@ namespace Wabbajack.Common
return (Math.Sign(byteCount) * num).ToString() + Suffix[place];
}
public static void CreatePatch(byte[] a, byte[] b, Stream output)
{
var data_a = a.SHA256().FromBase64().ToHEX();
var data_b = b.SHA256().FromBase64().ToHEX();
var cache_file = Path.Combine("patch_cache", $"{data_a}_{data_b}.patch");
if (!Directory.Exists("patch_cache"))
Directory.CreateDirectory("patch_cache");
while (true)
{
if (File.Exists(cache_file))
{
using (var f = File.OpenRead(cache_file))
{
f.CopyTo(output);
}
}
else
{
var tmp_name = Path.Combine("patch_cache", Guid.NewGuid() + ".tmp");
using (var f = File.OpenWrite(tmp_name))
{
BSDiff.Create(a, b, f);
}
File.Move(tmp_name, cache_file);
continue;
}
break;
}
}
public static void TryGetPatch(string foundHash, string fileHash, out byte[] ePatch)
{
var patch_name = Path.Combine("patch_cache", $"{foundHash.FromBase64().ToHEX()}_{fileHash.FromBase64().ToHEX()}.patch");
ePatch = File.Exists(patch_name) ? File.ReadAllBytes(patch_name) : null;
}
}
}

View File

@ -68,10 +68,16 @@
<Reference Include="Newtonsoft.Json.Bson, Version=1.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.Bson.1.0.2\lib\net45\Newtonsoft.Json.Bson.dll</HintPath>
</Reference>
<Reference Include="protobuf-net, Version=2.4.0.0, Culture=neutral, PublicKeyToken=257b51d87d2e4d67, processorArchitecture=MSIL">
<HintPath>..\packages\protobuf-net.2.4.0\lib\net40\protobuf-net.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Configuration" />
<Reference Include="System.Core" />
<Reference Include="System.IO.Compression" />
<Reference Include="System.Numerics" />
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.ServiceModel" />
<Reference Include="System.Transactions" />
<Reference Include="System.Xml.Linq" />
<Reference Include="Microsoft.CSharp" />

View File

@ -5,6 +5,7 @@
<package id="murmurhash" version="1.0.3" targetFramework="net472" />
<package id="Newtonsoft.Json" version="12.0.2" targetFramework="net472" />
<package id="Newtonsoft.Json.Bson" version="1.0.2" targetFramework="net472" />
<package id="protobuf-net" version="2.4.0" targetFramework="net472" />
<package id="SharpZipLib" version="1.1.0" targetFramework="net472" />
<package id="System.Runtime.Numerics" version="4.3.0" targetFramework="net472" />
</packages>

View File

@ -215,9 +215,9 @@ namespace Wabbajack
}
}
internal void ConfigureForInstall(string modlist)
internal void ConfigureForInstall(ModList modlist)
{
_modList = modlist.FromJSONString<ModList>();
_modList = modlist;
Mode = "Installing";
ModListName = _modList.Name;
HTMLReport = _modList.ReportHTML;

View File

@ -1,38 +1,53 @@
using Compression.BSA;
using Newtonsoft.Json;
using System;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Security.Cryptography;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
using System.Text.RegularExpressions;
using System.Web;
using System.Windows.Documents;
using CommonMark;
using Compression.BSA;
using ICSharpCode.SharpZipLib.BZip2;
using K4os.Compression.LZ4;
using K4os.Compression.LZ4.Streams;
using Microsoft.SqlServer.Server;
using Newtonsoft.Json;
using SharpCompress.Compressors.BZip2;
using VFS;
using Wabbajack.Common;
using static Wabbajack.NexusAPI;
using VFS;
namespace Wabbajack
{
public class Compiler
{
private string _mo2DownloadsFolder;
public Dictionary<string, IEnumerable<IndexedFileMatch>> DirectMatchIndex;
public string MO2Folder;
public string MO2Profile;
public Compiler(string mo2_folder, Action<string> log_fn)
{
MO2Folder = mo2_folder;
Log_Fn = log_fn;
MO2Ini = Path.Combine(MO2Folder, "ModOrganizer.ini").LoadIniFile();
GamePath = ((string) MO2Ini.General.gamePath).Replace("\\\\", "\\");
}
public dynamic MO2Ini { get; }
public string GamePath { get; }
public bool IgnoreMissingFiles { get; set; }
private string _mo2DownloadsFolder;
public string MO2DownloadsFolder
{
get
@ -47,17 +62,7 @@ namespace Wabbajack
set => _mo2DownloadsFolder = value;
}
public string MO2Profile;
public string MO2ProfileDir
{
get
{
return Path.Combine(MO2Folder, "profiles", MO2Profile);
}
}
public string MO2ProfileDir => Path.Combine(MO2Folder, "profiles", MO2Profile);
public Action<string> Log_Fn { get; }
public List<Directive> InstallDirectives { get; private set; }
@ -69,38 +74,24 @@ namespace Wabbajack
public ConcurrentBag<Directive> ExtraFiles { get; private set; }
public Dictionary<string, dynamic> ModInis { get; private set; }
public VirtualFileSystem VFS
{
get
{
return VirtualFileSystem.VFS;
}
}
public VirtualFileSystem VFS => VirtualFileSystem.VFS;
public List<IndexedArchive> IndexedArchives { get; private set; }
public Dictionary<string, IEnumerable<VirtualFile>> IndexedFiles { get; private set; }
public class IndexedFileMatch
{
public IndexedArchive Archive;
public IndexedArchiveEntry Entry;
public DateTime LastModified;
}
public Dictionary<string, IEnumerable<IndexedFileMatch>> DirectMatchIndex;
public HashSet<string> SelectedProfiles { get; set; } = new HashSet<string>();
public void Info(string msg, params object[] args)
{
if (args.Length > 0)
msg = String.Format(msg, args);
msg = string.Format(msg, args);
Log_Fn(msg);
}
public void Status(string msg, params object[] args)
{
if (args.Length > 0)
msg = String.Format(msg, args);
msg = string.Format(msg, args);
WorkQueue.Report(msg, 0);
}
@ -108,58 +99,20 @@ namespace Wabbajack
private void Error(string msg, params object[] args)
{
if (args.Length > 0)
msg = String.Format(msg, args);
msg = string.Format(msg, args);
Log_Fn(msg);
throw new Exception(msg);
}
public Compiler(string mo2_folder, Action<string> log_fn)
{
MO2Folder = mo2_folder;
Log_Fn = log_fn;
MO2Ini = Path.Combine(MO2Folder, "ModOrganizer.ini").LoadIniFile();
GamePath = ((string)MO2Ini.General.gamePath).Replace("\\\\", "\\");
}
public HashSet<string> SelectedProfiles { get; set; } = new HashSet<string>();
private IndexedArchive LoadArchive(string file)
{
var info = new IndexedArchive();
info.File = VFS.Lookup(file);
info.Name = Path.GetFileName(file);
var ini_name = file + ".meta";
if (ini_name.FileExists())
{
info.IniData = ini_name.LoadIniFile();
info.Meta = File.ReadAllText(ini_name);
}
return info;
}
private bool IsArchiveFile(string name)
{
var ext = Path.GetExtension(name);
if (ext == ".bsa" || Consts.SupportedArchives.Contains(ext))
return true;
return false;
}
public void Compile()
{
Info($"Looking for other profiles");
Info("Looking for other profiles");
var other_profiles_path = Path.Combine(MO2ProfileDir, "otherprofiles.txt");
SelectedProfiles = new HashSet<string>();
if (File.Exists(other_profiles_path))
{
SelectedProfiles = File.ReadAllLines(other_profiles_path).ToHashSet();
}
if (File.Exists(other_profiles_path)) SelectedProfiles = File.ReadAllLines(other_profiles_path).ToHashSet();
SelectedProfiles.Add(MO2Profile);
Info($"Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p)));
Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p)));
Info($"Indexing {MO2Folder}");
VFS.AddRoot(MO2Folder);
@ -175,22 +128,25 @@ namespace Wabbajack
var game_files = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Lookup(p)) { Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath))});
.Select(p => new RawSourceFile(VFS.Lookup(p))
{Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath))});
var loot_path = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "LOOT");
var loot_path = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
"LOOT");
Info($"Indexing {loot_path}");
VFS.AddRoot(loot_path);
var loot_files = Directory.EnumerateFiles(loot_path, "userlist.yaml", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Lookup(p)) { Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(loot_path)) });
.Select(p => new RawSourceFile(VFS.Lookup(p))
{Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(loot_path))});
Info($"Indexing Archives");
Info("Indexing Archives");
IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder)
.Where(f => File.Exists(f + ".meta"))
.Select(f => new IndexedArchive()
.Select(f => new IndexedArchive
{
File = VFS.Lookup(f),
Name = Path.GetFileName(f),
@ -199,10 +155,14 @@ namespace Wabbajack
})
.ToList();
Info($"Indexing Files");
IndexedFiles = IndexedArchives.PMap(f => { Status($"Finding files in {Path.GetFileName(f.File.FullPath)}");
return VFS.FilesInArchive(f.File); })
Info("Indexing Files");
var grouped = VFS.GroupedByArchive();
IndexedFiles = IndexedArchives.Select(f =>
{
if (grouped.TryGetValue(f.File, out var result))
return result;
return new List<VirtualFile>();
})
.SelectMany(fs => fs)
.Concat(IndexedArchives.Select(f => f.File))
.OrderByDescending(f => f.TopLevelArchive.LastModified)
@ -251,7 +211,8 @@ namespace Wabbajack
{
Info("Continuing even though files were missing at the request of the user.");
}
else {
else
{
Info("Exiting due to no way to compile these files");
return;
}
@ -261,19 +222,16 @@ namespace Wabbajack
Info("Getting nexus api_key please click authorize if a browser window appears");
NexusKey = NexusAPI.GetNexusAPIKey();
User = NexusAPI.GetUserStatus(NexusKey);
NexusKey = GetNexusAPIKey();
User = GetUserStatus(NexusKey);
if (!User.is_premium)
{
Info($"User {User.name} is not a premium Nexus user, cannot continue");
}
if (!User.is_premium) Info($"User {User.name} is not a premium Nexus user, cannot continue");
GatherArchives();
BuildPatches();
ModList = new ModList()
ModList = new ModList
{
Archives = SelectedArchives,
Directives = InstallDirectives,
@ -289,6 +247,7 @@ namespace Wabbajack
Info("Done Building Modpack");
}
private void ShowReport()
{
var file = Path.GetTempFileName() + ".html";
@ -302,8 +261,11 @@ namespace Wabbajack
{
fs.SetLength(0);
using (var reporter = new ReportBuilder(fs))
{
reporter.Build(ModList);
}
}
ModList.ReportHTML = CommonMarkConverter.Convert(File.ReadAllText($"{ModList.Name}.md"));
}
@ -324,7 +286,9 @@ namespace Wabbajack
/// </summary>
private void BuildPatches()
{
Info("Gathering patch files");
var groups = InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => p.Patch == null)
.GroupBy(p => p.ArchiveHashPath[0])
.ToList();
@ -333,18 +297,17 @@ namespace Wabbajack
groups.PMap(group => BuildArchivePatches(group.Key, group, absolute_paths));
if (InstallDirectives.OfType<PatchedFromArchive>().FirstOrDefault(f => f.Patch == null) != null)
{
Error("Missing patches after generation, this should not happen");
}
}
private void BuildArchivePatches(string archive_sha, IEnumerable<PatchedFromArchive> group, Dictionary<string, string> absolute_paths)
private void BuildArchivePatches(string archive_sha, IEnumerable<PatchedFromArchive> group,
Dictionary<string, string> absolute_paths)
{
var archive = VFS.HashIndex[archive_sha];
using (var files = VFS.StageWith(group.Select(g => VFS.FileForArchiveHashPath(g.ArchiveHashPath))))
{
var by_path = files.GroupBy(f => string.Join("|", f.Paths.Skip(1))).ToDictionary(f => f.Key, f => f.First());
var by_path = files.GroupBy(f => string.Join("|", f.Paths.Skip(1)))
.ToDictionary(f => f.Key, f => f.First());
// Now Create the patches
group.PMap(entry =>
{
@ -354,13 +317,12 @@ namespace Wabbajack
{
var a = origin.ReadAll();
var b = LoadDataForTo(entry.To, absolute_paths);
BSDiff.Create(a, b, output);
entry.Patch = output.ToArray().ToBase64();
Utils.CreatePatch(a, b, output);
entry.Patch = output.ToArray();
Info($"Patch size {entry.Patch.Length} for {entry.To}");
}
});
}
}
private byte[] LoadDataForTo(string to, Dictionary<string, string> absolute_paths)
@ -378,15 +340,15 @@ namespace Wabbajack
var file = a.Files.First(e => e.Path == Path.Combine(to.Split('\\').Skip(2).ToArray()));
return file.GetData();
}
}
Error($"Couldn't load data for {to}");
return null;
}
private void GatherArchives()
{
Info($"Building a list of archives based on the files required");
Info("Building a list of archives based on the files required");
var shas = InstallDirectives.OfType<FromArchive>()
.Select(a => a.ArchiveHashPath[0])
@ -397,7 +359,6 @@ namespace Wabbajack
.ToDictionary(f => f.Key, f => f.First());
SelectedArchives = shas.PMap(sha => ResolveArchive(sha, archives));
}
private Archive ResolveArchive(string sha, IDictionary<string, IndexedArchive> archives)
@ -405,10 +366,14 @@ namespace Wabbajack
if (archives.TryGetValue(sha, out var found))
{
if (found.IniData == null)
Error("No download metadata found for {0}, please use MO2 to query info or add a .meta file and try again.", found.Name);
Error(
"No download metadata found for {0}, please use MO2 to query info or add a .meta file and try again.",
found.Name);
var general = found.IniData.General;
if (general == null)
Error("No General section in mod metadata found for {0}, please use MO2 to query info or add the info and try again.", found.Name);
Error(
"No General section in mod metadata found for {0}, please use MO2 to query info or add the info and try again.",
found.Name);
Archive result;
@ -416,14 +381,14 @@ namespace Wabbajack
{
var regex = new Regex("((?<=id=)[a-zA-Z0-9_-]*)|(?<=\\/file\\/d\\/)[a-zA-Z0-9_-]*");
var match = regex.Match(general.directURL);
result = new GoogleDriveMod()
result = new GoogleDriveMod
{
Id = match.ToString()
};
}
else if (general.directURL != null && general.directURL.StartsWith(Consts.MegaPrefix))
{
result = new MEGAArchive()
result = new MEGAArchive
{
URL = general.directURL
};
@ -440,14 +405,15 @@ namespace Wabbajack
uri.Query = query.ToString();
result = new DirectURLArchive()
result = new DirectURLArchive
{
URL = uri.ToString()
};
}
else if (general.directURL != null && general.directURL.StartsWith("https://www.moddb.com/downloads/start"))
else if (general.directURL != null &&
general.directURL.StartsWith("https://www.moddb.com/downloads/start"))
{
result = new MODDBArchive()
result = new MODDBArchive
{
URL = general.directURL
};
@ -463,8 +429,7 @@ namespace Wabbajack
}
else if (general.directURL != null)
{
var tmp = new DirectURLArchive()
var tmp = new DirectURLArchive
{
URL = general.directURL
};
@ -473,25 +438,26 @@ namespace Wabbajack
tmp.Headers = new List<string>();
tmp.Headers.AddRange(general.directURLHeaders.Split('|'));
}
result = tmp;
}
else if (general.manualURL != null)
{
result = new ManualURLArchive()
result = new ManualURLArchive
{
URL = general.manualURL.ToString()
};
}
else if (general.modID != null && general.fileID != null && general.gameName != null)
{
var nm = new NexusMod()
var nm = new NexusMod
{
GameName = general.gameName,
FileID = general.fileID,
ModID = general.modID,
Version = general.version ?? "0.0.0.0"
};
var info = NexusAPI.GetModInfo(nm, NexusKey);
var info = GetModInfo(nm, NexusKey);
nm.Author = info.author;
nm.UploadedBy = info.uploaded_by;
nm.UploaderProfile = info.uploaded_users_profile_url;
@ -513,10 +479,12 @@ namespace Wabbajack
var installer = new Installer(null, "", Utils.Log);
installer.NexusAPIKey = NexusKey;
if (!installer.DownloadArchive(result, false))
Error($"Unable to resolve link for {found.Name}. If this is hosted on the nexus the file may have been removed.");
Error(
$"Unable to resolve link for {found.Name}. If this is hosted on the nexus the file may have been removed.");
return result;
}
Error("No match found for Archive sha: {0} this shouldn't happen", sha);
return null;
}
@ -530,6 +498,7 @@ namespace Wabbajack
var result = f(source);
if (result != null) return result;
}
throw new InvalidDataException("Data fell out of the compilation stack");
}
@ -586,6 +555,8 @@ namespace Wabbajack
// Theme file MO2 downloads somehow
IgnoreEndsWith("splash.png"),
IgnoreWabbajackInstallCruft(),
PatchStockESMs(),
IncludeAllConfigs(),
@ -594,6 +565,18 @@ namespace Wabbajack
};
}
private Func<RawSourceFile, Directive> IgnoreWabbajackInstallCruft()
{
var cruft_files = new HashSet<string> {"7z.dll", "7z.exe", "vfs_staged_files\\", "nexus.key_cache", "patch_cache\\", Consts.NexusCacheDirectory + "\\"};
return source =>
{
if (!cruft_files.Any(f => source.Path.StartsWith(f))) return null;
var result = source.EvolveTo<IgnoredDirectly>();
result.Reason = "Wabbajack Cruft file";
return result;
};
}
private Func<RawSourceFile, Directive> IncludeAllConfigs()
{
return source =>
@ -609,41 +592,44 @@ namespace Wabbajack
{
return source =>
{
string filename = Path.GetFileName(source.Path);
string game_file = Path.Combine(GamePath, "Data", filename);
var filename = Path.GetFileName(source.Path);
var game_file = Path.Combine(GamePath, "Data", filename);
if (Consts.GameESMs.Contains(filename) && source.Path.StartsWith("mods\\") && File.Exists(game_file))
{
Info($"A ESM named {filename} was found in a mod that shares a name with a core game ESMs, it is assumed this is a cleaned ESM and it will be binary patched.");
Info(
$"A ESM named {filename} was found in a mod that shares a name with a core game ESMs, it is assumed this is a cleaned ESM and it will be binary patched.");
var result = source.EvolveTo<CleanedESM>();
result.SourceESMHash = VFS.Lookup(game_file).Hash;
Status($"Generating patch of {filename}");
using (var ms = new MemoryStream()) {
BSDiff.Create(File.ReadAllBytes(game_file), File.ReadAllBytes(source.AbsolutePath), ms);
using (var ms = new MemoryStream())
{
Utils.CreatePatch(File.ReadAllBytes(game_file), File.ReadAllBytes(source.AbsolutePath), ms);
result.SourceData = ms.ToArray().ToBase64();
}
Info($"Generated a {result.SourceData.Length} byte patch for {filename}");
return result;
}
return null;
};
}
private Func<RawSourceFile, Directive> IncludeLootFiles()
{
var prefix = Consts.LOOTFolderFilesDir + "\\";
return source =>
{
if (source.Path.StartsWith(prefix))
{
var result = source.EvolveTo<InlineFile>();
result.SourceData = File.ReadAllBytes(source.AbsolutePath).ToBase64();
return result;
}
return null;
};
}
@ -653,9 +639,7 @@ namespace Wabbajack
return source =>
{
if (Consts.ConfigFileExtensions.Contains(Path.GetExtension(source.Path)))
{
return RemapFile(source, GamePath);
}
return null;
};
}
@ -682,7 +666,6 @@ namespace Wabbajack
var result = source.EvolveTo<RemappedInlineFile>();
result.SourceData = Encoding.UTF8.GetBytes(data).ToBase64();
return result;
}
private Func<RawSourceFile, Directive> IgnorePathContains(string v)
@ -697,6 +680,7 @@ namespace Wabbajack
result.Reason = reason;
return result;
}
return null;
};
}
@ -709,7 +693,8 @@ namespace Wabbajack
/// <returns></returns>
private Func<RawSourceFile, Directive> IncludeTaggedFiles()
{
var include_directly = ModInis.Where(kv => {
var include_directly = ModInis.Where(kv =>
{
var general = kv.Value.General;
if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE))
return true;
@ -721,19 +706,15 @@ namespace Wabbajack
return source =>
{
if (source.Path.StartsWith("mods"))
{
foreach (var modpath in include_directly)
{
if (source.Path.StartsWith(modpath))
{
var result = source.EvolveTo<InlineFile>();
result.SourceData = File.ReadAllBytes(source.AbsolutePath).ToBase64();
return result;
}
}
}
return null;
};
}
@ -752,8 +733,10 @@ namespace Wabbajack
{
if (Path.GetExtension(source.AbsolutePath) == ".esp")
{
var bsa = Path.Combine(Path.GetDirectoryName(source.AbsolutePath), Path.GetFileNameWithoutExtension(source.AbsolutePath) + ".bsa");
var bsa_textures = Path.Combine(Path.GetDirectoryName(source.AbsolutePath), Path.GetFileNameWithoutExtension(source.AbsolutePath) + " - Textures.bsa");
var bsa = Path.Combine(Path.GetDirectoryName(source.AbsolutePath),
Path.GetFileNameWithoutExtension(source.AbsolutePath) + ".bsa");
var bsa_textures = Path.Combine(Path.GetDirectoryName(source.AbsolutePath),
Path.GetFileNameWithoutExtension(source.AbsolutePath) + " - Textures.bsa");
var esp_size = new FileInfo(source.AbsolutePath).Length;
if (esp_size <= 100 && (File.Exists(bsa) || File.Exists(bsa_textures)))
{
@ -776,7 +759,8 @@ namespace Wabbajack
/// <returns></returns>
private Func<RawSourceFile, Directive> DeconstructBSAs()
{
var include_directly = ModInis.Where(kv => {
var include_directly = ModInis.Where(kv =>
{
var general = kv.Value.General;
if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE))
return true;
@ -785,14 +769,14 @@ namespace Wabbajack
return false;
}).Select(kv => $"mods\\{kv.Key}\\");
var microstack = new List<Func<RawSourceFile, Directive>>()
var microstack = new List<Func<RawSourceFile, Directive>>
{
DirectMatch(),
IncludePatches(),
DropAll()
};
var microstack_with_include = new List<Func<RawSourceFile, Directive>>()
var microstack_with_include = new List<Func<RawSourceFile, Directive>>
{
DirectMatch(),
IncludePatches(),
@ -804,18 +788,14 @@ namespace Wabbajack
{
if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path))) return null;
bool default_include = false;
var default_include = false;
if (source.Path.StartsWith("mods"))
{
foreach (var modpath in include_directly)
{
if (source.Path.StartsWith(modpath))
{
default_include = true;
break;
}
}
}
var source_files = source.File.FileInArchive;
@ -826,34 +806,33 @@ namespace Wabbajack
var matches = source_files.PMap(e => RunStack(stack, new RawSourceFile(e)
{
Path = Path.Combine(Consts.BSACreationDir, id, e.Paths.Last())
}));
foreach (var match in matches)
{
if (match is IgnoredDirectly)
{
Error($"File required for BSA creation doesn't exist: {match.To}");
}
if (match is IgnoredDirectly) Error($"File required for BSA creation doesn't exist: {match.To}");
ExtraFiles.Add(match);
};
}
;
CreateBSA directive;
using (var bsa = new BSAReader(source.AbsolutePath))
{
directive = new CreateBSA()
directive = new CreateBSA
{
To = source.Path,
TempID = id,
Type = (uint) bsa.HeaderType,
FileFlags = (uint) bsa.FileFlags,
ArchiveFlags = (uint)bsa.ArchiveFlags,
};
ArchiveFlags = (uint) bsa.ArchiveFlags
};
}
;
return directive;
};
}
@ -881,7 +860,8 @@ namespace Wabbajack
return source =>
{
if (!source.Path.StartsWith("mods") || all_enabled_mods.Any(mod => source.Path.StartsWith(mod))) return null;
if (!source.Path.StartsWith("mods") || all_enabled_mods.Any(mod => source.Path.StartsWith(mod)))
return null;
var r = source.EvolveTo<IgnoredDirectly>();
r.Reason = "Disabled Mod";
return r;
@ -901,6 +881,11 @@ namespace Wabbajack
return false;
}
/// <summary>
/// This matches files based purely on filename, and then creates a binary patch.
/// In practice this is fine, because a single file tends to only come from one archive.
/// </summary>
/// <returns></returns>
private Func<RawSourceFile, Directive> IncludePatches()
{
var indexed = IndexedFiles.Values
@ -910,17 +895,17 @@ namespace Wabbajack
return source =>
{
if (indexed.TryGetValue(Path.GetFileName(source.File.Paths.Last().ToLower()), out var value))
{
// TODO: Improve this
var found = value.First();
if (!indexed.TryGetValue(Path.GetFileName(source.File.Paths.Last().ToLower()), out var value))
return null;
var found = value.OrderByDescending(f => (f.TopLevelArchive ?? f).LastModified).First();
var e = source.EvolveTo<PatchedFromArchive>();
e.ArchiveHashPath = found.MakeRelativePaths();
e.To = source.Path;
Utils.TryGetPatch(found.Hash, source.File.Hash, out e.Patch);
return e;
}
return null;
};
}
@ -934,6 +919,7 @@ namespace Wabbajack
e.SourceData = File.ReadAllBytes(source.AbsolutePath).ToBase64();
return e;
}
return null;
};
}
@ -949,6 +935,7 @@ namespace Wabbajack
i.Reason = "Default game file";
return i;
}
return null;
};
}
@ -971,6 +958,7 @@ namespace Wabbajack
e.SourceData = data.ToBase64();
return e;
}
return null;
};
}
@ -981,7 +969,7 @@ namespace Wabbajack
lines = (from line in lines
where !(line.StartsWith("-") && !line.EndsWith("_separator"))
select line).ToArray();
return Encoding.UTF8.GetBytes(String.Join("\r\n", lines));
return Encoding.UTF8.GetBytes(string.Join("\r\n", lines));
}
private Func<RawSourceFile, Directive> IgnoreOtherProfiles()
@ -994,21 +982,19 @@ namespace Wabbajack
{
if (source.Path.StartsWith("profiles\\"))
{
if (profiles.Any(profile => source.Path.StartsWith(profile)))
{
return null;
}
if (profiles.Any(profile => source.Path.StartsWith(profile))) return null;
var c = source.EvolveTo<IgnoredDirectly>();
c.Reason = "File not for selected profiles";
return c;
}
return null;
};
}
private Func<RawSourceFile, Directive> IgnoreEndsWith(string v)
{
var reason = String.Format("Ignored because path ends with {0}", v);
var reason = string.Format("Ignored because path ends with {0}", v);
return source =>
{
if (source.Path.EndsWith(v))
@ -1017,13 +1003,14 @@ namespace Wabbajack
result.Reason = reason;
return result;
}
return null;
};
}
private Func<RawSourceFile, Directive> IgnoreRegex(string p)
{
var reason = String.Format("Ignored because path matches regex {0}", p);
var reason = string.Format("Ignored because path matches regex {0}", p);
var regex = new Regex(p);
return source =>
{
@ -1033,6 +1020,7 @@ namespace Wabbajack
result.Reason = reason;
return result;
}
return null;
};
}
@ -1049,6 +1037,7 @@ namespace Wabbajack
result.SourceData = File.ReadAllBytes(source.AbsolutePath).ToBase64();
return result;
}
return null;
};
}
@ -1056,7 +1045,8 @@ namespace Wabbajack
private Func<RawSourceFile, Directive> DropAll()
{
return source => {
return source =>
{
var result = source.EvolveTo<NoMatch>();
result.Reason = "No Match in Stack";
Info($"No match for: {source.Path}");
@ -1066,14 +1056,14 @@ namespace Wabbajack
private Func<RawSourceFile, Directive> DirectMatch()
{
return source =>
{
if (IndexedFiles.TryGetValue(source.Hash, out var found))
{
var result = source.EvolveTo<FromArchive>();
var match = found.Where(f => Path.GetFileName(f.Paths[f.Paths.Length - 1]) == Path.GetFileName(source.Path))
var match = found.Where(f =>
Path.GetFileName(f.Paths[f.Paths.Length - 1]) == Path.GetFileName(source.Path))
.OrderBy(f => f.Paths.Length)
.FirstOrDefault();
@ -1084,13 +1074,14 @@ namespace Wabbajack
return result;
}
return null;
};
}
private Func<RawSourceFile, Directive> IgnoreStartsWith(string v)
{
var reason = String.Format("Ignored because path starts with {0}", v);
var reason = string.Format("Ignored because path starts with {0}", v);
return source =>
{
if (source.Path.StartsWith(v))
@ -1099,6 +1090,7 @@ namespace Wabbajack
result.Reason = reason;
return result;
}
return null;
};
}
@ -1106,8 +1098,7 @@ namespace Wabbajack
internal void PatchExecutable()
{
Utils.Log("Exporting Installer");
var settings = new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto };
var data = JsonConvert.SerializeObject(ModList, settings).BZip2String();
var settings = new JsonSerializerSettings {TypeNameHandling = TypeNameHandling.Auto};
var executable = Assembly.GetExecutingAssembly().Location;
var out_path = Path.Combine(Path.GetDirectoryName(executable), MO2Profile + ".exe");
Info("Patching Executable {0}", Path.GetFileName(out_path));
@ -1115,13 +1106,32 @@ namespace Wabbajack
using (var os = File.OpenWrite(out_path))
using (var bw = new BinaryWriter(os))
{
long orig_pos = os.Length;
var orig_pos = os.Length;
os.Position = os.Length;
bw.Write(data.LongLength);
bw.Write(data);
//using (var compressor = new BZip2OutputStream(bw.BaseStream))
/*using (var sw = new StreamWriter(compressor))
using (var writer = new JsonTextWriter(sw))
{
var serializer = new JsonSerializer();
serializer.TypeNameHandling = TypeNameHandling.Auto;
serializer.Serialize(writer, ModList);
}*/
//bw.Write(data);
var formatter = new BinaryFormatter();
using (var compressed = LZ4Stream.Encode(bw.BaseStream, new LZ4EncoderSettings { CompressionLevel = LZ4Level.L10_OPT }, true))
formatter.Serialize(compressed, ModList);
bw.Write(orig_pos);
bw.Write(Encoding.ASCII.GetBytes(Consts.ModListMagic));
}
}
public class IndexedFileMatch
{
public IndexedArchive Archive;
public IndexedArchiveEntry Entry;
public DateTime LastModified;
}
}
}

View File

@ -1,9 +1,6 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using VFS;
namespace Wabbajack
@ -44,20 +41,13 @@ namespace Wabbajack
}
}
[Serializable]
public class ModList
{
/// <summary>
/// Name of the ModList
/// </summary>
public string Name;
/// <summary>
/// Author of the Mod List
/// </summary>
public string Author;
/// <summary>
/// Version of this Mod List
/// </summary>
public string Version;
/// <summary>
/// Install directives
@ -75,6 +65,7 @@ namespace Wabbajack
public string ReportHTML;
}
[Serializable]
public class Directive
{
/// <summary>
@ -83,16 +74,19 @@ namespace Wabbajack
public string To;
}
[Serializable]
public class IgnoredDirectly : Directive
{
public string Reason;
}
[Serializable]
public class NoMatch : IgnoredDirectly
{
}
[Serializable]
public class InlineFile : Directive
{
/// <summary>
@ -101,6 +95,7 @@ namespace Wabbajack
public string SourceData;
}
[Serializable]
public class CleanedESM : InlineFile
{
public string SourceESMHash;
@ -109,10 +104,12 @@ namespace Wabbajack
/// <summary>
/// A file that has the game and MO2 folders remapped on installation
/// </summary>
[Serializable]
public class RemappedInlineFile : InlineFile
{
}
[Serializable]
public class FromArchive : Directive
{
/// <summary>
@ -121,9 +118,11 @@ namespace Wabbajack
public string[] ArchiveHashPath;
[JsonIgnore]
[NonSerialized]
public VirtualFile FromFile;
private string _fullPath = null;
[JsonIgnore]
public string FullPath
{
@ -137,6 +136,7 @@ namespace Wabbajack
}
}
[Serializable]
public class CreateBSA : Directive
{
public string TempID;
@ -150,14 +150,16 @@ namespace Wabbajack
public uint ArchiveFlags { get; set; }
}
[Serializable]
public class PatchedFromArchive : FromArchive
{
/// <summary>
/// The file to apply to the source file to patch it
/// </summary>
public string Patch;
public byte[] Patch;
}
[Serializable]
public class Archive
{
/// <summary>
@ -176,6 +178,7 @@ namespace Wabbajack
public long Size;
}
[Serializable]
public class NexusMod : Archive
{
public string GameName;
@ -187,6 +190,7 @@ namespace Wabbajack
public string Author;
}
[Serializable]
public class GoogleDriveMod : Archive
{
public string Id;
@ -195,6 +199,7 @@ namespace Wabbajack
/// <summary>
/// URL that can be downloaded directly without any additional options
/// </summary>
[Serializable]
public class DirectURLArchive : Archive
{
public string URL;
@ -206,6 +211,7 @@ namespace Wabbajack
/// <summary>
/// A URL that cannot be downloaded automatically and has to be downloaded by hand
/// </summary>
[Serializable]
public class ManualURLArchive : Archive
{
public string URL;
@ -214,6 +220,7 @@ namespace Wabbajack
/// <summary>
/// An archive that requires additional HTTP headers.
/// </summary>
[Serializable]
public class DirectURLArchiveEx : DirectURLArchive
{
public Dictionary<string, string> Headers;
@ -222,6 +229,7 @@ namespace Wabbajack
/// <summary>
/// Archive that comes from MEGA
/// </summary>
[Serializable]
public class MEGAArchive : DirectURLArchive
{
}
@ -229,6 +237,7 @@ namespace Wabbajack
/// <summary>
/// Archive that comes from MODDB
/// </summary>
[Serializable]
public class MODDBArchive : DirectURLArchive
{
}
@ -236,10 +245,12 @@ namespace Wabbajack
/// <summary>
/// Archive that comes from MediaFire
/// </summary>
[Serializable]
public class MediaFireArchive : DirectURLArchive
{
}
[Serializable]
public class IndexedArchive
{
public dynamic IniData;
@ -251,6 +262,7 @@ namespace Wabbajack
/// <summary>
/// A archive entry
/// </summary>
[Serializable]
public class IndexedEntry
{
/// <summary>
@ -267,6 +279,7 @@ namespace Wabbajack
public long Size;
}
[Serializable]
public class IndexedArchiveEntry : IndexedEntry
{
public string[] HashPath;
@ -275,6 +288,7 @@ namespace Wabbajack
/// <summary>
/// Data found inside a BSA file in an archive
/// </summary>
[Serializable]
public class BSAIndexedEntry : IndexedEntry
{
/// <summary>

View File

@ -7,10 +7,14 @@ using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Runtime.Serialization;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Windows;
using K4os.Compression.LZ4.Encoders;
using K4os.Compression.LZ4.Streams;
using VFS;
using Wabbajack.Common;
@ -394,7 +398,7 @@ namespace Wabbajack
Status("Patching {0}", Path.GetFileName(to_patch.To));
// Read in the patch data
var patch_data = to_patch.Patch.FromBase64();
var patch_data = to_patch.Patch;
var to_file = Path.Combine(Outputfolder, to_patch.To);
MemoryStream old_data = new MemoryStream(File.ReadAllBytes(to_file));
@ -630,7 +634,7 @@ namespace Wabbajack
return HashArchive(e);
}
public static string CheckForModList()
public static ModList CheckForModList()
{
Utils.Log("Looking for attached modlist");
using (var s = File.OpenRead(Assembly.GetExecutingAssembly().Location))
@ -650,13 +654,14 @@ namespace Wabbajack
s.Position = s.Length - magic_bytes.Length - 8;
var start_pos = br.ReadInt64();
s.Position = start_pos;
long length = br.ReadInt64();
Utils.Log("Modlist found, loading...");
var list = br.ReadBytes((int)length).BZip2String();
using (var dc = LZ4Stream.Decode(br.BaseStream, leaveOpen: true))
{
IFormatter formatter = new BinaryFormatter();
var list = formatter.Deserialize(dc);
Utils.Log("Modlist loaded.");
return list;
return (ModList)list;
}
}
}
}

View File

@ -68,7 +68,7 @@ namespace Wabbajack
var modlist = compiler.ModList.ToJSON();
compiler = null;
context.ConfigureForInstall(modlist);
//context.ConfigureForInstall(modlist);
}).Start();
}

View File

@ -19,7 +19,7 @@ namespace Wabbajack
public static string GetNexusAPIKey()
{
FileInfo fi = new FileInfo("nexus.key_cache");
if (fi.Exists && fi.LastWriteTime > DateTime.Now.AddHours(-12))
if (fi.Exists && fi.LastWriteTime > DateTime.Now.AddHours(-72))
{
return File.ReadAllText("nexus.key_cache");
}

View File

@ -102,12 +102,27 @@
<Reference Include="Costura, Version=4.0.0.0, Culture=neutral, PublicKeyToken=9919ef960d84173d, processorArchitecture=MSIL">
<HintPath>..\packages\Costura.Fody.4.0.0\lib\net40\Costura.dll</HintPath>
</Reference>
<Reference Include="ICSharpCode.SharpZipLib, Version=1.2.0.246, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
<HintPath>..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll</HintPath>
</Reference>
<Reference Include="K4os.Compression.LZ4, Version=1.1.11.0, Culture=neutral, PublicKeyToken=2186fa9121ef231d, processorArchitecture=MSIL">
<HintPath>..\packages\K4os.Compression.LZ4.1.1.11\lib\net46\K4os.Compression.LZ4.dll</HintPath>
</Reference>
<Reference Include="K4os.Compression.LZ4.Streams, Version=1.1.11.0, Culture=neutral, PublicKeyToken=2186fa9121ef231d, processorArchitecture=MSIL">
<HintPath>..\packages\K4os.Compression.LZ4.Streams.1.1.11\lib\net46\K4os.Compression.LZ4.Streams.dll</HintPath>
</Reference>
<Reference Include="K4os.Hash.xxHash, Version=1.0.6.0, Culture=neutral, PublicKeyToken=32cd54395057cec3, processorArchitecture=MSIL">
<HintPath>..\packages\K4os.Hash.xxHash.1.0.6\lib\net46\K4os.Hash.xxHash.dll</HintPath>
</Reference>
<Reference Include="MegaApiClient, Version=1.7.1.495, Culture=neutral, PublicKeyToken=0480d311efbeb4e2, processorArchitecture=MSIL">
<HintPath>..\packages\MegaApiClient.1.7.1\lib\net46\MegaApiClient.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json.Bson, Version=1.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.Bson.1.0.2\lib\net45\Newtonsoft.Json.Bson.dll</HintPath>
</Reference>
<Reference Include="Ookii.Dialogs.Wpf, Version=1.0.0.0, Culture=neutral, PublicKeyToken=66aa232afad40158, processorArchitecture=MSIL">
<HintPath>..\packages\Ookii.Dialogs.Wpf.1.1.0\lib\net45\Ookii.Dialogs.Wpf.dll</HintPath>
</Reference>
@ -115,10 +130,26 @@
<HintPath>..\packages\SharpCompress.0.23.0\lib\net45\SharpCompress.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Buffers, Version=4.0.2.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\System.Buffers.4.4.0\lib\netstandard2.0\System.Buffers.dll</HintPath>
</Reference>
<Reference Include="System.Configuration" />
<Reference Include="System.Data" />
<Reference Include="System.Design" />
<Reference Include="System.Drawing" />
<Reference Include="System.Memory, Version=4.0.1.1, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\System.Memory.4.5.3\lib\netstandard2.0\System.Memory.dll</HintPath>
</Reference>
<Reference Include="System.Numerics" />
<Reference Include="System.Numerics.Vectors, Version=4.1.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Numerics.Vectors.4.4.0\lib\net46\System.Numerics.Vectors.dll</HintPath>
</Reference>
<Reference Include="System.Runtime.CompilerServices.Unsafe, Version=4.0.4.1, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Runtime.CompilerServices.Unsafe.4.5.2\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll</HintPath>
</Reference>
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.Security" />
<Reference Include="System.ServiceModel" />
<Reference Include="System.Web" />
<Reference Include="System.Windows.Forms" />
<Reference Include="System.Xml" />

View File

@ -4,10 +4,20 @@
<package id="Costura.Fody" version="4.0.0" targetFramework="net472" />
<package id="Fody" version="5.1.1" targetFramework="net472" developmentDependency="true" />
<package id="GitInfo" version="2.0.20" targetFramework="net472" developmentDependency="true" />
<package id="K4os.Compression.LZ4" version="1.1.11" targetFramework="net472" />
<package id="K4os.Compression.LZ4.Streams" version="1.1.11" targetFramework="net472" />
<package id="K4os.Hash.xxHash" version="1.0.6" targetFramework="net472" />
<package id="MegaApiClient" version="1.7.1" targetFramework="net472" />
<package id="Newtonsoft.Json" version="12.0.2" targetFramework="net472" />
<package id="Newtonsoft.Json.Bson" version="1.0.2" targetFramework="net472" />
<package id="Ookii.Dialogs.Wpf" version="1.1.0" targetFramework="net472" />
<package id="protobuf-net" version="2.4.0" targetFramework="net472" />
<package id="SharpCompress" version="0.23.0" targetFramework="net472" />
<package id="SharpZipLib" version="1.2.0" targetFramework="net472" />
<package id="System.Buffers" version="4.4.0" targetFramework="net472" />
<package id="System.Memory" version="4.5.3" targetFramework="net472" />
<package id="System.Numerics.Vectors" version="4.4.0" targetFramework="net472" />
<package id="System.Runtime.CompilerServices.Unsafe" version="4.5.2" targetFramework="net472" />
<package id="WebSocketSharpFork" version="1.0.4.0" targetFramework="net472" />
<package id="WPFThemes.DarkBlend" version="1.0.8" targetFramework="net472" />
</packages>