wabbajack/Wabbajack.Lib/MO2Compiler.cs

725 lines
28 KiB
C#
Raw Normal View History

2020-10-01 03:50:09 +00:00
using System;
2019-07-26 20:59:14 +00:00
using System.Collections.Concurrent;
2019-07-21 04:40:54 +00:00
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
2020-10-18 12:59:49 +00:00
using Alphaleonis.Win32.Filesystem;
2019-07-21 04:40:54 +00:00
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.Validation;
using Wabbajack.VirtualFileSystem;
2019-07-21 04:40:54 +00:00
namespace Wabbajack.Lib
2019-07-21 04:40:54 +00:00
{
public class MO2Compiler : ACompiler
2019-07-21 04:40:54 +00:00
{
2020-03-25 12:47:25 +00:00
private AbsolutePath _mo2DownloadsFolder;
2019-11-15 13:06:34 +00:00
2020-10-18 12:59:49 +00:00
public MO2Compiler(AbsolutePath sourcePath, AbsolutePath downloadsPath, string mo2Profile, AbsolutePath outputFile)
: base(21, mo2Profile, sourcePath, downloadsPath, outputFile)
{
MO2Profile = mo2Profile;
MO2Ini = SourcePath.Combine("ModOrganizer.ini").LoadIniFile();
var mo2game = (string)MO2Ini.General.gameName;
CompilingGame = GameRegistry.Games.First(g => g.Value.MO2Name == mo2game).Value;
GamePath = CompilingGame.GameLocation();
ModListOutputFile = outputFile;
Settings = new CompilerSettings();
}
2019-07-21 04:40:54 +00:00
2020-10-18 12:59:49 +00:00
public AbsolutePath MO2ModsFolder => SourcePath.Combine(Consts.MO2ModFolderName);
2020-03-25 23:15:19 +00:00
public string MO2Profile { get; }
public override ModManager ModManager => ModManager.MO2;
2020-03-25 12:47:25 +00:00
public override AbsolutePath GamePath { get; }
2020-04-09 21:05:07 +00:00
public GameMetaData CompilingGame { get; }
2020-10-18 12:59:49 +00:00
2020-09-11 12:54:24 +00:00
public CompilerSettings Settings { get; set; }
2020-03-25 12:47:25 +00:00
public override AbsolutePath ModListOutputFolder => ((RelativePath)"output_folder").RelativeToEntryPoint();
2020-03-25 12:47:25 +00:00
public override AbsolutePath ModListOutputFile { get; }
2020-10-18 12:59:49 +00:00
public override AbsolutePath VFSCacheName =>
Consts.LocalAppDataPath.Combine(
$"vfs_compile_cache-2-{Path.Combine((string)SourcePath ?? "Unknown", "ModOrganizer.exe").StringSha256Hex()}.bin");
2020-04-09 21:05:07 +00:00
public dynamic MO2Ini { get; }
2020-10-18 12:59:49 +00:00
public AbsolutePath MO2ProfileDir => SourcePath.Combine("profiles", MO2Profile);
2020-04-09 21:05:07 +00:00
public ConcurrentBag<Directive> ExtraFiles { get; private set; } = new ConcurrentBag<Directive>();
2020-04-09 21:05:07 +00:00
public Dictionary<AbsolutePath, dynamic> ModInis { get; } = new Dictionary<AbsolutePath, dynamic>();
public HashSet<string> SelectedProfiles { get; set; } = new HashSet<string>();
2020-10-18 12:59:49 +00:00
public Dictionary<Game, HashSet<Hash>> GameHashes { get; set; } = new Dictionary<Game, HashSet<Hash>>();
public Dictionary<Hash, Game[]> GamesWithHashes { get; set; } = new Dictionary<Hash, Game[]>();
public bool UseGamePaths { get; set; } = true;
public static AbsolutePath GetTypicalDownloadsFolder(AbsolutePath mo2Folder)
2019-07-22 22:17:46 +00:00
{
2020-10-18 12:59:49 +00:00
return mo2Folder.Combine("downloads");
2019-07-21 04:40:54 +00:00
}
protected override async Task<bool> _Begin(CancellationToken cancel)
2019-07-21 04:40:54 +00:00
{
2020-06-14 13:13:29 +00:00
await Metrics.Send("begin_compiling", MO2Profile ?? "unknown");
2020-10-18 12:59:49 +00:00
if (cancel.IsCancellationRequested)
{
return false;
}
2020-09-12 20:23:03 +00:00
DesiredThreads.OnNext(DiskThreads);
FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;
2019-11-17 04:16:42 +00:00
UpdateTracker.Reset();
UpdateTracker.NextStep("Gathering information");
2020-09-11 12:54:24 +00:00
2020-10-18 12:59:49 +00:00
Utils.Log("Loading compiler Settings");
2020-09-11 12:54:24 +00:00
Settings = await CompilerSettings.Load(MO2ProfileDir);
Settings.IncludedGames = Settings.IncludedGames.Add(CompilingGame.Game);
2020-10-18 12:59:49 +00:00
Info("Looking for other profiles");
2020-03-25 12:47:25 +00:00
var otherProfilesPath = MO2ProfileDir.Combine("otherprofiles.txt");
SelectedProfiles = new HashSet<string>();
2020-10-18 12:59:49 +00:00
if (otherProfilesPath.Exists)
{
SelectedProfiles = (await otherProfilesPath.ReadAllLinesAsync()).ToHashSet();
}
2020-05-13 14:06:18 +00:00
SelectedProfiles.Add(MO2Profile!);
Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p)));
Utils.Log($"Compiling Game: {CompilingGame}");
2020-10-18 12:59:49 +00:00
Utils.Log("Games from setting files:");
foreach (var game in Settings.IncludedGames)
{
Utils.Log($"- {game}");
}
2020-02-15 13:12:06 +00:00
Utils.Log($"VFS File Location: {VFSCacheName}");
2020-10-18 12:59:49 +00:00
Utils.Log($"MO2 Folder: {SourcePath}");
Utils.Log($"Downloads Folder: {DownloadsPath}");
2020-07-14 12:15:01 +00:00
Utils.Log($"Game Folder: {GamePath}");
2020-10-18 12:59:49 +00:00
var watcher = new DiskSpaceWatcher(cancel,
new[] {SourcePath, DownloadsPath, GamePath, AbsolutePath.EntryPoint}, (long)2 << 31,
2020-07-27 21:33:45 +00:00
drive =>
{
Utils.Log($"Aborting due to low space on {drive.Name}");
Abort();
});
var watcherTask = watcher.Start();
2020-02-15 13:12:06 +00:00
2020-10-18 12:59:49 +00:00
if (cancel.IsCancellationRequested)
{
return false;
}
List<AbsolutePath> roots;
if (UseGamePaths)
{
2020-10-18 12:59:49 +00:00
roots = new List<AbsolutePath> {SourcePath, GamePath, DownloadsPath};
2020-09-11 12:54:24 +00:00
roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation()));
}
else
{
2020-10-18 12:59:49 +00:00
roots = new List<AbsolutePath> {SourcePath, DownloadsPath};
}
// TODO: make this generic so we can add more paths
2020-10-18 12:59:49 +00:00
var lootPath = (AbsolutePath)Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
"LOOT");
IEnumerable<RawSourceFile> lootFiles = new List<RawSourceFile>();
2020-03-25 22:49:32 +00:00
if (lootPath.Exists)
{
2020-10-18 12:59:49 +00:00
roots.Add(lootPath);
}
2020-10-18 12:59:49 +00:00
UpdateTracker.NextStep("Indexing folders");
2020-10-18 12:59:49 +00:00
if (cancel.IsCancellationRequested)
{
return false;
}
await VFS.AddRoots(roots);
2020-10-18 12:59:49 +00:00
2020-03-25 22:49:32 +00:00
if (lootPath.Exists)
{
2020-04-09 21:05:07 +00:00
if (CompilingGame.MO2Name == null)
{
throw new ArgumentException("Compiling game had no MO2 name specified.");
}
2020-10-18 12:59:49 +00:00
var lootGameDirs = new[]
{
CompilingGame.MO2Name, // most of the games use the MO2 name
CompilingGame.MO2Name.Replace(" ", "") //eg: Fallout 4 -> Fallout4
};
2020-04-03 22:56:14 +00:00
var lootGameDir = lootGameDirs.Select(x => lootPath.Combine(x))
.FirstOrDefault(p => p.IsDirectory);
2020-04-03 22:56:14 +00:00
if (lootGameDir != default)
{
Utils.Log($"Found LOOT game folder at {lootGameDir}");
2020-04-03 22:56:14 +00:00
lootFiles = lootGameDir.EnumerateFiles(false)
.Where(p => p.FileName == (RelativePath)"userlist.yaml")
.Where(p => p.IsFile)
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p],
2020-04-03 22:56:14 +00:00
Consts.LOOTFolderFilesDir.Combine(p.RelativeTo(lootPath))));
if (!lootFiles.Any())
2020-10-18 12:59:49 +00:00
{
Utils.Log(
$"Found no LOOT user data for {CompilingGame.HumanFriendlyGameName} at {lootGameDir}!");
2020-10-18 12:59:49 +00:00
}
}
}
2020-10-18 12:59:49 +00:00
if (cancel.IsCancellationRequested)
{
return false;
}
2019-11-17 04:16:42 +00:00
UpdateTracker.NextStep("Cleaning output folder");
2020-03-28 04:33:26 +00:00
await ModListOutputFolder.DeleteDirectory();
2020-10-18 12:59:49 +00:00
if (cancel.IsCancellationRequested)
{
return false;
}
UpdateTracker.NextStep("Inferring metas for game file downloads");
await InferMetas();
2020-10-18 12:59:49 +00:00
if (cancel.IsCancellationRequested)
{
return false;
}
UpdateTracker.NextStep("Reindexing downloads after meta inferring");
2020-10-18 12:59:49 +00:00
await VFS.AddRoot(DownloadsPath);
if (cancel.IsCancellationRequested)
{
return false;
}
UpdateTracker.NextStep("Pre-validating Archives");
2020-10-18 12:59:49 +00:00
2019-09-24 04:34:21 +00:00
// Find all Downloads
2020-10-18 12:59:49 +00:00
IndexedArchives = (await DownloadsPath.EnumerateFiles()
2020-03-25 22:49:32 +00:00
.Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
2020-10-18 12:59:49 +00:00
.PMap(Queue,
async f => new IndexedArchive(VFS.Index.ByRootPath[f])
{
Name = (string)f.FileName,
IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(),
Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync()
})).ToList();
2020-06-20 22:51:47 +00:00
if (UseGamePaths)
{
2020-09-11 12:54:24 +00:00
foreach (var ag in Settings.IncludedGames)
{
try
2020-06-11 16:18:21 +00:00
{
var files = await ClientAPI.GetExistingGameFiles(Queue, ag);
Utils.Log($"Including {files.Length} stock game files from {ag} as download sources");
GameHashes[ag] = files.Select(f => f.Hash).ToHashSet();
IndexedArchives.AddRange(files.Select(f =>
2020-06-20 22:51:47 +00:00
{
var meta = f.State.GetMetaIniString();
var ini = meta.LoadIniString();
var state = (GameFileSourceDownloader.State)f.State;
return new IndexedArchive(
VFS.Index.ByRootPath[ag.MetaData().GameLocation().Combine(state.GameFile)])
{
2020-10-18 12:59:49 +00:00
IniData = ini, Meta = meta
};
}));
}
catch (Exception e)
{
Utils.Error(e, "Unable to find existing game files, skipping.");
}
}
GamesWithHashes = GameHashes.SelectMany(g => g.Value.Select(h => (g, h)))
.GroupBy(gh => gh.h)
.ToDictionary(gh => gh.Key, gh => gh.Select(p => p.g.Key).ToArray());
}
2020-06-11 16:18:21 +00:00
IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();
await CleanInvalidArchivesAndFillState();
UpdateTracker.NextStep("Finding Install Files");
2020-03-25 22:49:32 +00:00
ModListOutputFolder.CreateDirectory();
2020-10-18 12:59:49 +00:00
var mo2Files = SourcePath.EnumerateFiles()
2020-03-25 22:49:32 +00:00
.Where(p => p.IsFile)
.Select(p =>
{
2020-03-25 22:49:32 +00:00
if (!VFS.Index.ByRootPath.ContainsKey(p))
2020-10-18 12:59:49 +00:00
{
Utils.Log($"WELL THERE'S YOUR PROBLEM: {p} {VFS.Index.ByRootPath.Count}");
2020-10-18 12:59:49 +00:00
}
return new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(SourcePath));
});
// If Game Folder Files exists, ignore the game folder
2019-11-15 13:06:34 +00:00
IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
.OrderBy(f => f.NestingFactor)
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, f => f.AsEnumerable());
2020-07-07 20:17:49 +00:00
AllFiles.SetTo(mo2Files
2019-11-21 15:51:57 +00:00
.Concat(lootFiles)
2020-04-09 21:05:07 +00:00
.DistinctBy(f => f.Path));
2019-09-26 22:32:15 +00:00
Info($"Found {AllFiles.Count} files to build into mod list");
2019-07-21 04:40:54 +00:00
2020-10-18 12:59:49 +00:00
if (cancel.IsCancellationRequested)
{
return false;
}
2019-11-24 23:03:36 +00:00
UpdateTracker.NextStep("Verifying destinations");
var dups = AllFiles.GroupBy(f => f.Path)
.Where(fs => fs.Count() > 1)
.Select(fs =>
{
2020-10-18 12:59:49 +00:00
Utils.Log(
$"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}");
return fs;
}).ToList();
if (dups.Count > 0)
{
Error($"Found {dups.Count} duplicates, exiting");
}
2020-10-18 12:59:49 +00:00
if (cancel.IsCancellationRequested)
{
return false;
}
2019-11-24 23:03:36 +00:00
UpdateTracker.NextStep("Loading INIs");
2020-10-18 12:59:49 +00:00
ModInis.SetTo(SourcePath.Combine(Consts.MO2ModFolderName)
2020-03-25 23:15:19 +00:00
.EnumerateDirectories()
.Select(f =>
{
2020-03-25 23:15:19 +00:00
var modName = f.FileName;
var metaPath = f.Combine("meta.ini");
2020-03-28 18:22:53 +00:00
return metaPath.Exists ? (mod_name: f, metaPath.LoadIniFile()) : default;
})
2020-03-28 18:22:53 +00:00
.Where(f => f.Item1 != default)
2020-04-09 21:05:07 +00:00
.Select(f => new KeyValuePair<AbsolutePath, dynamic>(f.Item1, f.Item2)));
ArchivesByFullPath = IndexedArchives.ToDictionary(a => a.File.AbsoluteName);
2020-10-18 12:59:49 +00:00
if (cancel.IsCancellationRequested)
{
return false;
}
var stack = MakeStack();
2019-11-17 04:16:42 +00:00
UpdateTracker.NextStep("Running Compilation Stack");
var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f));
2019-07-21 04:40:54 +00:00
2019-07-26 20:59:14 +00:00
// Add the extra files that were generated by the stack
2020-10-18 12:59:49 +00:00
if (cancel.IsCancellationRequested)
{
return false;
}
2019-11-24 23:03:36 +00:00
UpdateTracker.NextStep($"Adding {ExtraFiles.Count} that were generated by the stack");
results = results.Concat(ExtraFiles).ToArray();
2019-07-26 20:59:14 +00:00
var noMatch = results.OfType<NoMatch>().ToArray();
PrintNoMatches(noMatch);
2020-10-18 12:59:49 +00:00
if (CheckForNoMatchExit(noMatch))
{
return false;
}
2019-07-21 04:40:54 +00:00
2020-07-27 21:33:45 +00:00
foreach (var ignored in results.OfType<IgnoredDirectly>())
{
Utils.Log($"Ignored {ignored.To} because {ignored.Reason}");
}
2020-04-09 21:05:07 +00:00
InstallDirectives.SetTo(results.Where(i => !(i is IgnoredDirectly)));
2019-07-21 22:47:17 +00:00
2019-09-24 15:26:44 +00:00
Info("Getting Nexus api_key, please click authorize if a browser window appears");
2019-11-24 23:03:36 +00:00
UpdateTracker.NextStep("Verifying Files");
2019-09-23 21:37:10 +00:00
zEditIntegration.VerifyMerges(this);
UpdateTracker.NextStep("Building Patches");
await BuildPatches();
2020-10-18 12:59:49 +00:00
2019-11-24 23:03:36 +00:00
UpdateTracker.NextStep("Gathering Archives");
await GatherArchives();
2020-10-18 12:59:49 +00:00
2019-11-24 23:03:36 +00:00
UpdateTracker.NextStep("Including Archive Metadata");
await IncludeArchiveMetadata();
2019-07-23 04:27:26 +00:00
UpdateTracker.NextStep("Gathering Metadata");
await GatherMetaData();
ModList = new ModList
2019-07-23 04:27:26 +00:00
{
GameType = CompilingGame.Game,
WabbajackVersion = Consts.CurrentMinimumWabbajackVersion,
2019-12-03 21:13:29 +00:00
Archives = SelectedArchives.ToList(),
ModManager = ModManager.MO2,
2019-08-02 23:04:04 +00:00
Directives = InstallDirectives,
2020-05-13 14:06:18 +00:00
Name = ModListName ?? MO2Profile!,
Author = ModListAuthor ?? "",
Description = ModListDescription ?? "",
2020-04-15 17:40:41 +00:00
Readme = ModlistReadme ?? "",
2020-03-28 02:54:14 +00:00
Image = ModListImage != default ? ModListImage.FileName : default,
2020-04-27 09:58:33 +00:00
Website = !string.IsNullOrWhiteSpace(ModListWebsite) ? new Uri(ModListWebsite) : null,
2020-10-18 12:59:49 +00:00
Version = ModlistVersion ?? new Version(1, 0, 0, 0),
2020-04-27 09:58:33 +00:00
IsNSFW = ModlistIsNSFW
2019-07-23 04:27:26 +00:00
};
2020-10-18 12:59:49 +00:00
2020-09-05 19:36:44 +00:00
UpdateTracker.NextStep("Including required files");
await InlineFiles();
2019-07-23 04:27:26 +00:00
2019-11-24 23:03:36 +00:00
UpdateTracker.NextStep("Running Validation");
2020-03-22 15:50:53 +00:00
await ValidateModlist.RunValidation(ModList);
2019-11-24 23:03:36 +00:00
UpdateTracker.NextStep("Generating Report");
2020-02-01 12:13:12 +00:00
GenerateManifest();
2019-11-24 23:03:36 +00:00
UpdateTracker.NextStep("Exporting Modlist");
2020-04-01 23:59:22 +00:00
await ExportModList();
2019-07-23 04:27:26 +00:00
ResetMembers();
2019-11-24 23:03:36 +00:00
UpdateTracker.NextStep("Done Building Modlist");
2019-09-24 04:20:24 +00:00
return true;
2019-07-21 04:40:54 +00:00
}
private async Task CleanInvalidArchivesAndFillState()
{
var remove = (await IndexedArchives.PMap(Queue, async a =>
{
try
{
a.State = (await ResolveArchive(a)).State;
return null;
}
catch
{
return a;
}
2020-04-09 21:05:07 +00:00
})).NotNull().ToHashSet();
if (remove.Count == 0)
2020-10-18 12:59:49 +00:00
{
return;
2020-10-18 12:59:49 +00:00
}
Utils.Log(
$"Removing {remove.Count} archives from the compilation state, this is probably not an issue but reference this if you have compilation failures");
remove.Do(r => Utils.Log($"Resolution failed for: ({r.File.Size} {r.File.Hash}) {r.File.FullPath}"));
IndexedArchives.RemoveAll(a => remove.Contains(a));
}
private async Task InferMetas()
{
2020-03-25 23:15:19 +00:00
async Task<bool> HasInvalidMeta(AbsolutePath filename)
{
2020-03-25 23:15:19 +00:00
var metaname = filename.WithExtension(Consts.MetaFileExtension);
2020-10-18 12:59:49 +00:00
if (!metaname.Exists)
{
return true;
}
2020-07-06 06:58:13 +00:00
try
{
return await DownloadDispatcher.ResolveArchive(metaname.LoadIniFile()) == null;
}
catch (Exception e)
{
Utils.ErrorThrow(e, $"Exception while checking meta {filename}");
return false;
}
}
2020-10-18 12:59:49 +00:00
var to_find = (await DownloadsPath.EnumerateFiles()
.Where(f => f.Extension != Consts.MetaFileExtension && f.Extension != Consts.HashFileExtension)
.PMap(Queue, async f => await HasInvalidMeta(f) ? f : default))
2020-03-25 23:15:19 +00:00
.Where(f => f.Exists)
.ToList();
2020-10-18 12:59:49 +00:00
if (to_find.Count == 0)
{
return;
}
Utils.Log($"Attempting to infer {to_find.Count} metas from the server.");
await to_find.PMap(Queue, async f =>
{
2020-03-25 23:15:19 +00:00
var vf = VFS.Index.ByRootPath[f];
2020-06-21 22:03:54 +00:00
var meta = await ClientAPI.InferDownloadState(vf.Hash);
if (meta == null)
{
2020-03-25 23:15:19 +00:00
await vf.AbsoluteName.WithExtension(Consts.MetaFileExtension).WriteAllLinesAsync(
2020-10-18 12:59:49 +00:00
"[General]",
2020-03-25 23:15:19 +00:00
"unknownArchive=true");
return;
}
2020-03-25 23:15:19 +00:00
Utils.Log($"Inferred .meta for {vf.FullPath.FileName}, writing to disk");
2020-10-18 12:59:49 +00:00
await vf.AbsoluteName.WithExtension(Consts.MetaFileExtension)
.WriteAllTextAsync(meta.GetMetaIniString());
});
}
private async Task IncludeArchiveMetadata()
{
Utils.Log($"Including {SelectedArchives.Count} .meta files for downloads");
2020-03-25 23:33:52 +00:00
await SelectedArchives.PMap(Queue, async a =>
{
2020-10-18 12:59:49 +00:00
if (a.State is GameFileSourceDownloader.State)
{
return;
}
var source = DownloadsPath.Combine(a.Name + Consts.MetaFileExtension);
var ini = a.State.GetMetaIniString();
var (id, fullPath) = await IncludeString(ini);
2020-03-25 23:33:52 +00:00
InstallDirectives.Add(new ArchiveMeta
{
SourceDataID = id,
2020-06-20 22:51:47 +00:00
Size = fullPath.Size,
Hash = await fullPath.FileHashAsync(),
2020-03-25 23:33:52 +00:00
To = source.FileName
});
});
}
2019-07-23 04:27:26 +00:00
/// <summary>
/// Clear references to lists that hold a lot of data.
2019-07-23 04:27:26 +00:00
/// </summary>
private void ResetMembers()
{
AllFiles = new List<RawSourceFile>();
InstallDirectives = new List<Directive>();
SelectedArchives = new List<Archive>();
ExtraFiles = new ConcurrentBag<Directive>();
2019-07-23 04:27:26 +00:00
}
2019-07-22 03:36:25 +00:00
/// <summary>
/// Fills in the Patch fields in files that require them
2019-07-22 03:36:25 +00:00
/// </summary>
private async Task BuildPatches()
2019-07-22 03:36:25 +00:00
{
Info("Gathering patch files");
var toBuild = InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => p.Choices.Length > 0)
.SelectMany(p => p.Choices.Select(c => new PatchedFromArchive
2020-10-18 12:59:49 +00:00
{
To = p.To,
Hash = p.Hash,
ArchiveHashPath = c.MakeRelativePaths(),
FromFile = c,
Size = p.Size
}))
.ToArray();
2020-10-18 12:59:49 +00:00
if (toBuild.Length == 0)
{
return;
}
2019-07-22 03:36:25 +00:00
// Extract all the source files
2020-10-18 12:59:49 +00:00
var indexed = toBuild.GroupBy(f => VFS.Index.FileForArchiveHashPath(f.ArchiveHashPath))
.ToDictionary(f => f.Key);
2020-10-18 12:59:49 +00:00
await VFS.Extract(Queue, indexed.Keys.ToHashSet(),
async (vf, sf) =>
{
2020-10-18 12:59:49 +00:00
// For each, extract the destination
var matches = indexed[vf];
using var iqueue = new WorkQueue(1);
foreach (var match in matches)
{
2020-10-18 12:59:49 +00:00
var destFile = FindDestFile(match.To);
// Build the patch
await VFS.Extract(iqueue, new[] {destFile}.ToHashSet(),
async (destvf, destsfn) =>
{
Info($"Patching {match.To}");
Status($"Patching {match.To}");
await using var srcStream = await sf.GetStream();
await using var destStream = await destsfn.GetStream();
var patchSize =
await Utils.CreatePatchCached(srcStream, vf.Hash, destStream, destvf.Hash);
Info($"Patch size {patchSize} for {match.To}");
});
}
});
// Load in the patches
await InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => p.PatchID == default)
.PMap(Queue, async pfa =>
{
var patches = pfa.Choices
.Select(c => (Utils.TryGetPatch(c.Hash, pfa.Hash, out var data), data, c))
.ToArray();
// Pick the best patch
if (patches.All(p => p.Item1))
{
var (_, bytes, file) = IncludePatches.PickPatch(this, patches);
pfa.FromFile = file;
pfa.FromHash = file.Hash;
pfa.ArchiveHashPath = file.MakeRelativePaths();
pfa.PatchID = await IncludeFile(bytes!);
}
});
2020-10-18 12:59:49 +00:00
var firstFailedPatch =
InstallDirectives.OfType<PatchedFromArchive>().FirstOrDefault(f => f.PatchID == default);
if (firstFailedPatch != null)
{
2020-10-18 12:59:49 +00:00
Utils.Log("Missing data from failed patch, starting data dump");
Utils.Log($"Dest File: {firstFailedPatch.To}");
Utils.Log($"Options ({firstFailedPatch.Choices.Length}:");
foreach (var choice in firstFailedPatch.Choices)
{
Utils.Log($" {choice.FullPath}");
}
2020-10-18 12:59:49 +00:00
Error(
$"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}");
}
2019-07-22 03:36:25 +00:00
}
private VirtualFile FindDestFile(RelativePath to)
2019-07-26 20:59:14 +00:00
{
2020-10-18 12:59:49 +00:00
var abs = to.RelativeTo(SourcePath);
if (abs.Exists)
2020-10-18 12:59:49 +00:00
{
return VFS.Index.ByRootPath[abs];
2020-10-18 12:59:49 +00:00
}
2019-07-26 20:59:14 +00:00
if (to.StartsWith(Consts.BSACreationDir))
{
2020-03-25 23:33:52 +00:00
var bsaId = (RelativePath)((string)to).Split('\\')[1];
var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsaId);
var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray());
2020-10-18 12:59:49 +00:00
return VFS.Index.ByRootPath[SourcePath.Combine(bsa.To)].Children.First(c => c.RelativeName == find);
2019-07-26 20:59:14 +00:00
}
2020-04-09 21:05:07 +00:00
throw new ArgumentException($"Couldn't load data for {to}");
2019-07-26 20:59:14 +00:00
}
public override IEnumerable<ICompilationStep> GetStack()
{
2020-03-25 23:33:52 +00:00
return MakeStack();
}
2019-07-21 04:40:54 +00:00
/// <summary>
/// Creates a execution stack. The stack should be passed into Run stack. Each function
/// in this stack will be run in-order and the first to return a non-null result will have its
/// result included into the pack
2019-07-21 04:40:54 +00:00
/// </summary>
/// <returns></returns>
public override IEnumerable<ICompilationStep> MakeStack()
2019-07-21 04:40:54 +00:00
{
Utils.Log("Generating compilation stack");
return new List<ICompilationStep>
{
2020-01-07 00:24:33 +00:00
new IgnoreGameFilesIfGameFolderFilesExist(this),
new IncludePropertyFiles(this),
2020-07-05 15:11:52 +00:00
//new IncludeSteamWorkshopItems(this),
new IgnoreSaveFiles(this),
2020-10-18 12:59:49 +00:00
new IgnoreStartsWith(this, "logs\\"),
new IgnoreStartsWith(this, "downloads\\"),
2020-10-18 12:59:49 +00:00
new IgnoreStartsWith(this, "webcache\\"),
new IgnoreStartsWith(this, "overwrite\\"),
2020-03-04 05:23:08 +00:00
new IgnoreStartsWith(this, "crashDumps\\"),
2020-10-18 12:59:49 +00:00
new IgnorePathContains(this, "temporary_logs"),
new IgnorePathContains(this, "GPUCache"),
new IgnorePathContains(this, "SSEEdit Cache"),
new IgnoreOtherProfiles(this),
new IgnoreDisabledMods(this),
new IncludeThisProfile(this),
2019-07-21 04:40:54 +00:00
// Ignore the ModOrganizer.ini file it contains info created by MO2 on startup
new IncludeStubbedConfigFiles(this),
new IncludeLootFiles(this),
new IgnoreStartsWith(this, Path.Combine((string)Consts.GameFolderFilesDir, "Data")),
new IgnoreStartsWith(this, Path.Combine((string)Consts.GameFolderFilesDir, "Papyrus Compiler")),
2020-10-18 12:59:49 +00:00
new IgnoreStartsWith(this, Path.Combine((string)Consts.GameFolderFilesDir, "Skyrim")),
new IgnoreRegex(this, Consts.GameFolderFilesDir + "\\\\.*\\.bsa"),
new IncludeRegex(this, "^[^\\\\]*\\.bat$"),
new IncludeModIniData(this),
new DirectMatch(this),
new IncludeTaggedMods(this, Consts.WABBAJACK_INCLUDE),
2020-07-27 21:33:45 +00:00
new IgnoreEndsWith(this, ".pyc"),
new IgnoreEndsWith(this, ".log"),
2020-10-18 12:59:49 +00:00
new DeconstructBSAs(
this), // Deconstruct BSAs before building patches so we don't generate massive patch files
new IncludePatches(this),
new IncludeDummyESPs(this),
2019-07-21 12:42:29 +00:00
2019-09-26 23:08:10 +00:00
// There are some types of files that will error the compilation, because they're created on-the-fly via tools
2019-07-26 20:59:14 +00:00
// so if we don't have a match by this point, just drop them.
new IgnoreEndsWith(this, ".ini"),
new IgnoreEndsWith(this, ".html"),
new IgnoreEndsWith(this, ".txt"),
2019-07-26 20:59:14 +00:00
// Don't know why, but this seems to get copied around a bit
new IgnoreEndsWith(this, "HavokBehaviorPostProcess.exe"),
2019-08-03 17:37:32 +00:00
// Theme file MO2 downloads somehow
new IgnoreEndsWith(this, "splash.png"),
// File to force MO2 into portable mode
2020-10-18 12:59:49 +00:00
new IgnoreEndsWith(this, "portable.txt"),
new IgnoreEndsWith(this, ".bin"),
new IgnoreEndsWith(this, ".refcache"),
//Include custom categories
new IncludeRegex(this, "categories.dat$"),
new IgnoreWabbajackInstallCruft(this),
2019-07-21 22:47:17 +00:00
2020-06-20 22:51:47 +00:00
//new PatchStockESMs(this),
new IncludeAllConfigs(this),
new zEditIntegration.IncludeZEditPatches(this),
2019-11-02 18:36:38 +00:00
new IncludeTaggedMods(this, Consts.WABBAJACK_NOMATCH_INCLUDE),
new DropAll(this)
2019-07-21 04:40:54 +00:00
};
}
}
}