mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
Redid the entire Vortex Compiler
This commit is contained in:
erri120
parent
dcf0113c17
commit
5bc6f44aeb
@ -4,15 +4,18 @@ using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading;
|
||||
using DynamicData;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.WindowsAPICodePack.Shell;
|
||||
using Newtonsoft.Json;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib.CompilationSteps;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
using File = Alphaleonis.Win32.Filesystem.File;
|
||||
using Wabbajack.Lib.Validation;
|
||||
using Directory = Alphaleonis.Win32.Filesystem.Directory;
|
||||
using File = System.IO.File;
|
||||
using Game = Wabbajack.Common.Game;
|
||||
using Path = Alphaleonis.Win32.Filesystem.Path;
|
||||
|
||||
namespace Wabbajack.Lib
|
||||
{
|
||||
@ -29,90 +32,84 @@ namespace Wabbajack.Lib
|
||||
public Game Game { get; }
|
||||
public string GameName { get; }
|
||||
|
||||
public bool IgnoreMissingFiles { get; set; }
|
||||
|
||||
public string VortexFolder { get; set; }
|
||||
public string StagingFolder { get; set; }
|
||||
public string DownloadsFolder { get; set; }
|
||||
|
||||
public bool IgnoreMissingFiles { get; set; }
|
||||
|
||||
public override ModManager ModManager => ModManager.Vortex;
|
||||
public override string GamePath { get; }
|
||||
public override string ModListOutputFolder { get; }
|
||||
public override string ModListOutputFolder => "output_folder";
|
||||
public override string ModListOutputFile { get; }
|
||||
|
||||
public const string StagingMarkerName = "__vortex_staging_folder";
|
||||
public const string DownloadMarkerName = "__vortex_downloads_folder";
|
||||
|
||||
private bool _isSteamGame;
|
||||
private SteamGame _steamGame;
|
||||
private bool _hasSteamWorkshopItems;
|
||||
|
||||
public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder, string stagingFolder, string outputFile)
|
||||
public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder,
|
||||
string stagingFolder, string outputFile)
|
||||
{
|
||||
Game = game;
|
||||
|
||||
GamePath = gamePath;
|
||||
GameName = game.MetaData().NexusName;
|
||||
VortexFolder = vortexFolder;
|
||||
DownloadsFolder = downloadsFolder;
|
||||
StagingFolder = stagingFolder;
|
||||
ModListOutputFolder = "output_folder";
|
||||
ModListOutputFile = outputFile;
|
||||
|
||||
// there can be max one game after filtering
|
||||
SteamHandler.Instance.Games.Where(g => g.Game != null && g.Game == game).Do(g =>
|
||||
if (string.IsNullOrEmpty(ModListName))
|
||||
{
|
||||
_isSteamGame = true;
|
||||
_steamGame = g;
|
||||
SteamHandler.Instance.LoadWorkshopItems(_steamGame);
|
||||
_hasSteamWorkshopItems = _steamGame.WorkshopItems.Count > 0;
|
||||
});
|
||||
ModListName = $"Vortex ModList for {Game.ToString()}";
|
||||
ModListOutputFile = $"{ModListName}{ExtensionManager.Extension}";
|
||||
}
|
||||
|
||||
GameName = Game.MetaData().NexusName;
|
||||
|
||||
ActiveArchives = new List<string>();
|
||||
}
|
||||
|
||||
|
||||
protected override async Task<bool> _Begin(CancellationToken cancel)
|
||||
{
|
||||
if (cancel.IsCancellationRequested) return false;
|
||||
|
||||
ConfigureProcessor(10);
|
||||
if (string.IsNullOrEmpty(ModListName))
|
||||
ModListName = $"Vortex ModList for {Game.ToString()}";
|
||||
Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}.");
|
||||
|
||||
Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}.");
|
||||
ConfigureProcessor(12);
|
||||
UpdateTracker.Reset();
|
||||
|
||||
if (cancel.IsCancellationRequested) return false;
|
||||
UpdateTracker.NextStep("Parsing deployment file");
|
||||
ParseDeploymentFile();
|
||||
|
||||
if (cancel.IsCancellationRequested) return false;
|
||||
Info("Starting pre-compilation steps");
|
||||
UpdateTracker.NextStep("Creating metas for archives");
|
||||
await CreateMetaFiles();
|
||||
|
||||
if (cancel.IsCancellationRequested) return false;
|
||||
Info($"Indexing {StagingFolder}");
|
||||
await VFS.AddRoot(StagingFolder);
|
||||
await VFS.IntegrateFromFile(_vfsCacheName);
|
||||
|
||||
Info($"Indexing {GamePath}");
|
||||
await VFS.AddRoot(GamePath);
|
||||
|
||||
Info($"Indexing {DownloadsFolder}");
|
||||
await VFS.AddRoot(DownloadsFolder);
|
||||
var roots = new List<string> {StagingFolder, GamePath, DownloadsFolder};
|
||||
AddExternalFolder(ref roots);
|
||||
|
||||
if (cancel.IsCancellationRequested) return false;
|
||||
await AddExternalFolder();
|
||||
UpdateTracker.NextStep("Indexing folders");
|
||||
await VFS.AddRoots(roots);
|
||||
await VFS.WriteToFile(_vfsCacheName);
|
||||
|
||||
if (cancel.IsCancellationRequested) return false;
|
||||
Info("Cleaning output folder");
|
||||
if (Directory.Exists(ModListOutputFolder)) Utils.DeleteDirectory(ModListOutputFolder);
|
||||
UpdateTracker.NextStep("Cleaning output folder");
|
||||
if (Directory.Exists(ModListOutputFolder))
|
||||
Utils.DeleteDirectory(ModListOutputFolder);
|
||||
|
||||
Directory.CreateDirectory(ModListOutputFolder);
|
||||
|
||||
|
||||
UpdateTracker.NextStep("Finding Install Files");
|
||||
var vortexStagingFiles = Directory.EnumerateFiles(StagingFolder, "*", SearchOption.AllDirectories)
|
||||
.Where(p => p.FileExists() && p != StagingMarkerName)
|
||||
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
|
||||
{Path = p.RelativeTo(StagingFolder)});
|
||||
|
||||
var vortexDownloads = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.AllDirectories)
|
||||
.Where(p => p.FileExists())
|
||||
.Where(p => p.FileExists() && p != DownloadMarkerName)
|
||||
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
|
||||
{Path = p.RelativeTo(DownloadsFolder)});
|
||||
|
||||
@ -139,7 +136,6 @@ namespace Wabbajack.Lib
|
||||
.GroupBy(f => f.Hash)
|
||||
.ToDictionary(f => f.Key, f => f.AsEnumerable());
|
||||
|
||||
Info("Searching for mod files");
|
||||
AllFiles = vortexStagingFiles.Concat(vortexDownloads)
|
||||
.Concat(gameFiles)
|
||||
.DistinctBy(f => f.Path)
|
||||
@ -148,7 +144,8 @@ namespace Wabbajack.Lib
|
||||
Info($"Found {AllFiles.Count} files to build into mod list");
|
||||
|
||||
if (cancel.IsCancellationRequested) return false;
|
||||
Info("Verifying destinations");
|
||||
UpdateTracker.NextStep("Verifying destinations");
|
||||
|
||||
var duplicates = AllFiles.GroupBy(f => f.Path)
|
||||
.Where(fs => fs.Count() > 1)
|
||||
.Select(fs =>
|
||||
@ -162,52 +159,10 @@ namespace Wabbajack.Lib
|
||||
Error($"Found {duplicates.Count} duplicates, exiting");
|
||||
}
|
||||
|
||||
/*for (var i = 0; i < AllFiles.Count; i++)
|
||||
{
|
||||
var f = AllFiles[i];
|
||||
if (!f.Path.StartsWith(Consts.GameFolderFilesDir) || !IndexedFiles.ContainsKey(f.Hash))
|
||||
continue;
|
||||
|
||||
if (!IndexedFiles.TryGetValue(f.Hash, out var value))
|
||||
continue;
|
||||
|
||||
var element = value.ElementAt(0);
|
||||
|
||||
if (!f.Path.Contains(element.Name))
|
||||
continue;
|
||||
|
||||
IndexedArchive targetArchive = null;
|
||||
IndexedArchives.Where(a => a.File.Children.Contains(element)).Do(a => targetArchive = a);
|
||||
|
||||
if (targetArchive == null)
|
||||
continue;
|
||||
|
||||
if(targetArchive.IniData?.General?.tag == null || targetArchive.IniData?.General?.tag != Consts.WABBAJACK_VORTEX_MANUAL)
|
||||
continue;
|
||||
|
||||
#if DEBUG
|
||||
Utils.Log($"Double hash for: {f.AbsolutePath}");
|
||||
#endif
|
||||
|
||||
var replace = f;
|
||||
var name = replace.File.Name;
|
||||
var archiveName = targetArchive.Name;
|
||||
var elementPath = element.FullPath.Substring(element.FullPath.IndexOf('|')+1);
|
||||
var gameToFile = name.Substring(GamePath.Length + 1).Replace(elementPath, "");
|
||||
if (gameToFile.EndsWith("\\"))
|
||||
gameToFile = gameToFile.Substring(0, gameToFile.Length - 1);
|
||||
//replace.Path = replace.Path.Replace(Consts.GameFolderFilesDir, Consts.ManualGameFilesDir);
|
||||
replace.Path = Path.Combine(Consts.ManualGameFilesDir, archiveName, gameToFile, elementPath);
|
||||
//replace.Path = Path.Combine(Consts.ManualGameFilesDir, element.FullPath.Substring(DownloadsFolder.Length + 1).Replace('|', '\\'));
|
||||
AllFiles.RemoveAt(i);
|
||||
AllFiles.Insert(i, replace);
|
||||
//AllFiles.Replace(f, replace);
|
||||
}*/
|
||||
|
||||
if (cancel.IsCancellationRequested) return false;
|
||||
var stack = MakeStack();
|
||||
|
||||
Info("Running Compilation Stack");
|
||||
var results = await AllFiles.PMap(Queue, f => RunStack(stack.Where(s => s != null), f));
|
||||
UpdateTracker.NextStep("Running Compilation Stack");
|
||||
var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f));
|
||||
|
||||
IEnumerable<NoMatch> noMatch = results.OfType<NoMatch>().ToList();
|
||||
Info($"No match for {noMatch.Count()} files");
|
||||
@ -228,17 +183,17 @@ namespace Wabbajack.Lib
|
||||
|
||||
InstallDirectives = results.Where(i => !(i is IgnoredDirectly)).ToList();
|
||||
|
||||
// TODO: nexus stuff
|
||||
/*Info("Getting Nexus api_key, please click authorize if a browser window appears");
|
||||
Info("Getting Nexus api_key, please click authorize if a browser window appears");
|
||||
|
||||
if (IndexedArchives.Any(a => a.IniData?.General?.gameName != null))
|
||||
{
|
||||
var nexusClient = new NexusApiClient();
|
||||
if (!nexusClient.IsPremium) Error($"User {nexusClient.Username} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue");
|
||||
var nexusClient = await NexusApiClient.Get();
|
||||
if (!await nexusClient.IsPremium()) Error($"User {await nexusClient.Username()} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue");
|
||||
|
||||
}
|
||||
*/
|
||||
|
||||
if (cancel.IsCancellationRequested) return false;
|
||||
UpdateTracker.NextStep("Gathering Archives");
|
||||
await GatherArchives();
|
||||
|
||||
ModList = new ModList
|
||||
@ -254,16 +209,46 @@ namespace Wabbajack.Lib
|
||||
Directives = InstallDirectives,
|
||||
GameType = Game
|
||||
};
|
||||
|
||||
|
||||
UpdateTracker.NextStep("Running Validation");
|
||||
await ValidateModlist.RunValidation(Queue, ModList);
|
||||
|
||||
UpdateTracker.NextStep("Generating Report");
|
||||
GenerateReport();
|
||||
|
||||
UpdateTracker.NextStep("Exporting ModList");
|
||||
ExportModList();
|
||||
|
||||
Info("Done Building ModList");
|
||||
ResetMembers();
|
||||
|
||||
ShowReport();
|
||||
|
||||
UpdateTracker.NextStep("Done Building ModList");
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clear references to lists that hold a lot of data.
|
||||
/// </summary>
|
||||
private void ResetMembers()
|
||||
{
|
||||
AllFiles = null;
|
||||
InstallDirectives = null;
|
||||
SelectedArchives = null;
|
||||
}
|
||||
|
||||
private void AddExternalFolder(ref List<string> roots)
|
||||
{
|
||||
var currentGame = Game.MetaData();
|
||||
if (currentGame.AdditionalFolders == null || currentGame.AdditionalFolders.Count == 0) return;
|
||||
foreach (var path in currentGame.AdditionalFolders.Select(f => f.Replace("%documents%", KnownFolders.Documents.Path)))
|
||||
{
|
||||
if (!Directory.Exists(path)) return;
|
||||
roots.Add(path);
|
||||
}
|
||||
}
|
||||
|
||||
private void ParseDeploymentFile()
|
||||
{
|
||||
Info("Searching for vortex.deployment.json...");
|
||||
@ -280,10 +265,10 @@ namespace Wabbajack.Lib
|
||||
|
||||
if (string.IsNullOrEmpty(deploymentFile))
|
||||
{
|
||||
Info("vortex.deployment.json not found!");
|
||||
Error("vortex.deployment.json not found!");
|
||||
return;
|
||||
}
|
||||
Info("vortex.deployment.json found at "+deploymentFile);
|
||||
Info($"vortex.deployment.json found at {deploymentFile}");
|
||||
|
||||
Info("Parsing vortex.deployment.json...");
|
||||
try
|
||||
@ -306,127 +291,42 @@ namespace Wabbajack.Lib
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Some have mods outside their game folder located
|
||||
/// </summary>
|
||||
private async Task AddExternalFolder()
|
||||
{
|
||||
var currentGame = Game.MetaData();
|
||||
if (currentGame.AdditionalFolders == null || currentGame.AdditionalFolders.Count == 0) return;
|
||||
foreach (var f in currentGame.AdditionalFolders)
|
||||
{
|
||||
var path = f.Replace("%documents%", KnownFolders.Documents.Path);
|
||||
if (!Directory.Exists(path)) return;
|
||||
Info($"Indexing {path}");
|
||||
await VFS.AddRoot(path);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CreateMetaFiles()
|
||||
{
|
||||
Utils.Log("Getting Nexus api_key, please click authorize if a browser window appears");
|
||||
var nexusClient = await NexusApiClient.Get();
|
||||
|
||||
await Task.WhenAll(
|
||||
Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly)
|
||||
.Where(File.Exists)
|
||||
.Select(async f =>
|
||||
{
|
||||
if (Path.GetExtension(f) != ".meta" && Path.GetExtension(f) != ".xxHash" && !File.Exists($"{f}.meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)))
|
||||
{
|
||||
Utils.Log($"Trying to create meta file for {Path.GetFileName(f)}");
|
||||
var metaString = "[General]\n" +
|
||||
"repository=Nexus\n" +
|
||||
"installed=true\n" +
|
||||
"uninstalled=false\n" +
|
||||
"paused=false\n" +
|
||||
"removed=false\n" +
|
||||
$"gameName={GameName}\n";
|
||||
string hash;
|
||||
using(var md5 = MD5.Create())
|
||||
using (var stream = File.OpenRead(f))
|
||||
{
|
||||
Utils.Log($"Calculating hash for {Path.GetFileName(f)}");
|
||||
var cH = md5.ComputeHash(stream);
|
||||
hash = BitConverter.ToString(cH).Replace("-", "").ToLowerInvariant();
|
||||
Utils.Log($"Hash is {hash}");
|
||||
}
|
||||
var archives = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly).Where(f =>
|
||||
File.Exists(f) && Path.GetExtension(f) != ".meta" && Path.GetExtension(f) != ".xxHash" &&
|
||||
!File.Exists($"{f}.meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)));
|
||||
|
||||
var md5Response = await nexusClient.GetModInfoFromMD5(Game, hash);
|
||||
if (md5Response.Count >= 1)
|
||||
{
|
||||
var modInfo = md5Response[0].mod;
|
||||
metaString += $"modID={modInfo.mod_id}\n" +
|
||||
$"modName={modInfo.name}\nfileID={md5Response[0].file_details.file_id}";
|
||||
File.WriteAllText(f+".meta",metaString, Encoding.UTF8);
|
||||
}
|
||||
else
|
||||
{
|
||||
Error("Error while getting information from NexusMods via MD5 hash!");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (Path.GetExtension(f) != ".meta" ||
|
||||
ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)))
|
||||
return;
|
||||
|
||||
Utils.Log($"File {f} is not in ActiveArchives");
|
||||
var lines = File.ReadAllLines(f);
|
||||
if (lines.Length == 0 || !lines.Any(line => line.Contains("directURL=")))
|
||||
{
|
||||
if (lines.Length == 0)
|
||||
return;
|
||||
|
||||
lines.Do(line =>
|
||||
{
|
||||
var tag = "";
|
||||
if (line.Contains("tag="))
|
||||
tag = line.Substring("tag=".Length);
|
||||
|
||||
//if (tag != Consts.WABBAJACK_VORTEX_MANUAL)
|
||||
return;
|
||||
|
||||
//Utils.Log($"File {f} contains the {Consts.WABBAJACK_VORTEX_MANUAL} tag, adding to ActiveArchives");
|
||||
//ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
Utils.Log($"File {f} appears to not come from the Nexus, adding to ActiveArchives");
|
||||
ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
Utils.Log($"Checking for Steam Workshop Items...");
|
||||
if (!_isSteamGame || _steamGame == null || _steamGame.WorkshopItems.Count <= 0)
|
||||
return;
|
||||
|
||||
_steamGame.WorkshopItems.Do(item =>
|
||||
await archives.PMap(Queue, async f =>
|
||||
{
|
||||
var filePath = Path.Combine(DownloadsFolder, $"steamWorkshopItem_{item.ItemID}.meta");
|
||||
if (File.Exists(filePath))
|
||||
Info($"Creating meta file for {Path.GetFileName(f)}");
|
||||
var metaString = "[General]\n" +
|
||||
"repository=Nexus\n" +
|
||||
$"gameName={GameName}\n";
|
||||
string hash;
|
||||
using(var md5 = MD5.Create())
|
||||
using (var stream = File.OpenRead(f))
|
||||
{
|
||||
Utils.Log($"File {filePath} already exists, skipping...");
|
||||
return;
|
||||
Info($"Calculating hash for {Path.GetFileName(f)}");
|
||||
var cH = md5.ComputeHash(stream);
|
||||
hash = BitConverter.ToString(cH).Replace("-", "").ToLowerInvariant();
|
||||
Info($"Hash is {hash}");
|
||||
}
|
||||
|
||||
Utils.Log($"Creating meta file for {item.ItemID}");
|
||||
var metaString = "[General]\n" +
|
||||
"repository=Steam\n" +
|
||||
"installed=true\n" +
|
||||
$"gameName={GameName}\n" +
|
||||
$"steamID={_steamGame.AppId}\n" +
|
||||
$"itemID={item.ItemID}\n" +
|
||||
$"itemSize={item.Size}\n";
|
||||
try
|
||||
var md5Response = await nexusClient.GetModInfoFromMD5(Game, hash);
|
||||
if (md5Response.Count >= 1)
|
||||
{
|
||||
File.WriteAllText(filePath, metaString);
|
||||
var modInfo = md5Response[0].mod;
|
||||
metaString += $"modID={modInfo.mod_id}\n" +
|
||||
$"modName={modInfo.name}\nfileID={md5Response[0].file_details.file_id}";
|
||||
File.WriteAllText(f+".meta",metaString, Encoding.UTF8);
|
||||
}
|
||||
catch (Exception e)
|
||||
else
|
||||
{
|
||||
Utils.Error(e, $"Exception while writing to disk at {filePath}");
|
||||
Error("Error while getting information from NexusMods via MD5 hash!");
|
||||
}
|
||||
});
|
||||
}
|
||||
@ -440,20 +340,18 @@ namespace Wabbajack.Lib
|
||||
|
||||
var stack = MakeStack();
|
||||
|
||||
File.WriteAllText(Path.Combine(s, "_current_compilation_stack.yml"),
|
||||
Serialization.Serialize(stack));
|
||||
var compilationSteps = stack.ToList();
|
||||
File.WriteAllText(Path.Combine(s, "_current_compilation_stack.yml"), Serialization.Serialize(compilationSteps));
|
||||
|
||||
return stack;
|
||||
return compilationSteps;
|
||||
}
|
||||
|
||||
public override IEnumerable<ICompilationStep> MakeStack()
|
||||
{
|
||||
Utils.Log("Generating compilation stack");
|
||||
Info("Generating compilation stack");
|
||||
return new List<ICompilationStep>
|
||||
{
|
||||
new IncludePropertyFiles(this),
|
||||
new IncludeSteamWorkshopItems(this, _steamGame),
|
||||
_hasSteamWorkshopItems ? new IncludeRegex(this, "^steamWorkshopItem_\\d*\\.meta$") : null,
|
||||
new IgnoreDisabledVortexMods(this),
|
||||
new IncludeVortexDeployment(this),
|
||||
new IgnoreVortex(this),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user