using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.GitHub; using Wabbajack.Lib.Validation; using Wabbajack.VirtualFileSystem; namespace Wabbajack.Lib { public class NativeCompiler : ACompiler { public NativeCompiler(NativeCompilerSettings settings, AbsolutePath sourcePath, AbsolutePath downloadsPath, AbsolutePath outputModListPath, UpdateRequest? publishData = null) : base(5, settings.ModListName, sourcePath, downloadsPath, outputModListPath, publishData) { CompilingGame = settings.CompilingGame.MetaData(); GamePath = CompilingGame.GameLocation(); NativeSettings = settings; } public NativeCompilerSettings NativeSettings { get; set; } protected override async Task _Begin(CancellationToken cancel) { await Metrics.Send("begin_compiling", ModListName ?? "unknown"); if (cancel.IsCancellationRequested) { return false; } DesiredThreads.OnNext(DiskThreads); FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam; UpdateTracker.Reset(); UpdateTracker.NextStep("Running Preflight Checks"); await PreflightChecks(); UpdateTracker.NextStep("Gathering information"); Utils.Log($"Compiling Game: {CompilingGame.Game}"); Utils.Log("Games from setting files:"); foreach (var game in Settings.IncludedGames) { Utils.Log($"- {game}"); } Utils.Log($"VFS File Location: {VFSCacheName}"); Utils.Log($"MO2 Folder: {SourcePath}"); Utils.Log($"Downloads Folder: {DownloadsPath}"); Utils.Log($"Game Folder: {GamePath}"); var watcher = new DiskSpaceWatcher(cancel, new[] {SourcePath, DownloadsPath, GamePath, AbsolutePath.EntryPoint}, (long)2 << 31, drive => { Utils.Log($"Aborting due to low space on {drive.Name}"); Abort(); }); var watcherTask = watcher.Start(); if (cancel.IsCancellationRequested) { return false; } List roots = new List {SourcePath, GamePath, DownloadsPath}; roots.AddRange(Settings.IncludedGames.Select(g => g.MetaData().GameLocation())); UpdateTracker.NextStep("Indexing folders"); if (cancel.IsCancellationRequested) { return false; } await VFS.AddRoots(roots); UpdateTracker.NextStep("Cleaning output folder"); await ModListOutputFolder.DeleteDirectory(); if (cancel.IsCancellationRequested) { return false; } UpdateTracker.NextStep("Inferring metas for game file downloads"); await InferMetas(); if (cancel.IsCancellationRequested) { return false; } UpdateTracker.NextStep("Reindexing downloads after meta inferring"); await VFS.AddRoot(DownloadsPath); if (cancel.IsCancellationRequested) { return false; } UpdateTracker.NextStep("Pre-validating Archives"); // Find all Downloads IndexedArchives = (await DownloadsPath.EnumerateFiles() .Where(f => f.WithExtension(Consts.MetaFileExtension).Exists) .PMap(Queue, async f => new IndexedArchive(VFS.Index.ByRootPath[f]) { Name = (string)f.FileName, IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(), Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync() })).ToList(); await IndexGameFileHashes(); IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList(); await CleanInvalidArchivesAndFillState(); UpdateTracker.NextStep("Finding Install Files"); ModListOutputFolder.CreateDirectory(); var mo2Files = SourcePath.EnumerateFiles() .Where(p => p.IsFile) .Select(p => { if (!VFS.Index.ByRootPath.ContainsKey(p)) { Utils.Log($"WELL THERE'S YOUR PROBLEM: {p} {VFS.Index.ByRootPath.Count}"); } return new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(SourcePath)); }); // If Game Folder Files exists, ignore the game folder IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren) .OrderBy(f => f.NestingFactor) .GroupBy(f => f.Hash) .ToDictionary(f => f.Key, f => f.AsEnumerable()); AllFiles.SetTo(mo2Files .DistinctBy(f => f.Path)); Info($"Found {AllFiles.Count} files to build into mod list"); if (cancel.IsCancellationRequested) { return false; } UpdateTracker.NextStep("Verifying destinations"); var dups = AllFiles.GroupBy(f => f.Path) .Where(fs => fs.Count() > 1) .Select(fs => { Utils.Log( $"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}"); return fs; }).ToList(); if (dups.Count > 0) { Error($"Found {dups.Count} duplicates, exiting"); } if (cancel.IsCancellationRequested) { return false; } UpdateTracker.NextStep("Loading INIs"); ArchivesByFullPath = IndexedArchives.ToDictionary(a => a.File.AbsoluteName); if (cancel.IsCancellationRequested) { return false; } var stack = MakeStack(); UpdateTracker.NextStep("Running Compilation Stack"); var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f)); // Add the extra files that were generated by the stack if (cancel.IsCancellationRequested) { return false; } var noMatch = results.OfType().ToArray(); PrintNoMatches(noMatch); if (CheckForNoMatchExit(noMatch)) { return false; } foreach (var ignored in results.OfType()) { Utils.Log($"Ignored {ignored.To} because {ignored.Reason}"); } InstallDirectives.SetTo(results.Where(i => !(i is IgnoredDirectly))); Info("Getting Nexus api_key, please click authorize if a browser window appears"); UpdateTracker.NextStep("Building Patches"); await BuildPatches(); UpdateTracker.NextStep("Gathering Archives"); await GatherArchives(); UpdateTracker.NextStep("Gathering Metadata"); await GatherMetaData(); ModList = new ModList { GameType = CompilingGame.Game, WabbajackVersion = Consts.CurrentMinimumWabbajackVersion, Archives = SelectedArchives.ToList(), ModManager = ModManager.MO2, Directives = InstallDirectives, Name = ModListName ?? "untitled", Author = ModListAuthor ?? "", Description = ModListDescription ?? "", Readme = ModlistReadme ?? "", Image = ModListImage != default ? ModListImage.FileName : default, Website = !string.IsNullOrWhiteSpace(ModListWebsite) ? new Uri(ModListWebsite) : null, Version = ModlistVersion ?? new Version(1, 0, 0, 0), IsNSFW = ModlistIsNSFW }; UpdateTracker.NextStep("Including required files"); await InlineFiles(); UpdateTracker.NextStep("Running Validation"); await ValidateModlist.RunValidation(ModList); UpdateTracker.NextStep("Generating Report"); GenerateManifest(); UpdateTracker.NextStep("Exporting Modlist"); await ExportModList(); UpdateTracker.NextStep("Publishing Modlist"); await PublishModlist(); ResetMembers(); UpdateTracker.NextStep("Done Building Modlist"); return true; } /// /// Clear references to lists that hold a lot of data. /// private void ResetMembers() { AllFiles = new List(); InstallDirectives = new List(); SelectedArchives = new List(); } public override AbsolutePath GamePath { get; } public override IEnumerable GetStack() { return MakeStack(); } public override IEnumerable MakeStack() { List steps = NativeSettings.CompilationSteps.Select(InterpretStep).ToList(); steps.Add(new DropAll(this)); return steps; } public ICompilationStep InterpretStep(string[] step) { return step[0] switch { "IgnoreStartsWith" => new IgnoreStartsWith(this, step[1]), "IgnoreTaggedFolders" => new IgnoreTaggedFolders(this, Consts.WABBAJACK_IGNORE), "IgnoreTaggedFiles" => new IgnoreTaggedFiles(this, Consts.WABBAJACK_IGNORE_FILES), "IncludeTaggedFolders" => new IncludeTaggedFolders(this, Consts.WABBAJACK_INCLUDE), "IncludeConfigs" => new IncludeAllConfigs(this), "IncludeDirectMatches" => new DirectMatch(this), "IncludePatches" => new IncludePatches(this), "IncludeUnmatchedFilesInTaggedFolders" => new IncludeTaggedFolders(this, Consts.WABBAJACK_NOMATCH_INCLUDE), "IncludeUnmatchedTaggedFiles" => new IncludeTaggedFiles(this, Consts.WABBAJACK_NOMATCH_INCLUDE_FILES), _ => throw new ArgumentException($"No interpretation for step {step[0]}") }; } } }