2020-04-04 18:26:14 +00:00
|
|
|
|
using System;
|
2019-11-03 15:26:51 +00:00
|
|
|
|
using System.Collections.Generic;
|
2020-08-11 12:31:58 +00:00
|
|
|
|
using System.Diagnostics;
|
2020-06-20 22:51:47 +00:00
|
|
|
|
using System.Diagnostics.CodeAnalysis;
|
2019-11-17 15:00:33 +00:00
|
|
|
|
using System.IO;
|
|
|
|
|
using System.IO.Compression;
|
2019-11-03 15:26:51 +00:00
|
|
|
|
using System.Linq;
|
2019-11-15 23:13:27 +00:00
|
|
|
|
using System.Reactive.Subjects;
|
2020-08-11 12:31:58 +00:00
|
|
|
|
using System.Reflection;
|
2020-05-02 21:09:29 +00:00
|
|
|
|
using System.Text;
|
2020-12-31 06:44:42 +00:00
|
|
|
|
using System.Threading;
|
2019-12-04 01:26:26 +00:00
|
|
|
|
using System.Threading.Tasks;
|
2020-12-31 06:44:42 +00:00
|
|
|
|
using Org.BouncyCastle.Asn1.Cms;
|
2019-11-03 16:43:43 +00:00
|
|
|
|
using Wabbajack.Common;
|
2019-11-03 15:26:51 +00:00
|
|
|
|
using Wabbajack.Lib.CompilationSteps;
|
2019-11-17 15:00:33 +00:00
|
|
|
|
using Wabbajack.Lib.Downloaders;
|
|
|
|
|
using Wabbajack.Lib.ModListRegistry;
|
2019-11-15 13:06:34 +00:00
|
|
|
|
using Wabbajack.VirtualFileSystem;
|
2019-11-03 15:26:51 +00:00
|
|
|
|
|
|
|
|
|
namespace Wabbajack.Lib
|
|
|
|
|
{
|
2019-11-17 23:48:32 +00:00
|
|
|
|
public abstract class ACompiler : ABatchProcessor
|
2019-11-03 15:26:51 +00:00
|
|
|
|
{
|
2020-10-18 13:18:16 +00:00
|
|
|
|
protected readonly Subject<(string, float)> _progressUpdates = new Subject<(string, float)>();
|
|
|
|
|
|
|
|
|
|
public List<IndexedArchive> IndexedArchives = new List<IndexedArchive>();
|
|
|
|
|
|
|
|
|
|
public Dictionary<Hash, IEnumerable<VirtualFile>> IndexedFiles =
|
|
|
|
|
new Dictionary<Hash, IEnumerable<VirtualFile>>();
|
|
|
|
|
|
|
|
|
|
public ModList ModList = new ModList();
|
|
|
|
|
public AbsolutePath ModListImage;
|
|
|
|
|
public bool ModlistIsNSFW;
|
|
|
|
|
|
|
|
|
|
public string? ModListName, ModListAuthor, ModListDescription, ModListWebsite, ModlistReadme;
|
|
|
|
|
public Version? ModlistVersion;
|
|
|
|
|
protected Version? WabbajackVersion;
|
|
|
|
|
|
|
|
|
|
public ACompiler(int steps, string modlistName, AbsolutePath sourcePath, AbsolutePath downloadsPath,
|
|
|
|
|
AbsolutePath outputModListName)
|
|
|
|
|
: base(steps)
|
|
|
|
|
{
|
|
|
|
|
SourcePath = sourcePath;
|
|
|
|
|
DownloadsPath = downloadsPath;
|
|
|
|
|
ModListName = modlistName;
|
|
|
|
|
ModListOutputFile = outputModListName;
|
|
|
|
|
//set in MainWindowVM
|
|
|
|
|
WabbajackVersion = Consts.CurrentMinimumWabbajackVersion;
|
|
|
|
|
Settings = new CompilerSettings();
|
|
|
|
|
ModListOutputFolder = AbsolutePath.EntryPoint.Combine("output_folder", Guid.NewGuid().ToString());
|
|
|
|
|
CompilingGame = new GameMetaData();
|
|
|
|
|
}
|
|
|
|
|
|
2020-10-18 13:06:22 +00:00
|
|
|
|
/// <summary>
|
|
|
|
|
/// Set to true to include game files during compilation, only ever disabled
|
|
|
|
|
/// in testing (to speed up tests)
|
|
|
|
|
/// </summary>
|
|
|
|
|
public bool UseGamePaths { get; set; } = true;
|
2020-10-18 13:18:16 +00:00
|
|
|
|
|
2020-10-18 13:06:22 +00:00
|
|
|
|
public CompilerSettings Settings { get; set; }
|
2019-11-17 14:30:06 +00:00
|
|
|
|
|
2020-10-18 13:00:55 +00:00
|
|
|
|
public AbsolutePath VFSCacheName =>
|
|
|
|
|
Consts.LocalAppDataPath.Combine(
|
|
|
|
|
$"vfs_compile_cache-2-{Path.Combine((string)SourcePath ?? "Unknown", "ModOrganizer.exe").StringSha256Hex()}.bin");
|
|
|
|
|
|
2020-02-24 15:24:13 +00:00
|
|
|
|
//protected string VFSCacheName => Path.Combine(Consts.LocalAppDataPath, $"vfs_compile_cache.bin");
|
2019-11-15 23:13:27 +00:00
|
|
|
|
/// <summary>
|
|
|
|
|
/// A stream of tuples of ("Update Title", 0.25) which represent the name of the current task
|
|
|
|
|
/// and the current progress.
|
|
|
|
|
/// </summary>
|
|
|
|
|
public IObservable<(string, float)> ProgressUpdates => _progressUpdates;
|
|
|
|
|
|
2020-03-25 12:47:25 +00:00
|
|
|
|
public abstract AbsolutePath GamePath { get; }
|
2020-10-18 13:06:22 +00:00
|
|
|
|
public Dictionary<Game, HashSet<Hash>> GameHashes { get; set; } = new Dictionary<Game, HashSet<Hash>>();
|
|
|
|
|
public Dictionary<Hash, Game[]> GamesWithHashes { get; set; } = new Dictionary<Hash, Game[]>();
|
2020-10-18 13:18:16 +00:00
|
|
|
|
|
|
|
|
|
public AbsolutePath SourcePath { get; }
|
|
|
|
|
public AbsolutePath DownloadsPath { get; }
|
|
|
|
|
|
2020-10-18 13:16:27 +00:00
|
|
|
|
public GameMetaData CompilingGame { get; set; }
|
2020-10-18 12:59:49 +00:00
|
|
|
|
|
2020-10-18 13:13:01 +00:00
|
|
|
|
public AbsolutePath ModListOutputFolder { get; }
|
|
|
|
|
public AbsolutePath ModListOutputFile { get; }
|
2019-11-03 16:43:43 +00:00
|
|
|
|
|
2020-01-20 01:36:09 +00:00
|
|
|
|
public bool IgnoreMissingFiles { get; set; }
|
|
|
|
|
|
2020-04-12 18:55:59 +00:00
|
|
|
|
public List<Archive> SelectedArchives { get; protected set; } = new List<Archive>();
|
|
|
|
|
public List<Directive> InstallDirectives { get; protected set; } = new List<Directive>();
|
|
|
|
|
public List<RawSourceFile> AllFiles { get; protected set; } = new List<RawSourceFile>();
|
2019-11-15 14:19:39 +00:00
|
|
|
|
|
2020-10-18 13:18:16 +00:00
|
|
|
|
public Dictionary<AbsolutePath, IndexedArchive> ArchivesByFullPath { get; set; } =
|
|
|
|
|
new Dictionary<AbsolutePath, IndexedArchive>();
|
2020-04-04 18:26:14 +00:00
|
|
|
|
|
2020-01-20 01:36:09 +00:00
|
|
|
|
public static void Info(string msg)
|
2019-11-17 15:00:33 +00:00
|
|
|
|
{
|
|
|
|
|
Utils.Log(msg);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void Status(string msg)
|
|
|
|
|
{
|
2020-02-08 04:35:08 +00:00
|
|
|
|
Queue.Report(msg, Percent.Zero);
|
2019-11-17 15:00:33 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-01-20 01:36:09 +00:00
|
|
|
|
public static void Error(string msg)
|
2019-11-17 15:00:33 +00:00
|
|
|
|
{
|
|
|
|
|
Utils.Log(msg);
|
|
|
|
|
throw new Exception(msg);
|
|
|
|
|
}
|
|
|
|
|
|
2020-03-25 12:47:25 +00:00
|
|
|
|
internal RelativePath IncludeId()
|
2019-11-17 15:00:33 +00:00
|
|
|
|
{
|
2020-03-25 12:47:25 +00:00
|
|
|
|
return RelativePath.RandomFileName();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
internal async Task<RelativePath> IncludeFile(byte[] data)
|
|
|
|
|
{
|
|
|
|
|
var id = IncludeId();
|
|
|
|
|
await ModListOutputFolder.Combine(id).WriteAllBytesAsync(data);
|
2019-11-17 15:00:33 +00:00
|
|
|
|
return id;
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-25 17:34:25 +00:00
|
|
|
|
internal AbsolutePath IncludeFile(out RelativePath id)
|
2020-03-04 05:23:08 +00:00
|
|
|
|
{
|
2020-03-25 12:47:25 +00:00
|
|
|
|
id = IncludeId();
|
2020-05-25 17:34:25 +00:00
|
|
|
|
return ModListOutputFolder.Combine(id);
|
2020-03-04 05:23:08 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-03-25 12:47:25 +00:00
|
|
|
|
internal async Task<RelativePath> IncludeFile(string data)
|
|
|
|
|
{
|
|
|
|
|
var id = IncludeId();
|
|
|
|
|
await ModListOutputFolder.Combine(id).WriteAllTextAsync(data);
|
|
|
|
|
return id;
|
|
|
|
|
}
|
2020-10-18 13:18:16 +00:00
|
|
|
|
|
2020-09-05 19:36:44 +00:00
|
|
|
|
internal async Task<RelativePath> IncludeFile(Stream data)
|
2019-11-17 15:00:33 +00:00
|
|
|
|
{
|
2020-03-25 12:47:25 +00:00
|
|
|
|
var id = IncludeId();
|
2020-09-05 19:36:44 +00:00
|
|
|
|
await ModListOutputFolder.Combine(id).WriteAllAsync(data);
|
2019-11-17 15:00:33 +00:00
|
|
|
|
return id;
|
|
|
|
|
}
|
2020-10-18 13:18:16 +00:00
|
|
|
|
|
2020-09-05 19:36:44 +00:00
|
|
|
|
internal async Task<RelativePath> IncludeFile(AbsolutePath data)
|
|
|
|
|
{
|
|
|
|
|
await using var stream = await data.OpenRead();
|
|
|
|
|
return await IncludeFile(stream);
|
|
|
|
|
}
|
2019-11-17 15:00:33 +00:00
|
|
|
|
|
2020-10-18 13:18:16 +00:00
|
|
|
|
|
2020-05-02 21:09:29 +00:00
|
|
|
|
internal async Task<(RelativePath, AbsolutePath)> IncludeString(string str)
|
|
|
|
|
{
|
|
|
|
|
var id = IncludeId();
|
|
|
|
|
var fullPath = ModListOutputFolder.Combine(id);
|
|
|
|
|
await fullPath.WriteAllTextAsync(str);
|
|
|
|
|
return (id, fullPath);
|
|
|
|
|
}
|
|
|
|
|
|
2020-03-04 12:10:49 +00:00
|
|
|
|
public async Task<bool> GatherMetaData()
|
|
|
|
|
{
|
|
|
|
|
Utils.Log($"Getting meta data for {SelectedArchives.Count} archives");
|
|
|
|
|
await SelectedArchives.PMap(Queue, async a =>
|
|
|
|
|
{
|
|
|
|
|
if (a.State is IMetaState metaState)
|
|
|
|
|
{
|
2020-04-23 10:38:43 +00:00
|
|
|
|
if (metaState.URL == null || metaState.URL == Consts.WabbajackOrg)
|
2020-04-04 17:53:05 +00:00
|
|
|
|
return;
|
|
|
|
|
|
2020-03-04 12:10:49 +00:00
|
|
|
|
var b = await metaState.LoadMetaData();
|
|
|
|
|
Utils.Log(b
|
|
|
|
|
? $"Getting meta data for {a.Name} was successful!"
|
|
|
|
|
: $"Getting meta data for {a.Name} failed!");
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
Utils.Log($"Archive {a.Name} is not an AbstractMetaState!");
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
2020-10-18 13:18:16 +00:00
|
|
|
|
|
2020-10-18 13:06:22 +00:00
|
|
|
|
protected async Task IndexGameFileHashes()
|
|
|
|
|
{
|
|
|
|
|
if (UseGamePaths)
|
|
|
|
|
{
|
2020-10-18 19:03:50 +00:00
|
|
|
|
foreach (var ag in Settings.IncludedGames.Cons(CompilingGame.Game))
|
2020-10-18 13:06:22 +00:00
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
var files = await ClientAPI.GetExistingGameFiles(Queue, ag);
|
|
|
|
|
Utils.Log($"Including {files.Length} stock game files from {ag} as download sources");
|
|
|
|
|
GameHashes[ag] = files.Select(f => f.Hash).ToHashSet();
|
|
|
|
|
|
|
|
|
|
IndexedArchives.AddRange(files.Select(f =>
|
|
|
|
|
{
|
|
|
|
|
var meta = f.State.GetMetaIniString();
|
|
|
|
|
var ini = meta.LoadIniString();
|
2020-10-18 13:18:16 +00:00
|
|
|
|
var state = (GameFileSourceDownloader.State)f.State;
|
2020-10-18 13:06:22 +00:00
|
|
|
|
return new IndexedArchive(
|
|
|
|
|
VFS.Index.ByRootPath[ag.MetaData().GameLocation().Combine(state.GameFile)])
|
|
|
|
|
{
|
|
|
|
|
IniData = ini, Meta = meta
|
|
|
|
|
};
|
|
|
|
|
}));
|
|
|
|
|
}
|
|
|
|
|
catch (Exception e)
|
|
|
|
|
{
|
|
|
|
|
Utils.Error(e, "Unable to find existing game files, skipping.");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
GamesWithHashes = GameHashes.SelectMany(g => g.Value.Select(h => (g, h)))
|
|
|
|
|
.GroupBy(gh => gh.h)
|
|
|
|
|
.ToDictionary(gh => gh.Key, gh => gh.Select(p => p.g.Key).ToArray());
|
|
|
|
|
}
|
|
|
|
|
}
|
2020-10-18 13:18:16 +00:00
|
|
|
|
|
2020-10-18 13:07:18 +00:00
|
|
|
|
protected async Task CleanInvalidArchivesAndFillState()
|
|
|
|
|
{
|
|
|
|
|
var remove = (await IndexedArchives.PMap(Queue, async a =>
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
a.State = (await ResolveArchive(a)).State;
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
catch
|
|
|
|
|
{
|
|
|
|
|
return a;
|
|
|
|
|
}
|
|
|
|
|
})).NotNull().ToHashSet();
|
|
|
|
|
|
|
|
|
|
if (remove.Count == 0)
|
|
|
|
|
{
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Utils.Log(
|
|
|
|
|
$"Removing {remove.Count} archives from the compilation state, this is probably not an issue but reference this if you have compilation failures");
|
|
|
|
|
remove.Do(r => Utils.Log($"Resolution failed for: ({r.File.Size} {r.File.Hash}) {r.File.FullPath}"));
|
|
|
|
|
IndexedArchives.RemoveAll(a => remove.Contains(a));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
protected async Task InferMetas()
|
|
|
|
|
{
|
|
|
|
|
async Task<bool> HasInvalidMeta(AbsolutePath filename)
|
|
|
|
|
{
|
|
|
|
|
var metaname = filename.WithExtension(Consts.MetaFileExtension);
|
|
|
|
|
if (!metaname.Exists)
|
|
|
|
|
{
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
return await DownloadDispatcher.ResolveArchive(metaname.LoadIniFile()) == null;
|
|
|
|
|
}
|
|
|
|
|
catch (Exception e)
|
|
|
|
|
{
|
|
|
|
|
Utils.ErrorThrow(e, $"Exception while checking meta {filename}");
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var to_find = (await DownloadsPath.EnumerateFiles()
|
|
|
|
|
.Where(f => f.Extension != Consts.MetaFileExtension && f.Extension != Consts.HashFileExtension)
|
|
|
|
|
.PMap(Queue, async f => await HasInvalidMeta(f) ? f : default))
|
|
|
|
|
.Where(f => f.Exists)
|
|
|
|
|
.ToList();
|
|
|
|
|
|
|
|
|
|
if (to_find.Count == 0)
|
|
|
|
|
{
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Utils.Log($"Attempting to infer {to_find.Count} metas from the server.");
|
|
|
|
|
|
|
|
|
|
await to_find.PMap(Queue, async f =>
|
|
|
|
|
{
|
|
|
|
|
var vf = VFS.Index.ByRootPath[f];
|
|
|
|
|
|
|
|
|
|
var meta = await ClientAPI.InferDownloadState(vf.Hash);
|
|
|
|
|
|
|
|
|
|
if (meta == null)
|
|
|
|
|
{
|
|
|
|
|
await vf.AbsoluteName.WithExtension(Consts.MetaFileExtension).WriteAllLinesAsync(
|
|
|
|
|
"[General]",
|
|
|
|
|
"unknownArchive=true");
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Utils.Log($"Inferred .meta for {vf.FullPath.FileName}, writing to disk");
|
|
|
|
|
await vf.AbsoluteName.WithExtension(Consts.MetaFileExtension)
|
|
|
|
|
.WriteAllTextAsync(meta.GetMetaIniString());
|
|
|
|
|
});
|
|
|
|
|
}
|
2020-03-04 12:10:49 +00:00
|
|
|
|
|
2020-10-18 13:18:16 +00:00
|
|
|
|
protected async Task ExportModList()
|
2019-11-17 15:00:33 +00:00
|
|
|
|
{
|
2020-01-13 21:11:07 +00:00
|
|
|
|
Utils.Log($"Exporting ModList to {ModListOutputFile}");
|
2019-11-17 15:00:33 +00:00
|
|
|
|
|
2020-01-13 21:11:07 +00:00
|
|
|
|
// Modify readme and ModList image to relative paths if they exist
|
2020-03-25 22:30:43 +00:00
|
|
|
|
if (ModListImage.Exists)
|
2019-12-03 04:56:06 +00:00
|
|
|
|
{
|
2020-03-25 22:30:43 +00:00
|
|
|
|
ModList.Image = (RelativePath)"modlist-image.png";
|
2019-12-03 04:56:06 +00:00
|
|
|
|
}
|
2019-12-20 07:14:43 +00:00
|
|
|
|
|
2020-10-18 13:18:16 +00:00
|
|
|
|
await using (var of = await ModListOutputFolder.Combine("modlist").Create())
|
2020-04-06 20:48:54 +00:00
|
|
|
|
ModList.ToJson(of);
|
2019-11-17 15:00:33 +00:00
|
|
|
|
|
2020-08-05 00:34:09 +00:00
|
|
|
|
await ModListOutputFolder.Combine("sig")
|
|
|
|
|
.WriteAllBytesAsync((await ModListOutputFolder.Combine("modlist").FileHashAsync()).ToArray());
|
|
|
|
|
|
2020-06-14 13:13:29 +00:00
|
|
|
|
await ClientAPI.SendModListDefinition(ModList);
|
|
|
|
|
|
2020-05-26 11:31:11 +00:00
|
|
|
|
await ModListOutputFile.DeleteAsync();
|
2019-11-17 15:00:33 +00:00
|
|
|
|
|
2020-06-13 18:32:30 +00:00
|
|
|
|
await using (var fs = await ModListOutputFile.Create())
|
2019-11-17 15:00:33 +00:00
|
|
|
|
{
|
2020-06-13 18:32:30 +00:00
|
|
|
|
using var za = new ZipArchive(fs, ZipArchiveMode.Create);
|
2020-10-18 13:18:16 +00:00
|
|
|
|
|
2020-06-13 18:32:30 +00:00
|
|
|
|
await ModListOutputFolder.EnumerateFiles()
|
|
|
|
|
.DoProgress("Compressing ModList",
|
2020-10-18 13:18:16 +00:00
|
|
|
|
async f =>
|
|
|
|
|
{
|
|
|
|
|
var ze = za.CreateEntry((string)f.FileName);
|
|
|
|
|
await using var os = ze.Open();
|
|
|
|
|
await using var ins = await f.OpenRead();
|
|
|
|
|
await ins.CopyToAsync(os);
|
|
|
|
|
});
|
2020-06-13 18:32:30 +00:00
|
|
|
|
|
|
|
|
|
// Copy in modimage
|
|
|
|
|
if (ModListImage.Exists)
|
|
|
|
|
{
|
|
|
|
|
var ze = za.CreateEntry((string)ModList.Image);
|
|
|
|
|
await using var os = ze.Open();
|
|
|
|
|
await using var ins = await ModListImage.OpenRead();
|
|
|
|
|
await ins.CopyToAsync(os);
|
2019-11-17 15:00:33 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Utils.Log("Exporting ModList metadata");
|
2019-11-29 05:52:33 +00:00
|
|
|
|
var metadata = new DownloadMetadata
|
2019-11-17 15:00:33 +00:00
|
|
|
|
{
|
2020-03-25 22:30:43 +00:00
|
|
|
|
Size = ModListOutputFile.Size,
|
2020-05-25 17:34:25 +00:00
|
|
|
|
Hash = await ModListOutputFile.FileHashAsync(),
|
2019-11-17 15:00:33 +00:00
|
|
|
|
NumberOfArchives = ModList.Archives.Count,
|
|
|
|
|
SizeOfArchives = ModList.Archives.Sum(a => a.Size),
|
|
|
|
|
NumberOfInstalledFiles = ModList.Directives.Count,
|
|
|
|
|
SizeOfInstalledFiles = ModList.Directives.Sum(a => a.Size)
|
|
|
|
|
};
|
2020-04-06 20:48:54 +00:00
|
|
|
|
metadata.ToJson(ModListOutputFile + ".meta.json");
|
2019-11-03 15:26:51 +00:00
|
|
|
|
|
2019-11-17 15:00:33 +00:00
|
|
|
|
Utils.Log("Removing ModList staging folder");
|
2020-04-01 23:59:22 +00:00
|
|
|
|
await Utils.DeleteDirectory(ModListOutputFolder);
|
2019-11-17 15:00:33 +00:00
|
|
|
|
}
|
2020-10-18 13:18:16 +00:00
|
|
|
|
|
|
|
|
|
/// <summary>
|
2020-10-18 13:16:27 +00:00
|
|
|
|
/// Fills in the Patch fields in files that require them
|
|
|
|
|
/// </summary>
|
|
|
|
|
protected async Task BuildPatches()
|
|
|
|
|
{
|
|
|
|
|
Info("Gathering patch files");
|
|
|
|
|
|
|
|
|
|
var toBuild = InstallDirectives.OfType<PatchedFromArchive>()
|
|
|
|
|
.Where(p => p.Choices.Length > 0)
|
|
|
|
|
.SelectMany(p => p.Choices.Select(c => new PatchedFromArchive
|
|
|
|
|
{
|
|
|
|
|
To = p.To,
|
|
|
|
|
Hash = p.Hash,
|
|
|
|
|
ArchiveHashPath = c.MakeRelativePaths(),
|
|
|
|
|
FromFile = c,
|
|
|
|
|
Size = p.Size
|
|
|
|
|
}))
|
|
|
|
|
.ToArray();
|
|
|
|
|
|
|
|
|
|
if (toBuild.Length == 0)
|
|
|
|
|
{
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Extract all the source files
|
|
|
|
|
var indexed = toBuild.GroupBy(f => VFS.Index.FileForArchiveHashPath(f.ArchiveHashPath))
|
|
|
|
|
.ToDictionary(f => f.Key);
|
|
|
|
|
await VFS.Extract(Queue, indexed.Keys.ToHashSet(),
|
|
|
|
|
async (vf, sf) =>
|
|
|
|
|
{
|
|
|
|
|
// For each, extract the destination
|
|
|
|
|
var matches = indexed[vf];
|
|
|
|
|
using var iqueue = new WorkQueue(1);
|
|
|
|
|
foreach (var match in matches)
|
|
|
|
|
{
|
|
|
|
|
var destFile = FindDestFile(match.To);
|
|
|
|
|
// Build the patch
|
|
|
|
|
await VFS.Extract(iqueue, new[] {destFile}.ToHashSet(),
|
|
|
|
|
async (destvf, destsfn) =>
|
|
|
|
|
{
|
|
|
|
|
Info($"Patching {match.To}");
|
|
|
|
|
Status($"Patching {match.To}");
|
|
|
|
|
await using var srcStream = await sf.GetStream();
|
|
|
|
|
await using var destStream = await destsfn.GetStream();
|
|
|
|
|
var patchSize =
|
|
|
|
|
await Utils.CreatePatchCached(srcStream, vf.Hash, destStream, destvf.Hash);
|
|
|
|
|
Info($"Patch size {patchSize} for {match.To}");
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// Load in the patches
|
|
|
|
|
await InstallDirectives.OfType<PatchedFromArchive>()
|
|
|
|
|
.Where(p => p.PatchID == default)
|
|
|
|
|
.PMap(Queue, async pfa =>
|
|
|
|
|
{
|
|
|
|
|
var patches = pfa.Choices
|
|
|
|
|
.Select(c => (Utils.TryGetPatch(c.Hash, pfa.Hash, out var data), data, c))
|
|
|
|
|
.ToArray();
|
|
|
|
|
|
|
|
|
|
// Pick the best patch
|
|
|
|
|
if (patches.All(p => p.Item1))
|
|
|
|
|
{
|
|
|
|
|
var (_, bytes, file) = IncludePatches.PickPatch(this, patches);
|
|
|
|
|
pfa.FromFile = file;
|
|
|
|
|
pfa.FromHash = file.Hash;
|
|
|
|
|
pfa.ArchiveHashPath = file.MakeRelativePaths();
|
|
|
|
|
pfa.PatchID = await IncludeFile(bytes!);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
var firstFailedPatch =
|
|
|
|
|
InstallDirectives.OfType<PatchedFromArchive>().FirstOrDefault(f => f.PatchID == default);
|
|
|
|
|
if (firstFailedPatch != null)
|
|
|
|
|
{
|
|
|
|
|
Utils.Log("Missing data from failed patch, starting data dump");
|
|
|
|
|
Utils.Log($"Dest File: {firstFailedPatch.To}");
|
|
|
|
|
Utils.Log($"Options ({firstFailedPatch.Choices.Length}:");
|
|
|
|
|
foreach (var choice in firstFailedPatch.Choices)
|
|
|
|
|
{
|
|
|
|
|
Utils.Log($" {choice.FullPath}");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Error(
|
|
|
|
|
$"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private VirtualFile FindDestFile(RelativePath to)
|
|
|
|
|
{
|
|
|
|
|
var abs = to.RelativeTo(SourcePath);
|
|
|
|
|
if (abs.Exists)
|
|
|
|
|
{
|
|
|
|
|
return VFS.Index.ByRootPath[abs];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (to.StartsWith(Consts.BSACreationDir))
|
|
|
|
|
{
|
|
|
|
|
var bsaId = (RelativePath)((string)to).Split('\\')[1];
|
|
|
|
|
var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsaId);
|
|
|
|
|
var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray());
|
|
|
|
|
|
|
|
|
|
return VFS.Index.ByRootPath[SourcePath.Combine(bsa.To)].Children.First(c => c.RelativeName == find);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
throw new ArgumentException($"Couldn't load data for {to}");
|
|
|
|
|
}
|
2019-11-17 15:00:33 +00:00
|
|
|
|
|
2020-02-01 12:13:12 +00:00
|
|
|
|
public void GenerateManifest()
|
2019-11-17 15:00:33 +00:00
|
|
|
|
{
|
2020-02-01 12:13:12 +00:00
|
|
|
|
var manifest = new Manifest(ModList);
|
2020-04-06 20:48:54 +00:00
|
|
|
|
manifest.ToJson(ModListOutputFile + ".manifest.json");
|
2019-11-17 15:00:33 +00:00
|
|
|
|
}
|
|
|
|
|
|
2019-12-04 01:26:26 +00:00
|
|
|
|
public async Task GatherArchives()
|
2019-11-17 15:00:33 +00:00
|
|
|
|
{
|
|
|
|
|
Info("Building a list of archives based on the files required");
|
|
|
|
|
|
2020-03-22 15:50:53 +00:00
|
|
|
|
var hashes = InstallDirectives.OfType<FromArchive>()
|
2020-03-28 02:54:14 +00:00
|
|
|
|
.Select(a => a.ArchiveHashPath.BaseHash)
|
2019-11-17 15:00:33 +00:00
|
|
|
|
.Distinct();
|
|
|
|
|
|
|
|
|
|
var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified)
|
|
|
|
|
.GroupBy(f => f.File.Hash)
|
|
|
|
|
.ToDictionary(f => f.Key, f => f.First());
|
|
|
|
|
|
2020-04-09 21:05:07 +00:00
|
|
|
|
SelectedArchives.SetTo(await hashes.PMap(Queue, hash => ResolveArchive(hash, archives)));
|
2019-11-17 15:00:33 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-03-22 15:50:53 +00:00
|
|
|
|
public async Task<Archive> ResolveArchive(Hash hash, IDictionary<Hash, IndexedArchive> archives)
|
2019-11-17 15:00:33 +00:00
|
|
|
|
{
|
2020-03-22 15:50:53 +00:00
|
|
|
|
if (archives.TryGetValue(hash, out var found))
|
2019-11-17 15:00:33 +00:00
|
|
|
|
{
|
2019-12-13 00:40:21 +00:00
|
|
|
|
return await ResolveArchive(found);
|
2019-12-12 23:24:27 +00:00
|
|
|
|
}
|
2019-11-17 15:00:33 +00:00
|
|
|
|
|
2020-04-09 17:17:24 +00:00
|
|
|
|
throw new ArgumentException($"No match found for Archive sha: {hash.ToBase64()} this shouldn't happen");
|
2019-12-12 23:24:27 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-06-20 22:51:47 +00:00
|
|
|
|
public async Task<Archive> ResolveArchive([NotNull] IndexedArchive archive)
|
2019-12-12 23:24:27 +00:00
|
|
|
|
{
|
2020-10-18 13:18:16 +00:00
|
|
|
|
if (!string.IsNullOrWhiteSpace(archive.Name))
|
2020-06-20 22:51:47 +00:00
|
|
|
|
Utils.Status($"Checking link for {archive.Name}", alsoLog: true);
|
2019-12-22 18:14:48 +00:00
|
|
|
|
|
2019-12-12 23:24:27 +00:00
|
|
|
|
if (archive.IniData == null)
|
|
|
|
|
Error(
|
|
|
|
|
$"No download metadata found for {archive.Name}, please use MO2 to query info or add a .meta file and try again.");
|
2019-11-17 15:00:33 +00:00
|
|
|
|
|
2020-04-10 01:29:53 +00:00
|
|
|
|
var result = new Archive(await DownloadDispatcher.ResolveArchive(archive.IniData));
|
2019-11-17 15:00:33 +00:00
|
|
|
|
|
2019-12-12 23:24:27 +00:00
|
|
|
|
if (result.State == null)
|
|
|
|
|
Error($"{archive.Name} could not be handled by any of the downloaders");
|
2019-11-17 15:00:33 +00:00
|
|
|
|
|
2020-06-20 22:51:47 +00:00
|
|
|
|
result.Name = archive.Name ?? "";
|
2019-12-12 23:24:27 +00:00
|
|
|
|
result.Hash = archive.File.Hash;
|
|
|
|
|
result.Size = archive.File.Size;
|
2019-11-17 15:00:33 +00:00
|
|
|
|
|
2020-04-23 10:38:43 +00:00
|
|
|
|
await result.State!.GetDownloader().Prepare();
|
|
|
|
|
|
2020-12-31 06:44:42 +00:00
|
|
|
|
var token = new CancellationTokenSource();
|
|
|
|
|
token.CancelAfter(Consts.MaxVerifyTime);
|
|
|
|
|
if (result.State != null && !await result.State.Verify(result, token.Token))
|
2019-12-12 23:24:27 +00:00
|
|
|
|
Error(
|
|
|
|
|
$"Unable to resolve link for {archive.Name}. If this is hosted on the Nexus the file may have been removed.");
|
2019-11-17 15:00:33 +00:00
|
|
|
|
|
2020-04-30 21:43:54 +00:00
|
|
|
|
result.Meta = string.Join("\n", result.State!.GetMetaIni());
|
|
|
|
|
|
2020-10-18 13:18:16 +00:00
|
|
|
|
|
2019-12-12 23:24:27 +00:00
|
|
|
|
return result;
|
2019-11-17 15:00:33 +00:00
|
|
|
|
}
|
2019-11-03 15:26:51 +00:00
|
|
|
|
|
2019-12-04 01:26:26 +00:00
|
|
|
|
public async Task<Directive> RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
|
2019-11-17 15:00:33 +00:00
|
|
|
|
{
|
|
|
|
|
Utils.Status($"Compiling {source.Path}");
|
|
|
|
|
foreach (var step in stack)
|
|
|
|
|
{
|
2019-12-04 01:26:26 +00:00
|
|
|
|
var result = await step.Run(source);
|
2019-11-17 15:00:33 +00:00
|
|
|
|
if (result != null) return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
throw new InvalidDataException("Data fell out of the compilation stack");
|
|
|
|
|
}
|
|
|
|
|
|
2019-11-03 15:26:51 +00:00
|
|
|
|
public abstract IEnumerable<ICompilationStep> GetStack();
|
|
|
|
|
public abstract IEnumerable<ICompilationStep> MakeStack();
|
2020-01-20 01:36:09 +00:00
|
|
|
|
|
|
|
|
|
public static void PrintNoMatches(ICollection<NoMatch> noMatches)
|
|
|
|
|
{
|
|
|
|
|
const int max = 10;
|
|
|
|
|
Info($"No match for {noMatches.Count} files");
|
|
|
|
|
if (noMatches.Count > 0)
|
|
|
|
|
{
|
|
|
|
|
int count = 0;
|
|
|
|
|
foreach (var file in noMatches)
|
|
|
|
|
{
|
|
|
|
|
if (count++ < max)
|
|
|
|
|
{
|
|
|
|
|
Utils.Log($" {file.To} - {file.Reason}");
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
Utils.LogStraightToFile($" {file.To} - {file.Reason}");
|
|
|
|
|
}
|
2020-10-18 13:18:16 +00:00
|
|
|
|
|
2020-01-20 01:36:09 +00:00
|
|
|
|
if (count == max && noMatches.Count > max)
|
|
|
|
|
{
|
|
|
|
|
Utils.Log($" ...");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-05 19:36:44 +00:00
|
|
|
|
protected async Task InlineFiles()
|
|
|
|
|
{
|
|
|
|
|
var grouped = ModList.Directives.OfType<InlineFile>()
|
|
|
|
|
.Where(f => f.SourceDataID == default)
|
|
|
|
|
.GroupBy(f => f.SourceDataFile)
|
|
|
|
|
.ToDictionary(f => f.Key);
|
|
|
|
|
|
2020-09-06 03:19:05 +00:00
|
|
|
|
if (grouped.Count == 0) return;
|
2020-09-05 19:36:44 +00:00
|
|
|
|
await VFS.Extract(Queue, grouped.Keys.ToHashSet(), async (vf, sfn) =>
|
|
|
|
|
{
|
|
|
|
|
await using var stream = await sfn.GetStream();
|
|
|
|
|
var id = await IncludeFile(stream);
|
|
|
|
|
foreach (var file in grouped[vf])
|
|
|
|
|
{
|
|
|
|
|
file.SourceDataID = id;
|
|
|
|
|
file.SourceDataFile = null;
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2020-01-20 01:36:09 +00:00
|
|
|
|
public bool CheckForNoMatchExit(ICollection<NoMatch> noMatches)
|
|
|
|
|
{
|
|
|
|
|
if (noMatches.Count > 0)
|
|
|
|
|
{
|
|
|
|
|
if (IgnoreMissingFiles)
|
|
|
|
|
{
|
|
|
|
|
Info("Continuing even though files were missing at the request of the user.");
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
Info("Exiting due to no way to compile these files");
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
2020-10-18 13:18:16 +00:00
|
|
|
|
|
2020-01-20 01:36:09 +00:00
|
|
|
|
return false;
|
|
|
|
|
}
|
2019-11-03 15:26:51 +00:00
|
|
|
|
}
|
|
|
|
|
}
|