Moved core Compile functions to ACompiler

This commit is contained in:
erri120 2019-11-17 16:00:33 +01:00
parent b21fdfbd8c
commit ac178ed0c5
No known key found for this signature in database
GPG Key ID: A8C0A18D8D4D3135
3 changed files with 177 additions and 352 deletions

View File

@ -1,12 +1,19 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Reactive.Subjects;
using System.Text;
using System.Threading.Tasks;
using CommonMark;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.VirtualFileSystem;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Lib
{
@ -44,16 +51,176 @@ namespace Wabbajack.Lib
public List<IndexedArchive> IndexedArchives = new List<IndexedArchive>();
public Dictionary<string, IEnumerable<VirtualFile>> IndexedFiles = new Dictionary<string, IEnumerable<VirtualFile>>();
public abstract void Info(string msg);
public abstract void Status(string msg);
public abstract void Error(string msg);
public void Info(string msg)
{
Utils.Log(msg);
}
internal abstract string IncludeFile(byte[] data);
internal abstract string IncludeFile(string data);
public void Status(string msg)
{
Queue.Report(msg, 0);
}
public void Error(string msg)
{
Utils.Log(msg);
throw new Exception(msg);
}
internal string IncludeFile(byte[] data)
{
var id = Guid.NewGuid().ToString();
File.WriteAllBytes(Path.Combine(ModListOutputFolder, id), data);
return id;
}
internal string IncludeFile(string data)
{
var id = Guid.NewGuid().ToString();
File.WriteAllText(Path.Combine(ModListOutputFolder, id), data);
return id;
}
public void ExportModList()
{
Utils.Log($"Exporting ModList to : {ModListOutputFile}");
//ModList.ToJSON(Path.Combine(ModListOutputFolder, "modlist.json"));
ModList.ToCERAS(Path.Combine(ModListOutputFolder, "modlist"), ref CerasConfig.Config);
if (File.Exists(ModListOutputFile))
File.Delete(ModListOutputFile);
using (var fs = new FileStream(ModListOutputFile, FileMode.Create))
{
using (var za = new ZipArchive(fs, ZipArchiveMode.Create))
{
Directory.EnumerateFiles(ModListOutputFolder, "*.*")
.DoProgress("Compressing ModList",
f =>
{
var ze = za.CreateEntry(Path.GetFileName(f));
using (var os = ze.Open())
using (var ins = File.OpenRead(f))
{
ins.CopyTo(os);
}
});
}
}
Utils.Log("Exporting ModList metadata");
var metadata = new ModlistMetadata.DownloadMetadata
{
Size = File.GetSize(ModListOutputFile),
Hash = ModListOutputFile.FileHash(),
NumberOfArchives = ModList.Archives.Count,
SizeOfArchives = ModList.Archives.Sum(a => a.Size),
NumberOfInstalledFiles = ModList.Directives.Count,
SizeOfInstalledFiles = ModList.Directives.Sum(a => a.Size)
};
metadata.ToJSON(ModListOutputFile + ".meta.json");
Utils.Log("Removing ModList staging folder");
Directory.Delete(ModListOutputFolder, true);
}
public void ShowReport()
{
//if (!ShowReportWhenFinished) return;
var file = Path.GetTempFileName() + ".html";
File.WriteAllText(file, ModList.ReportHTML);
Process.Start(file);
}
public void GenerateReport()
{
string css;
using (var cssStream = Utils.GetResourceStream("Wabbajack.Lib.css-min.css"))
{
using (var reader = new StreamReader(cssStream))
{
css = reader.ReadToEnd();
}
}
using (var fs = File.OpenWrite($"{ModList.Name}.md"))
{
fs.SetLength(0);
using (var reporter = new ReportBuilder(fs, ModListOutputFolder))
{
reporter.Build(this, ModList);
}
}
ModList.ReportHTML = "<style>" + css + "</style>"
+ CommonMarkConverter.Convert(File.ReadAllText($"{ModList.Name}.md"));
}
public void GatherArchives()
{
Info("Building a list of archives based on the files required");
var shas = InstallDirectives.OfType<FromArchive>()
.Select(a => a.ArchiveHashPath[0])
.Distinct();
var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified)
.GroupBy(f => f.File.Hash)
.ToDictionary(f => f.Key, f => f.First());
SelectedArchives = shas.PMap(Queue, sha => ResolveArchive(sha, archives));
}
public Archive ResolveArchive(string sha, IDictionary<string, IndexedArchive> archives)
{
if (archives.TryGetValue(sha, out var found))
{
if (found.IniData == null)
Error($"No download metadata found for {found.Name}, please use MO2 to query info or add a .meta file and try again.");
var result = new Archive
{
State = (AbstractDownloadState) DownloadDispatcher.ResolveArchive(found.IniData)
};
if (result.State == null)
Error($"{found.Name} could not be handled by any of the downloaders");
result.Name = found.Name;
result.Hash = found.File.Hash;
result.Meta = found.Meta;
result.Size = found.File.Size;
Info($"Checking link for {found.Name}");
if (result.State != null && !result.State.Verify())
Error(
$"Unable to resolve link for {found.Name}. If this is hosted on the Nexus the file may have been removed.");
return result;
}
Error($"No match found for Archive sha: {sha} this shouldn't happen");
return null;
}
public abstract bool Compile();
public abstract Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source);
public Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
{
Utils.Status($"Compiling {source.Path}");
foreach (var step in stack)
{
var result = step.Run(source);
if (result != null) return result;
}
throw new InvalidDataException("Data fell out of the compilation stack");
}
public abstract IEnumerable<ICompilationStep> GetStack();
public abstract IEnumerable<ICompilationStep> MakeStack();

View File

@ -1,17 +1,12 @@
using CommonMark;
using Compression.BSA;
using Compression.BSA;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Wabbajack.VirtualFileSystem;
@ -76,36 +71,6 @@ namespace Wabbajack.Lib
public HashSet<string> SelectedProfiles { get; set; } = new HashSet<string>();
public override void Info(string msg)
{
Utils.Log(msg);
}
public override void Status(string msg)
{
Queue.Report(msg, 0);
}
public override void Error(string msg)
{
Utils.Log(msg);
throw new Exception(msg);
}
internal override string IncludeFile(byte[] data)
{
var id = Guid.NewGuid().ToString();
File.WriteAllBytes(Path.Combine(ModListOutputFolder, id), data);
return id;
}
internal override string IncludeFile(string data)
{
var id = Guid.NewGuid().ToString();
File.WriteAllText(Path.Combine(ModListOutputFolder, id), data);
return id;
}
public override bool Compile()
{
UpdateTracker.Reset();
@ -287,7 +252,7 @@ namespace Wabbajack.Lib
ValidateModlist.RunValidation(ModList);
GenerateReport();
ExportModlist();
ExportModList();
ResetMembers();
@ -313,87 +278,6 @@ namespace Wabbajack.Lib
});
}
private void ExportModlist()
{
Utils.Log($"Exporting Modlist to : {ModListOutputFile}");
//ModList.ToJSON(Path.Combine(ModListOutputFolder, "modlist.json"));
ModList.ToCERAS(Path.Combine(ModListOutputFolder, "modlist"), ref CerasConfig.Config);
if (File.Exists(ModListOutputFile))
File.Delete(ModListOutputFile);
using (var fs = new FileStream(ModListOutputFile, FileMode.Create))
{
using (var za = new ZipArchive(fs, ZipArchiveMode.Create))
{
Directory.EnumerateFiles(ModListOutputFolder, "*.*")
.DoProgress("Compressing Modlist",
f =>
{
var ze = za.CreateEntry(Path.GetFileName(f));
using (var os = ze.Open())
using (var ins = File.OpenRead(f))
{
ins.CopyTo(os);
}
});
}
}
Utils.Log("Exporting Modlist metadata");
var metadata = new ModlistMetadata.DownloadMetadata
{
Size = File.GetSize(ModListOutputFile),
Hash = ModListOutputFile.FileHash(),
NumberOfArchives = ModList.Archives.Count,
SizeOfArchives = ModList.Archives.Sum(a => a.Size),
NumberOfInstalledFiles = ModList.Directives.Count,
SizeOfInstalledFiles = ModList.Directives.Sum(a => a.Size)
};
metadata.ToJSON(ModListOutputFile + ".meta.json");
Utils.Log("Removing modlist staging folder");
Directory.Delete(ModListOutputFolder, true);
}
private void ShowReport()
{
if (!ShowReportWhenFinished) return;
var file = Path.GetTempFileName() + ".html";
File.WriteAllText(file, ModList.ReportHTML);
Process.Start(file);
}
private void GenerateReport()
{
string css = "";
using (Stream cssStream = Utils.GetResourceStream("Wabbajack.Lib.css-min.css"))
{
using (StreamReader reader = new StreamReader(cssStream))
{
css = reader.ReadToEnd();
}
}
using (var fs = File.OpenWrite($"{ModList.Name}.md"))
{
fs.SetLength(0);
using (var reporter = new ReportBuilder(fs, ModListOutputFolder))
{
reporter.Build(this, ModList);
}
}
ModList.ReportHTML = "<style>" + css + "</style>"
+ CommonMarkConverter.Convert(File.ReadAllText($"{ModList.Name}.md"));
}
/// <summary>
/// Clear references to lists that hold a lot of data.
/// </summary>
@ -477,65 +361,6 @@ namespace Wabbajack.Lib
return null;
}
private void GatherArchives()
{
Info("Building a list of archives based on the files required");
var shas = InstallDirectives.OfType<FromArchive>()
.Select(a => a.ArchiveHashPath[0])
.Distinct();
var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified)
.GroupBy(f => f.File.Hash)
.ToDictionary(f => f.Key, f => f.First());
SelectedArchives = shas.PMap(Queue, sha => ResolveArchive(sha, archives));
}
private Archive ResolveArchive(string sha, IDictionary<string, IndexedArchive> archives)
{
if (archives.TryGetValue(sha, out var found))
{
if (found.IniData == null)
Error($"No download metadata found for {found.Name}, please use MO2 to query info or add a .meta file and try again.");
var result = new Archive();
result.State = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(found.IniData);
if (result.State == null)
Error($"{found.Name} could not be handled by any of the downloaders");
result.Name = found.Name;
result.Hash = found.File.Hash;
result.Meta = found.Meta;
result.Size = found.File.Size;
Info($"Checking link for {found.Name}");
if (!result.State.Verify())
Error(
$"Unable to resolve link for {found.Name}. If this is hosted on the Nexus the file may have been removed.");
return result;
}
Error($"No match found for Archive sha: {sha} this shouldn't happen");
return null;
}
public override Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
{
Utils.Status($"Compiling {source.Path}");
foreach (var step in stack)
{
var result = step.Run(source);
if (result != null) return result;
}
throw new InvalidDataException("Data fell out of the compilation stack");
}
public override IEnumerable<ICompilationStep> GetStack()
{
var user_config = Path.Combine(MO2ProfileDir, "compilation_stack.yml");

View File

@ -1,17 +1,12 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using CommonMark;
using Microsoft.WindowsAPICodePack.Shell;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
using Wabbajack.VirtualFileSystem;
using File = Alphaleonis.Win32.Filesystem.File;
@ -51,36 +46,6 @@ namespace Wabbajack.Lib
ModListOutputFolder = "output_folder";
}
public override void Info(string msg)
{
Utils.Log(msg);
}
public override void Status(string msg)
{
Queue.Report(msg, 0);
}
public override void Error(string msg)
{
Utils.Log(msg);
throw new Exception(msg);
}
internal override string IncludeFile(byte[] data)
{
var id = Guid.NewGuid().ToString();
File.WriteAllBytes(Path.Combine(ModListOutputFolder, id), data);
return id;
}
internal override string IncludeFile(string data)
{
var id = Guid.NewGuid().ToString();
File.WriteAllText(Path.Combine(ModListOutputFolder, id), data);
return id;
}
public override bool Compile()
{
if (string.IsNullOrEmpty(ModListName))
@ -237,79 +202,6 @@ namespace Wabbajack.Lib
});
}
private void ExportModList()
{
Utils.Log($"Exporting ModList to: {ModListOutputFolder}");
// using JSON for better debugging
ModList.ToJSON(Path.Combine(ModListOutputFolder, "modlist.json"));
//ModList.ToCERAS(Path.Combine(ModListOutputFolder, "modlist"), ref CerasConfig.Config);
if(File.Exists(ModListOutputFile))
File.Delete(ModListOutputFile);
using (var fs = new FileStream(ModListOutputFile, FileMode.Create))
{
using (var za = new ZipArchive(fs, ZipArchiveMode.Create))
{
Directory.EnumerateFiles(ModListOutputFolder, "*.*")
.DoProgress("Compressing ModList",
f =>
{
var ze = za.CreateEntry(Path.GetFileName(f));
using (var os = ze.Open())
using (var ins = File.OpenRead(f))
{
ins.CopyTo(os);
}
});
}
}
Utils.Log("Exporting ModList metadata");
var metadata = new ModlistMetadata.DownloadMetadata
{
Size = File.GetSize(ModListOutputFile),
Hash = ModListOutputFile.FileHash(),
NumberOfArchives = ModList.Archives.Count,
SizeOfArchives = ModList.Archives.Sum(a => a.Size),
NumberOfInstalledFiles = ModList.Directives.Count,
SizeOfInstalledFiles = ModList.Directives.Sum(a => a.Size)
};
metadata.ToJSON(ModListOutputFile + ".meta.json");
Utils.Log("Removing ModList staging folder");
Directory.Delete(ModListOutputFolder, true);
}
private void GenerateReport()
{
string css;
using (var cssStream = Utils.GetResourceStream("Wabbajack.Lib.css-min.css"))
using (var reader = new StreamReader(cssStream))
{
css = reader.ReadToEnd();
}
using (var fs = File.OpenWrite($"{ModList.Name}.md"))
{
fs.SetLength(0);
using (var reporter = new ReportBuilder(fs, ModListOutputFolder))
{
reporter.Build(this, ModList);
}
}
ModList.ReportHTML = "<style>" + css + "</style>" + CommonMarkConverter.Convert(File.ReadAllText($"{ModList.Name}.md"));
}
private void ShowReport()
{
var file = Alphaleonis.Win32.Filesystem.Path.GetTempFileName() + ".html";
File.WriteAllText(file, ModList.ReportHTML);
Process.Start(file);
}
private void CreateMetaFiles()
{
Utils.Log("Getting Nexus api_key, please click authorize if a browser window appears");
@ -353,65 +245,6 @@ namespace Wabbajack.Lib
});
}
private void GatherArchives()
{
Info("Building a list of archives based on the files required");
var shas = InstallDirectives.OfType<FromArchive>()
.Select(a => a.ArchiveHashPath[0])
.Distinct();
var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified)
.GroupBy(f => f.File.Hash)
.ToDictionary(f => f.Key, f => f.First());
SelectedArchives = shas.PMap(Queue, sha => ResolveArchive(sha, archives));
}
private Archive ResolveArchive(string sha, IDictionary<string, IndexedArchive> archives)
{
if (archives.TryGetValue(sha, out var found))
{
if(found.IniData == null)
Error($"No download metadata found for {found.Name}, please use MO2 to query info or add a .meta file and try again.");
var result = new Archive();
result.State = (AbstractDownloadState) DownloadDispatcher.ResolveArchive(found.IniData);
if (result.State == null)
Error($"{found.Name} could not be handled by any of the downloaders");
result.Name = found.Name;
result.Hash = found.File.Hash;
result.Meta = found.Meta;
result.Size = found.File.Size;
Info($"Checking link for {found.Name}");
if (!result.State.Verify())
Error(
$"Unable to resolve link for {found.Name}. If this is hosted on the Nexus the file may have been removed.");
return result;
}
Error($"No match found for Archive sha: {sha} this shouldn't happen");
return null;
}
public override Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
{
Utils.Status($"Compiling {source.Path}");
foreach (var step in stack)
{
var result = step.Run(source);
if (result != null) return result;
}
throw new InvalidDataException("Data fell out of the compilation stack");
}
public override IEnumerable<ICompilationStep> GetStack()
{
var s = Consts.TestMode ? DownloadsFolder : VortexFolder;