Patch creation into ACompiler

This commit is contained in:
Timothy Baldridge 2020-10-18 07:16:27 -06:00
parent 80d93caf96
commit 2bc3553dcf
3 changed files with 114 additions and 113 deletions

View File

@ -52,6 +52,8 @@ namespace Wabbajack.Lib
public AbsolutePath SourcePath { get;}
public AbsolutePath DownloadsPath { get;}
public GameMetaData CompilingGame { get; set; }
public AbsolutePath ModListOutputFolder { get; }
public AbsolutePath ModListOutputFile { get; }
@ -78,6 +80,7 @@ namespace Wabbajack.Lib
WabbajackVersion = Consts.CurrentMinimumWabbajackVersion;
Settings = new CompilerSettings();
ModListOutputFolder = AbsolutePath.EntryPoint.Combine("output_folder", Guid.NewGuid().ToString());
CompilingGame = new GameMetaData();
}
public static void Info(string msg)
@ -343,6 +346,114 @@ namespace Wabbajack.Lib
await Utils.DeleteDirectory(ModListOutputFolder);
}
/// <summary>
/// Fills in the Patch fields in files that require them
/// </summary>
protected async Task BuildPatches()
{
Info("Gathering patch files");
var toBuild = InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => p.Choices.Length > 0)
.SelectMany(p => p.Choices.Select(c => new PatchedFromArchive
{
To = p.To,
Hash = p.Hash,
ArchiveHashPath = c.MakeRelativePaths(),
FromFile = c,
Size = p.Size
}))
.ToArray();
if (toBuild.Length == 0)
{
return;
}
// Extract all the source files
var indexed = toBuild.GroupBy(f => VFS.Index.FileForArchiveHashPath(f.ArchiveHashPath))
.ToDictionary(f => f.Key);
await VFS.Extract(Queue, indexed.Keys.ToHashSet(),
async (vf, sf) =>
{
// For each, extract the destination
var matches = indexed[vf];
using var iqueue = new WorkQueue(1);
foreach (var match in matches)
{
var destFile = FindDestFile(match.To);
// Build the patch
await VFS.Extract(iqueue, new[] {destFile}.ToHashSet(),
async (destvf, destsfn) =>
{
Info($"Patching {match.To}");
Status($"Patching {match.To}");
await using var srcStream = await sf.GetStream();
await using var destStream = await destsfn.GetStream();
var patchSize =
await Utils.CreatePatchCached(srcStream, vf.Hash, destStream, destvf.Hash);
Info($"Patch size {patchSize} for {match.To}");
});
}
});
// Load in the patches
await InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => p.PatchID == default)
.PMap(Queue, async pfa =>
{
var patches = pfa.Choices
.Select(c => (Utils.TryGetPatch(c.Hash, pfa.Hash, out var data), data, c))
.ToArray();
// Pick the best patch
if (patches.All(p => p.Item1))
{
var (_, bytes, file) = IncludePatches.PickPatch(this, patches);
pfa.FromFile = file;
pfa.FromHash = file.Hash;
pfa.ArchiveHashPath = file.MakeRelativePaths();
pfa.PatchID = await IncludeFile(bytes!);
}
});
var firstFailedPatch =
InstallDirectives.OfType<PatchedFromArchive>().FirstOrDefault(f => f.PatchID == default);
if (firstFailedPatch != null)
{
Utils.Log("Missing data from failed patch, starting data dump");
Utils.Log($"Dest File: {firstFailedPatch.To}");
Utils.Log($"Options ({firstFailedPatch.Choices.Length}:");
foreach (var choice in firstFailedPatch.Choices)
{
Utils.Log($" {choice.FullPath}");
}
Error(
$"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}");
}
}
private VirtualFile FindDestFile(RelativePath to)
{
var abs = to.RelativeTo(SourcePath);
if (abs.Exists)
{
return VFS.Index.ByRootPath[abs];
}
if (to.StartsWith(Consts.BSACreationDir))
{
var bsaId = (RelativePath)((string)to).Split('\\')[1];
var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsaId);
var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray());
return VFS.Index.ByRootPath[SourcePath.Combine(bsa.To)].Children.First(c => c.RelativeName == find);
}
throw new ArgumentException($"Couldn't load data for {to}");
}
public void GenerateManifest()
{
var manifest = new Manifest(ModList);

View File

@ -126,7 +126,7 @@ namespace Wabbajack.Lib.CompilationSteps
return e;
}
public static (bool, byte[], VirtualFile) PickPatch(MO2Compiler mo2Compiler, IEnumerable<(bool foundHash, byte[]? data, VirtualFile file)> patches)
public static (bool, byte[], VirtualFile) PickPatch(ACompiler compiler, IEnumerable<(bool foundHash, byte[]? data, VirtualFile file)> patches)
{
var ordered = patches
.Select(f => (f.foundHash, f.data!, f.file))
@ -138,11 +138,11 @@ namespace Wabbajack.Lib.CompilationSteps
var baseHash = itm.file.TopParent.Hash;
// If this file doesn't come from a game use it
if (!mo2Compiler.GamesWithHashes.TryGetValue(baseHash, out var games))
if (!compiler.GamesWithHashes.TryGetValue(baseHash, out var games))
return true;
// Otherwise skip files that are not from the primary game
return games.Contains(mo2Compiler.CompilingGame.Game);
return games.Contains(compiler.CompilingGame.Game);
});
// If we didn't find a file from an archive or the primary game, use a secondary game file.

View File

@ -33,8 +33,6 @@ namespace Wabbajack.Lib
public override AbsolutePath GamePath { get; }
public GameMetaData CompilingGame { get; }
public dynamic MO2Ini { get; }
public AbsolutePath MO2ProfileDir => SourcePath.Combine("profiles", MO2Profile);
@ -407,114 +405,6 @@ namespace Wabbajack.Lib
ExtraFiles = new ConcurrentBag<Directive>();
}
/// <summary>
/// Fills in the Patch fields in files that require them
/// </summary>
private async Task BuildPatches()
{
Info("Gathering patch files");
var toBuild = InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => p.Choices.Length > 0)
.SelectMany(p => p.Choices.Select(c => new PatchedFromArchive
{
To = p.To,
Hash = p.Hash,
ArchiveHashPath = c.MakeRelativePaths(),
FromFile = c,
Size = p.Size
}))
.ToArray();
if (toBuild.Length == 0)
{
return;
}
// Extract all the source files
var indexed = toBuild.GroupBy(f => VFS.Index.FileForArchiveHashPath(f.ArchiveHashPath))
.ToDictionary(f => f.Key);
await VFS.Extract(Queue, indexed.Keys.ToHashSet(),
async (vf, sf) =>
{
// For each, extract the destination
var matches = indexed[vf];
using var iqueue = new WorkQueue(1);
foreach (var match in matches)
{
var destFile = FindDestFile(match.To);
// Build the patch
await VFS.Extract(iqueue, new[] {destFile}.ToHashSet(),
async (destvf, destsfn) =>
{
Info($"Patching {match.To}");
Status($"Patching {match.To}");
await using var srcStream = await sf.GetStream();
await using var destStream = await destsfn.GetStream();
var patchSize =
await Utils.CreatePatchCached(srcStream, vf.Hash, destStream, destvf.Hash);
Info($"Patch size {patchSize} for {match.To}");
});
}
});
// Load in the patches
await InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => p.PatchID == default)
.PMap(Queue, async pfa =>
{
var patches = pfa.Choices
.Select(c => (Utils.TryGetPatch(c.Hash, pfa.Hash, out var data), data, c))
.ToArray();
// Pick the best patch
if (patches.All(p => p.Item1))
{
var (_, bytes, file) = IncludePatches.PickPatch(this, patches);
pfa.FromFile = file;
pfa.FromHash = file.Hash;
pfa.ArchiveHashPath = file.MakeRelativePaths();
pfa.PatchID = await IncludeFile(bytes!);
}
});
var firstFailedPatch =
InstallDirectives.OfType<PatchedFromArchive>().FirstOrDefault(f => f.PatchID == default);
if (firstFailedPatch != null)
{
Utils.Log("Missing data from failed patch, starting data dump");
Utils.Log($"Dest File: {firstFailedPatch.To}");
Utils.Log($"Options ({firstFailedPatch.Choices.Length}:");
foreach (var choice in firstFailedPatch.Choices)
{
Utils.Log($" {choice.FullPath}");
}
Error(
$"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}");
}
}
private VirtualFile FindDestFile(RelativePath to)
{
var abs = to.RelativeTo(SourcePath);
if (abs.Exists)
{
return VFS.Index.ByRootPath[abs];
}
if (to.StartsWith(Consts.BSACreationDir))
{
var bsaId = (RelativePath)((string)to).Split('\\')[1];
var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsaId);
var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray());
return VFS.Index.ByRootPath[SourcePath.Combine(bsa.To)].Children.First(c => c.RelativeName == find);
}
throw new ArgumentException($"Couldn't load data for {to}");
}
public override IEnumerable<ICompilationStep> GetStack()
{
return MakeStack();