Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

More compiler fixes

Browse Source
This commit is contained in:
Timothy Baldridge 2022-05-26 23:41:11 -06:00
parent 06674a22f8
commit 06bb04c89f
16 changed files with 216 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs
View File

@ -194,7 +194,7 @@ namespace Wabbajack
{
State = CompilerState.Compiling;
var mo2Settings = new MO2CompilerSettings
var mo2Settings = new CompilerSettings
{
Game = BaseGame,
ModListName = ModListName,
@ -210,19 +210,7 @@ namespace Wabbajack
UseGamePaths = true
};
var compiler = new MO2Compiler(_serviceProvider.GetRequiredService<ILogger<MO2Compiler>>(),
_serviceProvider.GetRequiredService<FileExtractor.FileExtractor>(),
_serviceProvider.GetRequiredService<FileHashCache>(),
_serviceProvider.GetRequiredService<Context>(),
_serviceProvider.GetRequiredService<TemporaryFileManager>(),
mo2Settings,
_serviceProvider.GetRequiredService<ParallelOptions>(),
_serviceProvider.GetRequiredService<DownloadDispatcher>(),
_serviceProvider.GetRequiredService<Client>(),
_serviceProvider.GetRequiredService<IGameLocator>(),
_serviceProvider.GetRequiredService<DTOSerializer>(),
_serviceProvider.GetRequiredService<IResource<ACompiler>>(),
_serviceProvider.GetRequiredService<IBinaryPatchCache>());
var compiler = MO2Compiler.Create(_serviceProvider, mo2Settings);
await compiler.Begin(CancellationToken.None);

1
Wabbajack.CLI/Program.cs
View File

@ -73,6 +73,7 @@ internal class Program
services.AddSingleton<IVerb, Extract>();
services.AddSingleton<IVerb, DumpZipInfo>();
services.AddSingleton<IVerb, Install>();
services.AddSingleton<IVerb, InstallCompileInstallVerify>();
services.AddSingleton<IUserInterventionHandler, UserInterventionHandler>();
}).Build();

140
Wabbajack.CLI/Verbs/InstallCompileInstallVerify.cs Normal file
View File

@ -0,0 +1,140 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.CommandLine;
using System.CommandLine.Invocation;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.Compiler;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Installer;
using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.VFS;
namespace Wabbajack.CLI.Verbs;
public class InstallCompileInstallVerify : IVerb
{
private readonly ILogger<InstallCompileInstallVerify> _logger;
private readonly Client _wjClient;
private readonly DownloadDispatcher _dispatcher;
private readonly DTOSerializer _dtos;
private readonly IServiceProvider _serviceProvider;
private readonly FileHashCache _cache;
private readonly GameLocator _gameLocator;
private readonly CompilerSettingsInferencer _inferencer;
public InstallCompileInstallVerify(ILogger<InstallCompileInstallVerify> logger, Client wjClient, DownloadDispatcher dispatcher, DTOSerializer dtos,
FileHashCache cache, GameLocator gameLocator, IServiceProvider serviceProvider, CompilerSettingsInferencer inferencer)
{
_logger = logger;
_wjClient = wjClient;
_dispatcher = dispatcher;
_dtos = dtos;
_serviceProvider = serviceProvider;
_cache = cache;
_gameLocator = gameLocator;
_inferencer = inferencer;
}
public Command MakeCommand()
{
var command = new Command("install-compile-install-verify");
command.Add(new Option<AbsolutePath>(new[] {"-m", "-machineUrls"}, "Machine url(s) to download"));
command.Add(new Option<AbsolutePath>(new[] {"-d", "-downloads"}, "Downloads path"));
command.Add(new Option<AbsolutePath>(new[] {"-o", "-outputs"}, "Outputs path"));
command.Description = "Installs a modlist, compiles it, installs it again, verifies it";
command.Handler = CommandHandler.Create(Run);
return command;
}
public async Task<int> Run(AbsolutePath outputs, AbsolutePath downloads, IEnumerable<string> machineUrls, CancellationToken token)
{
foreach (var machineUrl in machineUrls)
{
_logger.LogInformation("Installing {MachineUrl}", machineUrl);
var wabbajackPath = downloads.Combine(machineUrl.Replace("/", "_@@_"));
if (!await DownloadMachineUrl(machineUrl, wabbajackPath, token))
throw new Exception("Can't download modlist");
var installPath = outputs.Combine(machineUrl);
var modlist = await StandardInstaller.LoadFromFile(_dtos, wabbajackPath);
var installer = StandardInstaller.Create(_serviceProvider, new InstallerConfiguration
{
Downloads = downloads,
Install = installPath,
ModList = modlist,
Game = modlist.GameType,
ModlistArchive = wabbajackPath,
GameFolder = _gameLocator.GameLocation(modlist.GameType)
});
var result = await installer.Begin(token);
if (!result)
{
_logger.LogInformation("Error installing {MachineUrl}", machineUrl);
return 1;
}
_logger.LogInformation("Inferring settings");
var inferedSettings = await _inferencer.InferFromRootPath(installPath);
if (inferedSettings == null)
{
_logger.LogInformation("Error inferencing settings for {MachineUrl}", machineUrl);
return 2;
}
inferedSettings.UseGamePaths = true;
var compiler = MO2Compiler.Create(_serviceProvider, inferedSettings);
result = await compiler.Begin(token);
return result ? 0 : 3;
}
return 0;
}
private async Task<bool> DownloadMachineUrl(string machineUrl, AbsolutePath wabbajack, CancellationToken token)
{
_logger.LogInformation("Downloading {MachineUrl}", machineUrl);
var lists = await _wjClient.LoadLists();
var list = lists.FirstOrDefault(l => l.NamespacedName == machineUrl);
if (list == null)
{
_logger.LogInformation("Couldn't find list {MachineUrl}", machineUrl);
return false;
}
if (wabbajack.FileExists() && await _cache.FileHashCachedAsync(wabbajack, token) == list.DownloadMetadata!.Hash)
{
_logger.LogInformation("File already exists, using cached file");
return true;
}
var state = _dispatcher.Parse(new Uri(list.Links.Download));
await _dispatcher.Download(new Archive
{
Name = wabbajack.FileName.ToString(),
Hash = list.DownloadMetadata!.Hash,
Size = list.DownloadMetadata.Size,
State = state!
}, wabbajack, token);
return true;
}
}

1
Wabbajack.Common/Ext.cs
View File

@ -21,7 +21,6 @@ public static class Ext
public static Extension Md = new(".md");
public static Extension MetaData = new(".metadata");
public static Extension CompilerSettings = new(".compiler_settings");
public static Extension MO2CompilerSettings = new(".mo2_compiler_settings");
public static Extension Temp = new(".temp");
public static Extension ModlistMetadataExtension = new(".modlist_metadata");
public static Extension Txt = new(".txt");

6
Wabbajack.Compiler.Test/ModListHarness.cs
View File

@ -66,7 +66,7 @@ public class ModListHarness
return mod;
}
public async Task<ModList?> CompileAndInstall(Action<MO2CompilerSettings>? configureSettings = null)
public async Task<ModList?> CompileAndInstall(Action<CompilerSettings>? configureSettings = null)
{
var modlist = await Compile(configureSettings);
await Install();
@ -74,13 +74,13 @@ public class ModListHarness
return modlist;
}
public async Task<ModList?> Compile(Action<MO2CompilerSettings>? configureSettings = null)
public async Task<ModList?> Compile(Action<CompilerSettings>? configureSettings = null)
{
configureSettings ??= x => { };
_source.Combine(Consts.MO2Profiles, _profileName).CreateDirectory();
using var scope = _serviceProvider.CreateScope();
var settings = scope.ServiceProvider.GetService<MO2CompilerSettings>()!;
var settings = scope.ServiceProvider.GetService<CompilerSettings>()!;
settings.Downloads = _downloadPath;
settings.Game = Game.SkyrimSpecialEdition;
settings.Source = _source;

2
Wabbajack.Compiler.Test/SanityTests.cs
View File

@ -56,7 +56,7 @@ public class CompilerSanityTests : IAsyncLifetime
{
}
private async Task CompileAndValidate(int expectedDirectives, Action<MO2CompilerSettings>? configureSettings = null)
private async Task CompileAndValidate(int expectedDirectives, Action<CompilerSettings>? configureSettings = null)
{
_modlist = await _harness.Compile(configureSettings);
Assert.NotNull(_modlist);

9
Wabbajack.Compiler/ACompiler.cs
View File

@ -275,14 +275,14 @@ public abstract class ACompiler
if (resolved == null) return null;
a.State = resolved.State;
return null;
return a;
}
catch (Exception ex)
{
_logger.LogWarning(ex.ToString(), "While resolving archive {archive}", a.Name);
return a;
_logger.LogWarning(ex, "While resolving archive {Archive}", a.Name);
return null;
}
}).ToHashSet(f => f != null);
}).ToHashSet(f => f == null);
if (remove.Count == 0) return;
@ -461,6 +461,7 @@ public abstract class ACompiler
_logger.LogInformation("Patching {from} {to}", destFile, match.To);
await using var srcStream = await sf.GetStream();
await using var destStream = await destsfn.GetStream();
using var _ = await CompilerLimiter.Begin($"Patching {match.To}", 100, token);
var patchSize =
await _patchCache.CreatePatch(srcStream, vf.Hash, destStream, destvf.Hash);
_logger.LogInformation("Patch size {patchSize} for {to}", patchSize, match.To);

8
Wabbajack.Compiler/CompilerSettingsInferencer.cs
View File

@ -24,6 +24,14 @@ public class CompilerSettingsInferencer
}
public async Task<CompilerSettings?> InferFromRootPath(AbsolutePath rootPath)
{
var mo2File = rootPath.Combine(Consts.MO2IniName).LoadIniFile();
var profile = mo2File["General"]["selected_profile"];
return await InferModListFromLocation(rootPath.Combine(Consts.MO2Profiles, profile, Consts.ModListTxt));
}
public async Task<CompilerSettings?> InferModListFromLocation(AbsolutePath settingsFile)
{
var cs = new CompilerSettings();

23
Wabbajack.Compiler/MO2Compiler.cs
View File

@ -1,9 +1,11 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using IniParser.Model;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Compiler.CompilationSteps;
@ -25,7 +27,7 @@ public class MO2Compiler : ACompiler
{
public MO2Compiler(ILogger<MO2Compiler> logger, FileExtractor.FileExtractor extractor, FileHashCache hashCache,
Context vfs,
TemporaryFileManager manager, MO2CompilerSettings settings, ParallelOptions parallelOptions,
TemporaryFileManager manager, CompilerSettings settings, ParallelOptions parallelOptions,
DownloadDispatcher dispatcher,
Client wjClient, IGameLocator locator, DTOSerializer dtos, IResource<ACompiler> compilerLimiter,
IBinaryPatchCache patchCache) :
@ -35,7 +37,24 @@ public class MO2Compiler : ACompiler
MaxSteps = 14;
}
public MO2CompilerSettings Mo2Settings => (MO2CompilerSettings) Settings;
public static MO2Compiler Create(IServiceProvider provider, CompilerSettings mo2Settings)
{
return new MO2Compiler(provider.GetRequiredService<ILogger<MO2Compiler>>(),
provider.GetRequiredService<FileExtractor.FileExtractor>(),
provider.GetRequiredService<FileHashCache>(),
provider.GetRequiredService<Context>(),
provider.GetRequiredService<TemporaryFileManager>(),
mo2Settings,
provider.GetRequiredService<ParallelOptions>(),
provider.GetRequiredService<DownloadDispatcher>(),
provider.GetRequiredService<Client>(),
provider.GetRequiredService<IGameLocator>(),
provider.GetRequiredService<DTOSerializer>(),
provider.GetRequiredService<IResource<ACompiler>>(),
provider.GetRequiredService<IBinaryPatchCache>());
}
public CompilerSettings Mo2Settings => (CompilerSettings) Settings;
public AbsolutePath MO2ModsFolder => Settings.Source.Combine(Consts.MO2ModFolderName);

9
Wabbajack.Compiler/MO2CompilerSettings.cs
View File

@ -1,9 +0,0 @@
using System;
using Wabbajack.Paths;
namespace Wabbajack.Compiler;
public class MO2CompilerSettings : CompilerSettings
{
}

5
Wabbajack.Compiler/PatchCache/BinaryPatchCache.cs
View File

@ -6,6 +6,7 @@ using Wabbajack.Compiler.PatchCache;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
namespace Wabbajack.Compiler;
@ -38,7 +39,7 @@ public class BinaryPatchCache : IBinaryPatchCache
cmd.ExecuteNonQuery();
}
public async Task<CacheEntry> CreatePatch(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash)
public async Task<CacheEntry> CreatePatch(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, IJob? job)
{
await using var rcmd = new SQLiteCommand(_conn);
rcmd.CommandText = "SELECT PatchSize FROM PatchCache WHERE FromHash = @fromHash AND ToHash = @toHash";
@ -57,7 +58,7 @@ public class BinaryPatchCache : IBinaryPatchCache
await using var sigStream = new MemoryStream();
await using var patchStream = new MemoryStream();
OctoDiff.Create(srcStream, destStream, sigStream, patchStream);
OctoDiff.Create(srcStream, destStream, sigStream, patchStream, job);
cmd.Parameters.AddWithValue("@patchSize", patchStream.Length);
cmd.Parameters.AddWithValue("@patch", patchStream.ToArray());

3
Wabbajack.Compiler/PatchCache/IBinaryPatchCache.cs
View File

@ -2,12 +2,13 @@ using System.IO;
using System.Threading.Tasks;
using Wabbajack.Compiler.PatchCache;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.RateLimiter;
namespace Wabbajack.Compiler;
public interface IBinaryPatchCache
{
public Task<CacheEntry> CreatePatch(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash);
public Task<CacheEntry> CreatePatch(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, IJob? job = null);
public Task<CacheEntry?> GetPatch(Hash hashA, Hash hashB);
public Task<byte[]> GetData(CacheEntry entry);

26
Wabbajack.Compiler/PatchCache/OctoDiff.cs
View File

@ -1,6 +1,8 @@
using System.IO;
using System.Threading;
using Octodiff.Core;
using Octodiff.Diagnostics;
using Wabbajack.RateLimiter;
namespace Wabbajack.Compiler.PatchCache;
@ -33,11 +35,29 @@ public class OctoDiff
sigStream.Position = 0;
}
public static void Create(Stream oldData, Stream newData, Stream signature, Stream output)
public static void Create(Stream oldData, Stream newData, Stream signature, Stream output, IJob? job)
{
CreateSignature(oldData, signature);
var db = new DeltaBuilder {ProgressReporter = new NullProgressReporter()};
db.BuildDelta(newData, new SignatureReader(signature, new NullProgressReporter()),
var db = new DeltaBuilder {ProgressReporter = new JobProgressReporter(job, 0)};
db.BuildDelta(newData, new SignatureReader(signature, new JobProgressReporter(job, 100)),
new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
}
private class JobProgressReporter : IProgressReporter
{
private readonly IJob _job;
private readonly int _offset;
public JobProgressReporter(IJob job, int offset)
{
_offset = offset;
_job = job;
}
public void ReportProgress(string operation, long currentPosition, long total)
{
var percent = Percent.FactoryPutInRange(currentPosition, total);
var toReport = (long) (percent.Value * 100) - (_job.Current - _offset);
_job.ReportNoWait((int) toReport);
}
}
}

6
Wabbajack.Installer/AInstaller.cs
View File

@ -399,8 +399,12 @@ public abstract class AInstaller<T>
.Concat(_gameLocator.GameLocation(_configuration.Game).EnumerateFiles())
.ToList();
var hashDict = allFiles.GroupBy(f => f.Size()).ToDictionary(g => g.Key);
_logger.LogInformation("Getting archive sizes");
var hashDict = (await allFiles.PMapAll(_limiter, async x => (x, x.Size())).ToList())
.GroupBy(f => f.Item2)
.ToDictionary(g => g.Key, g => g.Select(v => v.x));
_logger.LogInformation("Linking archives to downloads");
var toHash = ModList.Archives.Where(a => hashDict.ContainsKey(a.Size))
.SelectMany(a => hashDict[a.Size]).ToList();

1
Wabbajack.RateLimiter/IJob.cs
View File

@ -10,4 +10,5 @@ public interface IJob
public long Current { get; }
public string Description { get; }
public ValueTask Report(int processedSize, CancellationToken token);
public void ReportNoWait(int processedSize);
}

2
Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
View File

@ -152,7 +152,7 @@ public static class ServiceExtensions
// Installer/Compiler Configuration
service.AddScoped<InstallerConfiguration>();
service.AddScoped<StandardInstaller>();
service.AddScoped<MO2CompilerSettings>();
service.AddScoped<CompilerSettings>();
service.AddScoped<MO2Compiler>();
service.AddSingleton<CompilerSettingsInferencer>();