Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Compiler works, needs a lot of refinement

Browse Source
This commit is contained in:
Timothy Baldridge 2021-10-14 22:20:27 -06:00
parent 51015eaa81
commit c54e6249df
10 changed files with 135 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Wabbajack.App/App.axaml.cs
View File

@ -6,6 +6,7 @@ using Avalonia.Markup.Xaml;
using CefNet;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using ReactiveUI;
using Splat;
using Wabbajack.App.Controls;
@ -36,6 +37,10 @@ namespace Wabbajack.App
public override void OnFrameworkInitializationCompleted()
{
var host = Host.CreateDefaultBuilder(Array.Empty<string>())
.ConfigureLogging(c =>
{
c.ClearProviders();
})
.ConfigureServices((host, services) =>
{
services.AddAppServices();

2
Wabbajack.App/Controls/LogViewModel.cs
View File

@ -21,7 +21,7 @@ public class LogViewModel : ViewModelBase, IActivatableViewModel
public LogViewModel(LoggerProvider provider)
{
_messages = new SourceCache<LoggerProvider.ILogMessage, long>(m => m.MessageId);
_messages.LimitSizeTo(100);
//_messages.LimitSizeTo(100);
Activator = new ViewModelActivator();
_provider = provider;

4
Wabbajack.App/Models/SettingsManager.cs
View File

@ -46,9 +46,9 @@ public class SettingsManager
{
if (path.FileExists())
{
await using (var s = path.Open(FileMode.Create, FileAccess.Write))
await using (var s = path.Open(FileMode.Open))
{
await JsonSerializer.DeserializeAsync<T>(s, _dtos.Options);
return (await JsonSerializer.DeserializeAsync<T>(s, _dtos.Options))!;
}
}
}

14
Wabbajack.App/Screens/CompilationView.axaml.cs
View File

@ -1,3 +1,5 @@
using Avalonia.Controls.Mixins;
using ReactiveUI;
using Wabbajack.App.ViewModels;
using Wabbajack.App.Views;
@ -8,5 +10,17 @@ public partial class CompilationView : ScreenBase<CompilationViewModel>
public CompilationView()
{
InitializeComponent();
this.WhenActivated(disposables =>
{
this.OneWayBind(ViewModel, vm => vm.StatusText, view => view.StatusText.Text)
.DisposeWith(disposables);
this.OneWayBind(ViewModel, vm => vm.StepsProgress, view => view.StepsProgress.Value, p => p.Value * 1000)
.DisposeWith(disposables);
this.OneWayBind(ViewModel, vm => vm.StepProgress, view => view.StepProgress.Value, p => p.Value * 10000)
.DisposeWith(disposables);
});
}
}

19
Wabbajack.App/Screens/CompilationViewModel.cs
View File

@ -1,13 +1,16 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Avalonia.Threading;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using ReactiveUI;
using ReactiveUI.Fody.Helpers;
using Wabbajack.App.Messages;
using Wabbajack.App.ViewModels;
using Wabbajack.Common;
using Wabbajack.Compiler;
using Wabbajack.RateLimiter;
namespace Wabbajack.App.Screens;
@ -17,6 +20,11 @@ public class CompilationViewModel : ViewModelBase, IReceiverMarker, IReceiver<St
private ACompiler _compiler;
private readonly ILogger<CompilationViewModel> _logger;
[Reactive] public string StatusText { get; set; } = "";
[Reactive] public Percent StepsProgress { get; set; } = Percent.Zero;
[Reactive] public Percent StepProgress { get; set; } = Percent.Zero;
public CompilationViewModel(ILogger<CompilationViewModel> logger, IServiceProvider provider)
{
_logger = logger;
@ -32,7 +40,15 @@ public class CompilationViewModel : ViewModelBase, IReceiverMarker, IReceiver<St
var compiler = _provider.GetService<MO2Compiler>()!;
compiler.Settings = mo2;
_compiler = compiler;
_compiler.OnStatusUpdate += (sender, update) =>
{
Dispatcher.UIThread.InvokeAsync(() =>
{
StatusText = update.StatusText;
StepsProgress = update.StepsProgress;
StepProgress = update.StepProgress;
});
};
}
Compile().FireAndForget();
}
@ -46,6 +62,7 @@ public class CompilationViewModel : ViewModelBase, IReceiverMarker, IReceiver<St
catch (Exception ex)
{
_logger.LogError(ex, "During Compilation: {Message}", ex.Message);
StatusText = $"ERRORED: {ex.Message}";
}
}
}

21
Wabbajack.App/Screens/CompilerConfigurationViewModel.cs
View File

@ -3,15 +3,11 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reactive;
using System.Reactive.Disposables;
using System.Reactive.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using Avalonia.Controls.Mixins;
using DynamicData;
using ReactiveUI;
using ReactiveUI.Fody.Helpers;
using Wabbajack.App.Extensions;
using Wabbajack.App.Messages;
using Wabbajack.App.Models;
using Wabbajack.App.ViewModels;
@ -82,18 +78,29 @@ public class CompilerConfigurationViewModel : ViewModelBase, IReceiverMarker
OutputFolder = KnownFolders.EntryPoint;
this.WhenActivated((CompositeDisposable disposables) =>
this.WhenActivated(disposables =>
{
LoadLastCompilation().FireAndForget();
this.WhenAnyValue(v => v.SettingsFile)
.Subscribe( location =>
{
LoadNewSettingsFile(location).FireAndForget();
})
.DisposeWith(disposables);
});
}
private async Task LoadNewSettingsFile(AbsolutePath location)
{
if (location == default) return;
if (location.FileExists()) await LoadSettings(location);
}
private async Task LoadLastCompilation()
{
var location = await _settingsManager.Load<AbsolutePath>("last_compilation");
if (location == default) return;
if (location.FileExists()) await LoadSettings(location);
SettingsFile = location;
}
private async Task BeginCompilation()

68
Wabbajack.Compiler/ACompiler.cs
View File

@ -21,6 +21,7 @@ using Wabbajack.Installer;
using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
using Wabbajack.VFS;
namespace Wabbajack.Compiler
@ -50,6 +51,16 @@ namespace Wabbajack.Compiler
private readonly DTOSerializer _dtos;
public readonly IBinaryPatchCache _patchCache;
private long _maxStepProgress = 0;
private int _currentStep = 0;
private string _statusText;
private long _currentStepProgress;
private readonly Stopwatch _updateStopWatch = new();
protected long MaxSteps { get; set; }
public event EventHandler<StatusUpdate> OnStatusUpdate;
public ACompiler(ILogger logger, FileExtractor.FileExtractor extractor, FileHashCache hashCache, Context vfs, TemporaryFileManager manager, CompilerSettings settings,
ParallelOptions parallelOptions, DownloadDispatcher dispatcher, Client wjClient, IGameLocator locator, DTOSerializer dtos,
IBinaryPatchCache patchCache)
@ -70,9 +81,42 @@ namespace Wabbajack.Compiler
_patchOptions = new();
_sourceFileLinks = new();
_patchCache = patchCache;
_updateStopWatch = new();
}
public void NextStep(string statusText, long maxStepProgress = 1)
{
_updateStopWatch.Restart();
_maxStepProgress = maxStepProgress;
_currentStep += 1;
_statusText = statusText;
_logger.LogInformation("Compiler Step: {Step}", statusText);
if (OnStatusUpdate != null)
{
OnStatusUpdate(this, new StatusUpdate($"[{_currentStep}/{MaxSteps}] " + statusText, Percent.FactoryPutInRange(_currentStep, MaxSteps),
Percent.Zero));
}
}
public void UpdateProgress(long stepProgress)
{
Interlocked.Add(ref _currentStepProgress, stepProgress);
lock (_updateStopWatch)
{
if (_updateStopWatch.ElapsedMilliseconds < 100) return;
_updateStopWatch.Restart();
}
if (OnStatusUpdate != null)
{
OnStatusUpdate(this, new StatusUpdate(_statusText, Percent.FactoryPutInRange(_currentStep, MaxSteps),
Percent.FactoryPutInRange(_currentStepProgress, _maxStepProgress)));
}
}
public abstract Task<bool> Begin(CancellationToken token);
public CompilerSettings Settings
@ -151,8 +195,10 @@ namespace Wabbajack.Compiler
public async Task<bool> GatherMetaData()
{
_logger.LogInformation("Getting meta data for {count} archives", SelectedArchives.Count);
NextStep("Gathering Metadata", SelectedArchives.Count);
await SelectedArchives.PDo(_parallelOptions, async a =>
{
UpdateProgress(1);
await _dispatcher.FillInMetadata(a);
});
@ -162,6 +208,7 @@ namespace Wabbajack.Compiler
protected async Task IndexGameFileHashes()
{
NextStep("Indexing Game Files", 1);
if (_settings.UseGamePaths)
{
//taking the games in Settings.IncludedGames + currently compiling game so you can eg
@ -215,6 +262,7 @@ namespace Wabbajack.Compiler
protected async Task CleanInvalidArchivesAndFillState()
{
NextStep("Cleaning Invalid Archives", 1);
var remove = await IndexedArchives.PMap(_parallelOptions, async a =>
{
try
@ -248,6 +296,7 @@ namespace Wabbajack.Compiler
protected async Task InferMetas(CancellationToken token)
{
async Task<bool> HasInvalidMeta(AbsolutePath filename)
{
var metaName = filename.WithExtension(Ext.Meta);
@ -275,6 +324,7 @@ namespace Wabbajack.Compiler
.Where(f => f.FileExists())
.ToList();
NextStep("InferMetas", toFind.Count);
if (toFind.Count == 0)
{
return;
@ -284,6 +334,7 @@ namespace Wabbajack.Compiler
await toFind.PDoAll(async f =>
{
UpdateProgress(1);
var vf = _vfs.Index.ByRootPath[f];
var archives = await _wjClient.GetArchivesForHash(vf.Hash);
@ -315,6 +366,7 @@ namespace Wabbajack.Compiler
protected async Task ExportModList(CancellationToken token)
{
NextStep("Exporting Modlist");
_logger.LogInformation("Exporting ModList to {location}", _settings.OutputFile);
// Modify readme and ModList image to relative paths if they exist
@ -382,8 +434,6 @@ namespace Wabbajack.Compiler
/// </summary>
protected async Task BuildPatches(CancellationToken token)
{
_logger.LogInformation("Gathering patch files");
var toBuild = InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => _patchOptions.GetValueOrDefault(p, Array.Empty<VirtualFile>()).Length > 0)
.SelectMany(p => _patchOptions[p].Select(c => new PatchedFromArchive
@ -395,6 +445,7 @@ namespace Wabbajack.Compiler
}))
.ToArray();
NextStep("Generating Patches", toBuild.Length);
if (toBuild.Length == 0)
{
return;
@ -406,6 +457,7 @@ namespace Wabbajack.Compiler
await _vfs.Extract( indexed.Keys.ToHashSet(),
async (vf, sf) =>
{
UpdateProgress(1);
// For each, extract the destination
var matches = indexed[vf];
foreach (var match in matches)
@ -484,6 +536,7 @@ namespace Wabbajack.Compiler
public async Task GenerateManifest()
{
NextStep("Generating Manifest");
var manifest = new Manifest(ModList);
await using var of = _settings.OutputFile.Open(FileMode.Create, FileAccess.Write);
await _dtos.Serialize(manifest, of);
@ -491,6 +544,7 @@ namespace Wabbajack.Compiler
public async Task GatherArchives()
{
NextStep("Gathering Archives");
_logger.LogInformation("Building a list of archives based on the files required");
var hashes = InstallDirectives.OfType<FromArchive>()
@ -502,7 +556,11 @@ namespace Wabbajack.Compiler
.ToDictionary(f => f.Key, f => f.First());
SelectedArchives.Clear();
SelectedArchives.AddRange(await hashes.PMap(_parallelOptions, hash => ResolveArchive(hash, archives)).ToList());
SelectedArchives.AddRange(await hashes.PMap(_parallelOptions, hash =>
{
UpdateProgress(1);
return ResolveArchive(hash, archives);
}).ToList());
}
public async Task<Archive> ResolveArchive(Hash hash, IDictionary<Hash, IndexedArchive> archives)
@ -596,11 +654,13 @@ namespace Wabbajack.Compiler
.GroupBy(f => _sourceFileLinks[f].File)
.ToDictionary(k => k.Key);
NextStep("Inlining Files");
if (grouped.Count == 0) return;
await _vfs.Extract(grouped.Keys.ToHashSet(), async (vf, sfn) =>
{
UpdateProgress(1);
await using var stream = await sfn.GetStream();
var id = await IncludeFile(stream);
var id = await IncludeFile(stream, token);
foreach (var file in grouped[vf])
{
file.SourceDataID = id;

2
Wabbajack.Compiler/Consts.cs
View File

@ -35,7 +35,7 @@ namespace Wabbajack.Compiler
public static readonly HashSet<Extension> SupportedBSAs = new[] {".bsa", ".ba2"}
.Select(s => new Extension(s)).ToHashSet();
public static HashSet<Extension> ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml", ".yaml"}.Select(s => new Extension(s)).ToHashSet();
public static HashSet<Extension> ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml", ".yaml", ".compiler_settings", ".mo2_compiler_settings"}.Select(s => new Extension(s)).ToHashSet();
public static HashSet<Extension> ESPFileExtensions = new []{ ".esp", ".esm", ".esl"}.Select(s => new Extension(s)).ToHashSet();
public static HashSet<Extension> AssetFileExtensions = new[] {".dds", ".tga", ".nif", ".psc", ".pex"}.Select(s => new Extension(s)).ToHashSet();

21
Wabbajack.Compiler/MO2Compiler.cs
View File

@ -29,6 +29,7 @@ namespace Wabbajack.Compiler
Client wjClient, IGameLocator locator, DTOSerializer dtos, IBinaryPatchCache patchCache) :
base(logger, extractor, hashCache, vfs, manager, settings, parallelOptions, dispatcher, wjClient, locator, dtos, patchCache)
{
MaxSteps = 14;
}
public AbsolutePath MO2ModsFolder => Settings.Source.Combine(Consts.MO2ModFolderName);
@ -52,11 +53,13 @@ namespace Wabbajack.Compiler
var roots = new List<AbsolutePath> {Settings.Source, Settings.Downloads};
roots.AddRange(Settings.OtherGames.Append(Settings.Game).Select(g => _locator.GameLocation(g)));
await _vfs.AddRoots(roots, token);
NextStep("Add Roots", 1);
await _vfs.AddRoots(roots, token); // Step 1
await InferMetas(token);
await InferMetas(token); // Step 2
await _vfs.AddRoot(Settings.Downloads, token);
NextStep("Add Download Roots", 1);
await _vfs.AddRoot(Settings.Downloads, token); // Step 3
// Find all Downloads
IndexedArchives = await Settings.Downloads.EnumerateFiles()
@ -69,7 +72,6 @@ namespace Wabbajack.Compiler
Meta = await f.WithExtension(Ext.Meta).ReadAllTextAsync()
}).ToList();
await IndexGameFileHashes();
IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();
@ -117,10 +119,15 @@ namespace Wabbajack.Compiler
var stack = MakeStack();
var results = await AllFiles.PMap(_parallelOptions, f => RunStack(stack, f)).ToList();
NextStep("Running Compilation Stack", AllFiles.Count);
var results = await AllFiles.PMap(_parallelOptions, f =>
{
UpdateProgress(1);
return RunStack(stack, f);
}).ToList();
NextStep("Updating Extra files");
// Add the extra files that were generated by the stack
results = results.Concat(ExtraFiles).ToList();
var noMatch = results.OfType<NoMatch>().ToArray();
@ -176,9 +183,11 @@ namespace Wabbajack.Compiler
private async Task RunValidation(ModList modList)
{
NextStep("Validating Archives", modList.Archives.Length);
var allowList = await _wjClient.LoadDownloadAllowList();
foreach (var archive in modList.Archives)
{
UpdateProgress(1);
if (!_dispatcher.IsAllowed(archive, allowList))
{
_logger.LogCritical("Archive {name}, {primaryKeyString} is not allowed", archive.Name,

2
Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
View File

@ -3,8 +3,6 @@ using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Compiler;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;