Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Abstract the base components of compilers/installers into a single abstract class

Browse Source
This commit is contained in:
Timothy Baldridge 2019-11-17 16:48:32 -07:00
parent 3e02500f4d
commit 0c78680c09
18 changed files with 199 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Wabbajack.Common/WorkQueue.cs
View File

@ -23,17 +23,16 @@ namespace Wabbajack.Common
private static readonly Subject<CPUStatus> _Status = new Subject<CPUStatus>();
public IObservable<CPUStatus> Status => _Status;
public static int ThreadCount { get; } = Environment.ProcessorCount;
public static List<Thread> Threads { get; private set; }
public WorkQueue()
public WorkQueue(int threadCount = 0)
{
StartThreads();
StartThreads(threadCount == 0 ? Environment.ProcessorCount : threadCount);
}
private void StartThreads()
private void StartThreads(int threadCount)
{
Threads = Enumerable.Range(0, ThreadCount)
Threads = Enumerable.Range(0, threadCount)
.Select(idx =>
{
var thread = new Thread(() => ThreadBody(idx));

106
Wabbajack.Lib/ABatchProcessor.cs Normal file
View File

@ -0,0 +1,106 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reactive.Subjects;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using ReactiveUI;
using Wabbajack.Common;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.Lib
{
public abstract class ABatchProcessor : IBatchProcessor
{
public WorkQueue Queue { get; private set; }
private bool _configured = false;
public void Dispose()
{
Queue?.Shutdown();
}
public Context VFS { get; private set; }
protected StatusUpdateTracker UpdateTracker { get; private set; }
private Subject<float> _percentCompleted { get; set; } = new Subject<float>();
/// <summary>
/// The current progress of the entire processing system on a scale of 0.0 to 1.0
/// </summary>
public IObservable<float> PercentCompleted { get; }
private Subject<string> _textStatus { get; set; } = new Subject<string>();
/// <summary>
/// The current status of the processor as a text string
/// </summary>
public IObservable<string> TextStatus { get; }
private Subject<CPUStatus> _QueueStatus { get; set; } = new Subject<CPUStatus>();
public IObservable<CPUStatus> QueueStatus { get; }
private Subject<bool> _IsRunning { get; set; } = new Subject<bool>();
public IObservable<bool> IsRunning { get; }
private Thread _processorThread { get; set; }
protected ABatchProcessor()
{
QueueStatus = _QueueStatus;
}
protected void ConfigureProcessor(int steps, int threads = 0)
{
if (_configured)
throw new InvalidDataException("Can't configure a processor twice");
Queue = new WorkQueue(threads);
UpdateTracker = new StatusUpdateTracker(steps);
Queue.Status.Subscribe(_QueueStatus);
UpdateTracker.Progress.Subscribe(_percentCompleted);
UpdateTracker.StepName.Subscribe(_textStatus);
VFS = new Context(Queue) { UpdateTracker = UpdateTracker };
_configured = true;
}
protected abstract bool _Begin();
public Task<bool> Begin()
{
_IsRunning.OnNext(true);
var _tcs = new TaskCompletionSource<bool>();
if (_processorThread != null)
{
throw new InvalidDataException("Can't start the processor twice");
}
_processorThread = new Thread(() =>
{
try
{
_tcs.SetResult(_Begin());
}
catch (Exception ex)
{
_tcs.SetException(ex);
}
finally
{
_IsRunning.OnNext(false);
}
});
_processorThread.Priority = ThreadPriority.BelowNormal;
_processorThread.Start();
return _tcs.Task;
}
public void Terminate()
{
Queue?.Shutdown();
_processorThread?.Abort();
_IsRunning.OnNext(false);
}
}
}

20
Wabbajack.Lib/ACompiler.cs
View File

@ -17,15 +17,11 @@ using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Lib
{
public abstract class ACompiler
public abstract class ACompiler : ABatchProcessor
{
public string ModListName, ModListAuthor, ModListDescription, ModListImage, ModListWebsite, ModListReadme;
public string WabbajackVersion;
public StatusUpdateTracker UpdateTracker { get; protected set; }
public WorkQueue Queue { get; protected set; }
protected static string _vfsCacheName = "vfs_compile_cache.bin";
/// <summary>
/// A stream of tuples of ("Update Title", 0.25) which represent the name of the current task
@ -34,8 +30,6 @@ namespace Wabbajack.Lib
public IObservable<(string, float)> ProgressUpdates => _progressUpdates;
protected readonly Subject<(string, float)> _progressUpdates = new Subject<(string, float)>();
public Context VFS { get; internal set; }
public ModManager ModManager;
public string GamePath;
@ -43,6 +37,8 @@ namespace Wabbajack.Lib
public string ModListOutputFolder;
public string ModListOutputFile;
public bool ShowReportWhenFinished { get; set; } = true;
public List<Archive> SelectedArchives = new List<Archive>();
public List<Directive> InstallDirectives = new List<Directive>();
public List<RawSourceFile> AllFiles = new List<RawSourceFile>();
@ -128,7 +124,7 @@ namespace Wabbajack.Lib
public void ShowReport()
{
//if (!ShowReportWhenFinished) return;
if (!ShowReportWhenFinished) return;
var file = Path.GetTempFileName() + ".html";
File.WriteAllText(file, ModList.ReportHTML);
@ -207,8 +203,6 @@ namespace Wabbajack.Lib
return null;
}
public abstract bool Compile();
public Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
{
Utils.Status($"Compiling {source.Path}");
@ -223,11 +217,5 @@ namespace Wabbajack.Lib
public abstract IEnumerable<ICompilationStep> GetStack();
public abstract IEnumerable<ICompilationStep> MakeStack();
protected ACompiler()
{
ProgressUpdates.Subscribe(itm => Utils.Log($"{itm.Item2} - {itm.Item1}"));
Queue = new WorkQueue();
}
}
}

13
Wabbajack.Lib/AInstaller.cs
View File

@ -12,14 +12,10 @@ using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Lib
{
public abstract class AInstaller
public abstract class AInstaller : ABatchProcessor
{
public bool IgnoreMissingFiles { get; internal set; } = false;
public StatusUpdateTracker UpdateTracker { get; protected set; }
public WorkQueue Queue { get; protected set; }
public Context VFS { get; internal set; }
public string OutputFolder { get; set; }
public string DownloadFolder { get; set; }
@ -29,13 +25,6 @@ namespace Wabbajack.Lib
public ModList ModList { get; internal set; }
public Dictionary<string, string> HashedArchives { get; set; }
protected AInstaller()
{
Queue = new WorkQueue();
}
public abstract void Install();
public void Info(string msg)
{
Utils.Log(msg);

9
Wabbajack.Lib/Compiler.cs
View File

@ -29,8 +29,6 @@ namespace Wabbajack.Lib
public Compiler(string mo2_folder)
{
UpdateTracker = new StatusUpdateTracker(10);
VFS = new Context(Queue) {UpdateTracker = UpdateTracker};
ModManager = ModManager.MO2;
MO2Folder = mo2_folder;
@ -39,14 +37,10 @@ namespace Wabbajack.Lib
ModListOutputFolder = "output_folder";
ModListOutputFile = MO2Profile + ExtensionManager.Extension;
VFS.ProgressUpdates.Debounce(new TimeSpan(0, 0, 0, 0, 100))
.Subscribe(itm => _progressUpdates.OnNext(itm));
}
public dynamic MO2Ini { get; }
public bool ShowReportWhenFinished { get; set; } = true;
public bool IgnoreMissingFiles { get; set; }
public string MO2DownloadsFolder
@ -71,8 +65,9 @@ namespace Wabbajack.Lib
public HashSet<string> SelectedProfiles { get; set; } = new HashSet<string>();
public override bool Compile()
protected override bool _Begin()
{
ConfigureProcessor(10);
UpdateTracker.Reset();
UpdateTracker.NextStep("Gathering information");
Info("Looking for other profiles");

50
Wabbajack.Lib/IBatchProcessor.cs Normal file
View File

@ -0,0 +1,50 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.Lib
{
/// <summary>
/// Wabbajack runs mostly as a batch processor of sorts, we have a list of tasks we need to perform
/// and the Compilers/Installers run throught those tasks one at a time. At any given moment the processor
/// will be using multiple threads to complete that task. This interface defines a common implementation of
/// all reporting functionality of both the compilers and installers.
///
/// These processors are disposible because they contain WorkQueues which must be properly shutdown to keep
/// from leaking threads.
/// </summary>
public interface IBatchProcessor : IDisposable
{
/// <summary>
/// The current progress of the entire processing system on a scale of 0.0 to 1.0
/// </summary>
IObservable<float> PercentCompleted { get; }
/// <summary>
/// The current status of the processor as a text string
/// </summary>
IObservable<string> TextStatus { get; }
/// <summary>
/// The status of the processor's work queue
/// </summary>
IObservable<CPUStatus> QueueStatus { get; }
IObservable<bool> IsRunning { get; }
/// <summary>
/// Begin processing
/// </summary>
Task<bool> Begin();
/// <summary>
/// Terminate any processing currently in progress by the processor. The processor should be disposed of
/// after calling this function as processing cannot be resumed and the tasks may be half completed.
/// Should only be called while IsRunning = true;
/// </summary>
void Terminate();
}
}

10
Wabbajack.Lib/Installer.cs
View File

@ -19,8 +19,6 @@ namespace Wabbajack.Lib
{
public Installer(string archive, ModList mod_list, string output_folder)
{
UpdateTracker = new StatusUpdateTracker(10);
VFS = new Context(Queue) {UpdateTracker = UpdateTracker};
ModManager = ModManager.MO2;
ModListArchive = archive;
OutputFolder = output_folder;
@ -30,8 +28,9 @@ namespace Wabbajack.Lib
public string GameFolder { get; set; }
public override void Install()
protected override bool _Begin()
{
ConfigureProcessor(10);
var game = GameRegistry.Games[ModList.GameType];
if (GameFolder == null)
@ -44,7 +43,7 @@ namespace Wabbajack.Lib
"game location up in the windows registry but were unable to find it, please make sure you launch the game once before running this installer. ",
"Could not find game location", MessageBoxButton.OK);
Utils.Log("Exiting because we couldn't find the game folder.");
return;
return false;
}
ValidateGameESMs();
@ -63,7 +62,7 @@ namespace Wabbajack.Lib
MessageBoxImage.Exclamation) == MessageBoxResult.No)
{
Utils.Log("Existing installation at the request of the user, existing mods folder found.");
return;
return false;
}
}
@ -97,6 +96,7 @@ namespace Wabbajack.Lib
// Removed until we decide if we want this functionality
// Nexus devs weren't sure this was a good idea, I (halgari) agree.
//AskToEndorse();
return true;
}
private void InstallIncludedDownloadMetas()

6
Wabbajack.Lib/VortexCompiler.cs
View File

@ -38,9 +38,6 @@ namespace Wabbajack.Lib
public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder, string stagingFolder)
{
UpdateTracker = new StatusUpdateTracker(10);
VFS = new Context(Queue) {UpdateTracker = UpdateTracker};
ModManager = ModManager.Vortex;
Game = game;
@ -54,8 +51,9 @@ namespace Wabbajack.Lib
ActiveArchives = new List<string>();
}
public override bool Compile()
protected override bool _Begin()
{
ConfigureProcessor(10);
if (string.IsNullOrEmpty(ModListName))
ModListName = $"Vortex ModList for {Game.ToString()}";
ModListOutputFile = $"{ModListName}{ExtensionManager.Extension}";

6
Wabbajack.Lib/VortexInstaller.cs
View File

@ -13,8 +13,6 @@ namespace Wabbajack.Lib
public VortexInstaller(string archive, ModList modList)
{
UpdateTracker = new StatusUpdateTracker(10);
VFS = new Context(Queue) {UpdateTracker = UpdateTracker};
ModManager = ModManager.Vortex;
ModListArchive = archive;
ModList = modList;
@ -25,8 +23,9 @@ namespace Wabbajack.Lib
GameInfo = GameRegistry.Games[ModList.GameType];
}
public override void Install()
protected override bool _Begin()
{
ConfigureProcessor(10);
Directory.CreateDirectory(DownloadFolder);
HashArchives();
@ -52,6 +51,7 @@ namespace Wabbajack.Lib
//InstallIncludedDownloadMetas();
Info("Installation complete! You may exit the program.");
return true;
}
private void InstallIncludedFiles()

2
Wabbajack.Lib/Wabbajack.Lib.csproj
View File

@ -78,6 +78,7 @@
<Reference Include="WindowsBase" />
</ItemGroup>
<ItemGroup>
<Compile Include="ABatchProcessor.cs" />
<Compile Include="ACompiler.cs" />
<Compile Include="AInstaller.cs" />
<Compile Include="CerasConfig.cs" />
@ -125,6 +126,7 @@
<Compile Include="Downloaders\MEGADownloader.cs" />
<Compile Include="Downloaders\ModDBDownloader.cs" />
<Compile Include="Downloaders\NexusDownloader.cs" />
<Compile Include="IBatchProcessor.cs" />
<Compile Include="Installer.cs" />
<Compile Include="ModListRegistry\ModListMetadata.cs" />
<Compile Include="NexusApi\Dtos.cs" />

4
Wabbajack.Test/ACompilerTest.cs
View File

@ -37,7 +37,7 @@ namespace Wabbajack.Test
var compiler = MakeCompiler();
compiler.MO2Profile = profile;
compiler.ShowReportWhenFinished = false;
Assert.IsTrue(compiler.Compile());
Assert.IsTrue(compiler.Begin().Result);
return compiler;
}
@ -59,7 +59,7 @@ namespace Wabbajack.Test
var installer = new Installer(compiler.ModListOutputFile, modlist, utils.InstallFolder);
installer.DownloadFolder = utils.DownloadsFolder;
installer.GameFolder = utils.GameFolder;
installer.Install();
installer.Begin().Wait();
}
}
}

4
Wabbajack.Test/AVortexCompilerTest.cs
View File

@ -37,7 +37,7 @@ namespace Wabbajack.Test
vortexCompiler.DownloadsFolder = utils.DownloadsFolder;
vortexCompiler.StagingFolder = utils.InstallFolder;
Directory.CreateDirectory(utils.InstallFolder);
Assert.IsTrue(vortexCompiler.Compile());
Assert.IsTrue(vortexCompiler.Begin().Result);
return vortexCompiler;
}
@ -66,7 +66,7 @@ namespace Wabbajack.Test
DownloadFolder = utils.DownloadsFolder,
GameFolder = utils.GameFolder,
};
installer.Install();
installer.Begin().Wait();
}
}
}

6
Wabbajack.Test/EndToEndTests.cs
View File

@ -77,7 +77,7 @@ namespace Wabbajack.Test
compiler.MO2DownloadsFolder = Path.Combine(utils.DownloadsFolder);
compiler.MO2Profile = profile;
compiler.ShowReportWhenFinished = false;
Assert.IsTrue(compiler.Compile());
Assert.IsTrue(compiler.Begin().Result);
}
@ -150,7 +150,7 @@ namespace Wabbajack.Test
var installer = new Installer(compiler.ModListOutputFile, modlist, utils.InstallFolder);
installer.DownloadFolder = utils.DownloadsFolder;
installer.GameFolder = utils.GameFolder;
installer.Install();
installer.Begin().Wait();
}
private Compiler ConfigureAndRunCompiler(string profile)
@ -158,7 +158,7 @@ namespace Wabbajack.Test
var compiler = new Compiler(utils.MO2Folder);
compiler.MO2Profile = profile;
compiler.ShowReportWhenFinished = false;
Assert.IsTrue(compiler.Compile());
Assert.IsTrue(compiler.Begin().Result);
return compiler;
}
}

4
Wabbajack.Test/SanityTests.cs
View File

@ -77,8 +77,8 @@ namespace Wabbajack.Test
// Update the file and verify that it throws an error.
utils.GenerateRandomFileData(game_file, 20);
var exception = Assert.ThrowsException<Exception>(() => Install(compiler));
Assert.AreEqual(exception.Message, "Game ESM hash doesn't match, is the ESM already cleaned? Please verify your local game files.");
var exception = Assert.ThrowsException<AggregateException>(() => Install(compiler));
Assert.AreEqual(exception.InnerExceptions.First().Message, "Game ESM hash doesn't match, is the ESM already cleaned? Please verify your local game files.");
}

5
Wabbajack/View Models/Compilers/MO2CompilerVM.cs
View File

@ -120,12 +120,11 @@ namespace Wabbajack
Utils.Log($"Compiler error: {ex.ExceptionToString()}");
return;
}
await Task.Run(() =>
await Task.Run(async () =>
{
try
{
this.StatusTracker = compiler.UpdateTracker;
compiler.Compile();
await compiler.Begin();
}
catch (Exception ex)
{

5
Wabbajack/View Models/Compilers/VortexCompilerVM.cs
View File

@ -106,12 +106,11 @@ namespace Wabbajack
Utils.Log($"Compiler error: {ex.ExceptionToString()}");
return;
}
await Task.Run(() =>
await Task.Run(async () =>
{
try
{
this.StatusTracker = compiler.UpdateTracker;
compiler.Compile();
await compiler.Begin();
}
catch (Exception ex)
{

11
Wabbajack/View Models/InstallerVM.cs
View File

@ -9,6 +9,7 @@ using System.Reactive.Disposables;
using System.Reactive.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Media.Imaging;
using Wabbajack.Common;
@ -295,11 +296,11 @@ namespace Wabbajack
{
DownloadFolder = DownloadLocation.TargetPath
};
var th = new Thread(() =>
Task.Run(async () =>
{
try
{
installer.Install();
await installer.Begin();
}
catch (Exception ex)
{
@ -313,11 +314,7 @@ namespace Wabbajack
this.Installing = false;
}
})
{
Priority = ThreadPriority.BelowNormal
};
th.Start();
});
}
}
}

5
Wabbajack/View Models/MainWindowVM.cs
View File

@ -83,9 +83,10 @@ namespace Wabbajack
// Compile progress updates and populate ObservableCollection
/*
WorkQueue.Status
_Compiler.WhenAny(c => c.Value.Compiler.)
.ObserveOn(RxApp.TaskpoolScheduler)
.ToObservableChangeSet(x => x.ID)
.ToObservableChangeSet(x => x.)
/*
.Batch(TimeSpan.FromMilliseconds(250), RxApp.TaskpoolScheduler)
.EnsureUniqueChanges()
.ObserveOn(RxApp.MainThreadScheduler)