Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Large swap to make WorkQueue able to take Task work items

Browse Source
This commit is contained in:
Justin Swanson 2019-12-03 19:26:26 -06:00
parent f2014a1da3
commit e9deda9f44
50 changed files with 370 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Compression.BSA.Test/BSATests.cs
View File

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Newtonsoft.Json;
@ -28,7 +29,7 @@ namespace Compression.BSA.Test
private static WorkQueue Queue { get; set; }
[ClassInitialize]
public static void Setup(TestContext testContext)
public static async Task Setup(TestContext testContext)
{
Queue = new WorkQueue();
Utils.LogMessages.Subscribe(f => testContext.WriteLine(f.ShortDescription));
@ -52,13 +53,10 @@ namespace Compression.BSA.Test
var folder = Path.Combine(_bsaFolder, info.Item1.ToString(), info.Item2.ToString());
if (!Directory.Exists(folder))
Directory.CreateDirectory(folder);
FileExtractor.ExtractAll(Queue, filename, folder);
await FileExtractor.ExtractAll(Queue, filename, folder);
}
}
private static string DownloadMod((Game, int) info)
{
using (var client = new NexusApiClient())
@ -91,7 +89,7 @@ namespace Compression.BSA.Test
[TestMethod]
[DataTestMethod]
[DynamicData(nameof(BSAs), DynamicDataSourceType.Method)]
public void BSACompressionRecompression(string bsa)
public async Task BSACompressionRecompression(string bsa)
{
TestContext.WriteLine($"From {bsa}");
TestContext.WriteLine("Cleaning Output Dir");
@ -102,7 +100,7 @@ namespace Compression.BSA.Test
string tempFile = Path.Combine("tmp.bsa");
using (var a = BSADispatch.OpenRead(bsa))
{
a.Files.PMap(Queue, file =>
await a.Files.PMap(Queue, file =>
{
var absName = Path.Combine(_tempDir, file.Path);
ViaJson(file.State);
@ -123,7 +121,7 @@ namespace Compression.BSA.Test
using (var w = ViaJson(a.State).MakeBuilder())
{
a.Files.PMap(Queue, file =>
await a.Files.PMap(Queue, file =>
{
var absPath = Path.Combine(_tempDir, file.Path);
using (var str = File.OpenRead(absPath))
@ -145,7 +143,7 @@ namespace Compression.BSA.Test
Assert.AreEqual(a.Files.Count(), b.Files.Count());
var idx = 0;
a.Files.Zip(b.Files, (ai, bi) => (ai, bi))
await a.Files.Zip(b.Files, (ai, bi) => (ai, bi))
.PMap(Queue, pair =>
{
idx++;

7
Wabbajack.CacheServer/NexusCacheModule.cs
View File

@ -23,13 +23,16 @@ namespace Wabbajack.CacheServer
Get("/v1/games/{GameName}/mods/{ModID}.json", HandleModInfo);
Get("/nexus_api_cache/{request}.json", HandleCacheCall);
Get("/nexus_api_cache", ListCache);
// ToDo
// Handle what to do with the fact that this is now a task
throw new NotImplementedException("Unsure if following function still works when taking in a Task");
Get("/nexus_api_cache/update", UpdateCache);
}
private object UpdateCache(object arg)
private async Task<object> UpdateCache(object arg)
{
var api = new NexusApiClient(Request.Headers["apikey"].FirstOrDefault());
api.ClearUpdatedModsInCache();
await api.ClearUpdatedModsInCache();
return "Done";
}

9
Wabbajack.Common/FileExtractor.cs
View File

@ -5,6 +5,7 @@ using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Security.Cryptography;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Compression.BSA;
using ICSharpCode.SharpZipLib.GZip;
@ -37,12 +38,12 @@ namespace Wabbajack.Common
}
public static void ExtractAll(WorkQueue queue, string source, string dest)
public static async Task ExtractAll(WorkQueue queue, string source, string dest)
{
try
{
if (Consts.SupportedBSAs.Any(b => source.ToLower().EndsWith(b)))
ExtractAllWithBSA(queue, source, dest);
await ExtractAllWithBSA(queue, source, dest);
else if (source.EndsWith(".omod"))
ExtractAllWithOMOD(source, dest);
else if (source.EndsWith(".exe"))
@ -125,13 +126,13 @@ namespace Wabbajack.Common
return dest;
}
private static void ExtractAllWithBSA(WorkQueue queue, string source, string dest)
private static async Task ExtractAllWithBSA(WorkQueue queue, string source, string dest)
{
try
{
using (var arch = BSADispatch.OpenRead(source))
{
arch.Files
await arch.Files
.PMap(queue, f =>
{
var path = f.Path;

86
Wabbajack.Common/Utils.cs
View File

@ -482,24 +482,36 @@ namespace Wabbajack.Common
}
}
public static TR[] PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue, StatusUpdateTracker updateTracker,
public static async Task<TR[]> PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue, StatusUpdateTracker updateTracker,
Func<TI, TR> f)
{
var cnt = 0;
var collist = coll.ToList();
return collist.PMap(queue, itm =>
return await collist.PMap(queue, itm =>
{
updateTracker.MakeUpdate(collist.Count, Interlocked.Increment(ref cnt));
return f(itm);
});
}
public static void PMap<TI>(this IEnumerable<TI> coll, WorkQueue queue, StatusUpdateTracker updateTracker,
public static async Task<TR[]> PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue, StatusUpdateTracker updateTracker,
Func<TI, Task<TR>> f)
{
var cnt = 0;
var collist = coll.ToList();
return await collist.PMap(queue, itm =>
{
updateTracker.MakeUpdate(collist.Count, Interlocked.Increment(ref cnt));
return f(itm);
});
}
public static async Task PMap<TI>(this IEnumerable<TI> coll, WorkQueue queue, StatusUpdateTracker updateTracker,
Action<TI> f)
{
var cnt = 0;
var collist = coll.ToList();
collist.PMap(queue, itm =>
await collist.PMap(queue, itm =>
{
updateTracker.MakeUpdate(collist.Count, Interlocked.Increment(ref cnt));
f(itm);
@ -508,17 +520,17 @@ namespace Wabbajack.Common
}
public static TR[] PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue,
public static async Task<TR[]> PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue,
Func<TI, TR> f)
{
var colllst = coll.ToList();
var remainingTasks = colllst.Count;
var tasks = coll.Select(i =>
var tasks = colllst.Select(i =>
{
var tc = new TaskCompletionSource<TR>();
queue.QueueTask(() =>
queue.QueueTask(async () =>
{
try
{
@ -537,20 +549,49 @@ namespace Wabbajack.Common
if (WorkQueue.WorkerThread)
while (remainingTasks > 0)
if (queue.Queue.TryTake(out var a, 500))
a();
await a();
return tasks.Select(t =>
{
t.Wait();
if (t.IsFaulted)
throw t.Exception;
return t.Result;
}).ToArray();
return await Task.WhenAll(tasks);
}
public static void PMap<TI>(this IEnumerable<TI> coll, WorkQueue queue, Action<TI> f)
public static async Task<TR[]> PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue,
Func<TI, Task<TR>> f)
{
coll.PMap(queue, i =>
var colllst = coll.ToList();
var remainingTasks = colllst.Count;
var tasks = colllst.Select(i =>
{
var tc = new TaskCompletionSource<TR>();
queue.QueueTask(async () =>
{
try
{
tc.SetResult(await f(i));
}
catch (Exception ex)
{
tc.SetException(ex);
}
Interlocked.Decrement(ref remainingTasks);
});
return tc.Task;
}).ToList();
// To avoid thread starvation, we'll start to help out in the work queue
if (WorkQueue.WorkerThread)
while (remainingTasks > 0)
if (queue.Queue.TryTake(out var a, 500))
await a();
return await Task.WhenAll(tasks);
}
public static async Task PMap<TI>(this IEnumerable<TI> coll, WorkQueue queue, Action<TI> f)
{
await coll.PMap(queue, i =>
{
f(i);
return false;
@ -767,11 +808,11 @@ namespace Wabbajack.Common
Log(s);
}
private static long TestDiskSpeedInner(WorkQueue queue, string path)
private static async Task<long> TestDiskSpeedInner(WorkQueue queue, string path)
{
var startTime = DateTime.Now;
var seconds = 2;
return Enumerable.Range(0, queue.ThreadCount)
var results = await Enumerable.Range(0, queue.ThreadCount)
.PMap(queue, idx =>
{
var random = new Random();
@ -792,16 +833,17 @@ namespace Wabbajack.Common
}
File.Delete(file);
return size;
}).Sum() / seconds;
});
return results.Sum() / seconds;
}
private static Dictionary<string, long> _cachedDiskSpeeds = new Dictionary<string, long>();
public static long TestDiskSpeed(WorkQueue queue, string path)
public static async Task<long> TestDiskSpeed(WorkQueue queue, string path)
{
var driveName = Volume.GetUniqueVolumeNameForPath(path);
if (_cachedDiskSpeeds.TryGetValue(driveName, out long speed))
return speed;
speed = TestDiskSpeedInner(queue, path);
speed = await TestDiskSpeedInner(queue, path);
_cachedDiskSpeeds[driveName] = speed;
return speed;
}

22
Wabbajack.Common/WorkQueue.cs
View File

@ -5,14 +5,15 @@ using System.Linq;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Common.StatusFeed;
namespace Wabbajack.Common
{
public class WorkQueue : IDisposable
{
internal BlockingCollection<Action>
Queue = new BlockingCollection<Action>(new ConcurrentStack<Action>());
internal BlockingCollection<Func<Task>>
Queue = new BlockingCollection<Func<Task>>(new ConcurrentStack<Func<Task>>());
[ThreadStatic] private static int CpuId;
@ -37,7 +38,7 @@ namespace Wabbajack.Common
Threads = Enumerable.Range(0, threadCount)
.Select(idx =>
{
var thread = new Thread(() => ThreadBody(idx));
var thread = new Thread(() => ThreadBody(idx).Wait());
thread.Priority = ThreadPriority.BelowNormal;
thread.IsBackground = true;
thread.Name = string.Format("Wabbajack_Worker_{0}", idx);
@ -48,7 +49,7 @@ namespace Wabbajack.Common
public int ThreadCount { get; private set; }
private void ThreadBody(int idx)
private async Task ThreadBody(int idx)
{
CpuId = idx;
CurrentQueue = this;
@ -58,8 +59,9 @@ namespace Wabbajack.Common
while (true)
{
Report("Waiting", 0, false);
if (_cancel.IsCancellationRequested) return;
var f = Queue.Take(_cancel.Token);
f();
await f();
}
}
catch (OperationCanceledException)
@ -80,7 +82,7 @@ namespace Wabbajack.Common
});
}
public void QueueTask(Action a)
public void QueueTask(Func<Task> a)
{
Queue.Add(a);
}
@ -88,7 +90,13 @@ namespace Wabbajack.Common
public void Dispose()
{
_cancel.Cancel();
Threads.Do(th => th.Join());
Threads.Do(th =>
{
if (th.ManagedThreadId != Thread.CurrentThread.ManagedThreadId)
{
th.Join();
}
});
Queue?.Dispose();
}
}

25
Wabbajack.Lib/ABatchProcessor.cs
View File

@ -35,8 +35,6 @@ namespace Wabbajack.Lib
private Subject<bool> _isRunning { get; } = new Subject<bool>();
public IObservable<bool> IsRunning => _isRunning;
private Thread _processorThread { get; set; }
private int _configured;
private int _started;
@ -56,7 +54,7 @@ namespace Wabbajack.Lib
VFS = new Context(Queue) { UpdateTracker = UpdateTracker };
}
public static int RecommendQueueSize(string folder)
public static async Task<int> RecommendQueueSize(string folder)
{
if (!Directory.Exists(folder))
Directory.CreateDirectory(folder);
@ -64,7 +62,7 @@ namespace Wabbajack.Lib
using (var queue = new WorkQueue())
{
Utils.Log($"Benchmarking {folder}");
var raw_speed = Utils.TestDiskSpeed(queue, folder);
var raw_speed = await Utils.TestDiskSpeed(queue, folder);
Utils.Log($"{raw_speed.ToFileSizeString()}/sec for {folder}");
int speed = (int)(raw_speed / 1024 / 1024);
@ -73,7 +71,7 @@ namespace Wabbajack.Lib
}
}
protected abstract bool _Begin(CancellationToken cancel);
protected abstract Task<bool> _Begin(CancellationToken cancel);
public Task<bool> Begin()
{
if (1 == Interlocked.CompareExchange(ref _started, 1, 1))
@ -81,27 +79,18 @@ namespace Wabbajack.Lib
throw new InvalidDataException("Can't start the processor twice");
}
_isRunning.OnNext(true);
var _tcs = new TaskCompletionSource<bool>();
_processorThread = new Thread(() =>
{
return Task.Run(async () =>
{
try
{
_tcs.SetResult(_Begin(_cancel.Token));
}
catch (Exception ex)
{
_tcs.SetException(ex);
_isRunning.OnNext(true);
return await _Begin(_cancel.Token);
}
finally
{
_isRunning.OnNext(false);
}
});
_processorThread.Priority = ThreadPriority.BelowNormal;
_processorThread.Start();
return _tcs.Task;
}
public void Dispose()

9
Wabbajack.Lib/ACompiler.cs
View File

@ -5,6 +5,7 @@ using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Reactive.Subjects;
using System.Threading.Tasks;
using System.Threading;
using CommonMark;
using Wabbajack.Common;
@ -189,7 +190,7 @@ namespace Wabbajack.Lib
+ CommonMarkConverter.Convert(File.ReadAllText($"{ModList.Name}.md"));
}
public void GatherArchives()
public async Task GatherArchives()
{
Info("Building a list of archives based on the files required");
@ -201,7 +202,7 @@ namespace Wabbajack.Lib
.GroupBy(f => f.File.Hash)
.ToDictionary(f => f.Key, f => f.First());
SelectedArchives = shas.PMap(Queue, sha => ResolveArchive(sha, archives));
SelectedArchives = await shas.PMap(Queue, sha => ResolveArchive(sha, archives));
}
public Archive ResolveArchive(string sha, IDictionary<string, IndexedArchive> archives)
@ -237,12 +238,12 @@ namespace Wabbajack.Lib
return null;
}
public Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
public async Task<Directive> RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
{
Utils.Status($"Compiling {source.Path}");
foreach (var step in stack)
{
var result = step.Run(source);
var result = await step.Run(source);
if (result != null) return result;
}

41
Wabbajack.Lib/AInstaller.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
@ -115,7 +116,7 @@ namespace Wabbajack.Lib
});
}
public void InstallArchives()
public async Task InstallArchives()
{
Info("Installing Archives");
Info("Grouping Install Files");
@ -129,10 +130,10 @@ namespace Wabbajack.Lib
.ToList();
Info("Installing Archives");
archives.PMap(Queue, UpdateTracker,a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash]));
await archives.PMap(Queue, UpdateTracker,a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash]));
}
private void InstallArchive(Archive archive, string absolutePath, IGrouping<string, FromArchive> grouping)
private async Task InstallArchive(Archive archive, string absolutePath, IGrouping<string, FromArchive> grouping)
{
Status($"Extracting {archive.Name}");
@ -143,7 +144,7 @@ namespace Wabbajack.Lib
return g;
}).ToList();
var onFinish = VFS.Stage(vFiles.Select(f => f.FromFile).Distinct());
var onFinish = await VFS.Stage(vFiles.Select(f => f.FromFile).Distinct());
Status($"Copying files for {archive.Name}");
@ -221,7 +222,7 @@ namespace Wabbajack.Lib
}
}
public void DownloadArchives()
public async Task DownloadArchives()
{
var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
Info($"Missing {missing.Count} archives");
@ -233,10 +234,10 @@ namespace Wabbajack.Lib
foreach (var dispatcher in dispatchers)
dispatcher.Prepare();
DownloadMissingArchives(missing);
await DownloadMissingArchives(missing);
}
private void DownloadMissingArchives(List<Archive> missing, bool download = true)
private async Task DownloadMissingArchives(List<Archive> missing, bool download = true)
{
if (download)
{
@ -247,7 +248,7 @@ namespace Wabbajack.Lib
}
}
missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State))
await missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State))
.PMap(Queue, archive =>
{
Info($"Downloading {archive.Name}");
@ -280,11 +281,12 @@ namespace Wabbajack.Lib
return false;
}
public void HashArchives()
public async Task HashArchives()
{
HashedArchives = Directory.EnumerateFiles(DownloadFolder)
var hashResults = await Directory.EnumerateFiles(DownloadFolder)
.Where(e => !e.EndsWith(Consts.HashFileExtension))
.PMap(Queue, e => (e.FileHashCached(), e))
.PMap(Queue, e => (e.FileHashCached(), e));
HashedArchives = hashResults
.OrderByDescending(e => File.GetLastWriteTime(e.Item2))
.GroupBy(e => e.Item1)
.Select(e => e.First())
@ -322,11 +324,11 @@ namespace Wabbajack.Lib
*/
}
public int RecommendQueueSize()
public async Task<int> RecommendQueueSize()
{
var output_size = RecommendQueueSize(OutputFolder);
var download_size = RecommendQueueSize(DownloadFolder);
var scratch_size = RecommendQueueSize(Directory.GetCurrentDirectory());
var output_size = await RecommendQueueSize(OutputFolder);
var download_size = await RecommendQueueSize(DownloadFolder);
var scratch_size = await RecommendQueueSize(Directory.GetCurrentDirectory());
var result = Math.Min(output_size, Math.Min(download_size, scratch_size));
Utils.Log($"Recommending a queue size of {result} based on disk performance and number of cores");
return result;
@ -337,13 +339,13 @@ namespace Wabbajack.Lib
/// The user may already have some files in the OutputFolder. If so we can go through these and
/// figure out which need to be updated, deleted, or left alone
/// </summary>
public void OptimizeModlist()
public async Task OptimizeModlist()
{
Utils.Log("Optimizing Modlist directives");
var indexed = ModList.Directives.ToDictionary(d => d.To);
UpdateTracker.NextStep("Looking for files to delete");
Directory.EnumerateFiles(OutputFolder, "*", DirectoryEnumerationOptions.Recursive)
await Directory.EnumerateFiles(OutputFolder, "*", DirectoryEnumerationOptions.Recursive)
.PMap(Queue, UpdateTracker, f =>
{
var relative_to = f.RelativeTo(OutputFolder);
@ -358,7 +360,7 @@ namespace Wabbajack.Lib
});
UpdateTracker.NextStep("Looking for unmodified files");
indexed.Values.PMap(Queue, UpdateTracker, d =>
(await indexed.Values.PMap(Queue, UpdateTracker, d =>
{
// Bit backwards, but we want to return null for
// all files we *want* installed. We return the files
@ -371,7 +373,8 @@ namespace Wabbajack.Lib
if (fi.Length != d.Size) return null;
return path.FileHash() == d.Hash ? d : null;
}).Where(d => d != null)
}))
.Where(d => d != null)
.Do(d => indexed.Remove(d.To));
UpdateTracker.NextStep("Updating Modlist");

8
Wabbajack.Lib/CompilationSteps/ACompilationStep.cs
View File

@ -1,4 +1,6 @@
namespace Wabbajack.Lib.CompilationSteps
using System.Threading.Tasks;
namespace Wabbajack.Lib.CompilationSteps
{
public abstract class ACompilationStep : ICompilationStep
{
@ -9,7 +11,7 @@
_compiler = compiler;
}
public abstract Directive Run(RawSourceFile source);
public abstract ValueTask<Directive> Run(RawSourceFile source);
public abstract IState GetState();
}
}
}

5
Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs
View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Compression.BSA;
using Newtonsoft.Json;
@ -49,7 +50,7 @@ namespace Wabbajack.Lib.CompilationSteps
return new State();
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path).ToLower())) return null;
@ -64,7 +65,7 @@ namespace Wabbajack.Lib.CompilationSteps
var id = Guid.NewGuid().ToString();
var matches = sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e)
var matches = await sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e)
{
Path = Path.Combine(Consts.BSACreationDir, id, e.Name)
}));

5
Wabbajack.Lib/CompilationSteps/DirectMatch.cs
View File

@ -1,4 +1,5 @@
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
@ -10,7 +11,7 @@ namespace Wabbajack.Lib.CompilationSteps
{
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!_compiler.IndexedFiles.TryGetValue(source.Hash, out var found)) return null;
var result = source.EvolveTo<FromArchive>();
@ -39,4 +40,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

7
Wabbajack.Lib/CompilationSteps/DropAll.cs
View File

@ -1,4 +1,5 @@
using Newtonsoft.Json;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Wabbajack.Common;
namespace Wabbajack.Lib.CompilationSteps
@ -9,7 +10,7 @@ namespace Wabbajack.Lib.CompilationSteps
{
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
var result = source.EvolveTo<NoMatch>();
result.Reason = "No Match in Stack";
@ -31,4 +32,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

8
Wabbajack.Lib/CompilationSteps/IStackStep.cs
View File

@ -1,8 +1,10 @@
namespace Wabbajack.Lib.CompilationSteps
using System.Threading.Tasks;
namespace Wabbajack.Lib.CompilationSteps
{
public interface ICompilationStep
{
Directive Run(RawSourceFile source);
ValueTask<Directive> Run(RawSourceFile source);
IState GetState();
}
@ -10,4 +12,4 @@
{
ICompilationStep CreateStep(ACompiler compiler);
}
}
}

5
Wabbajack.Lib/CompilationSteps/IgnoreDisabledMods.cs
View File

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
using Wabbajack.Common;
@ -25,7 +26,7 @@ namespace Wabbajack.Lib.CompilationSteps
.ToList();
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!source.Path.StartsWith("mods") || _allEnabledMods.Any(mod => source.Path.StartsWith(mod)))
return null;
@ -63,4 +64,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

5
Wabbajack.Lib/CompilationSteps/IgnoreDisabledVortexMods.cs
View File

@ -1,4 +1,5 @@
using Wabbajack.Common;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.Lib.CompilationSteps
{
@ -11,7 +12,7 @@ namespace Wabbajack.Lib.CompilationSteps
_vortexCompiler = (VortexCompiler) compiler;
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
var b = false;
_vortexCompiler.ActiveArchives.Do(a =>

7
Wabbajack.Lib/CompilationSteps/IgnoreEndsWith.cs
View File

@ -1,4 +1,5 @@
using Newtonsoft.Json;
using System.Threading.Tasks;
using Newtonsoft.Json;
namespace Wabbajack.Lib.CompilationSteps
{
@ -13,7 +14,7 @@ namespace Wabbajack.Lib.CompilationSteps
_reason = $"Ignored because path ends with {postfix}";
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!source.Path.EndsWith(_postfix)) return null;
var result = source.EvolveTo<IgnoredDirectly>();
@ -46,4 +47,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

7
Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs
View File

@ -1,4 +1,5 @@
using Newtonsoft.Json;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Wabbajack.Common;
namespace Wabbajack.Lib.CompilationSteps
@ -12,7 +13,7 @@ namespace Wabbajack.Lib.CompilationSteps
_startDir = Consts.GameFolderFilesDir + "\\";
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!source.Path.StartsWith(_startDir)) return null;
var i = source.EvolveTo<IgnoredDirectly>();
@ -34,4 +35,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

7
Wabbajack.Lib/CompilationSteps/IgnorePathContains.cs
View File

@ -1,4 +1,5 @@
using Newtonsoft.Json;
using System.Threading.Tasks;
using Newtonsoft.Json;
namespace Wabbajack.Lib.CompilationSteps
{
@ -13,7 +14,7 @@ namespace Wabbajack.Lib.CompilationSteps
_reason = $"Ignored because path contains {_pattern}";
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!source.Path.Contains(_pattern)) return null;
var result = source.EvolveTo<IgnoredDirectly>();
@ -46,4 +47,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

5
Wabbajack.Lib/CompilationSteps/IgnoreRegex.cs
View File

@ -1,4 +1,5 @@
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Newtonsoft.Json;
namespace Wabbajack.Lib.CompilationSteps
@ -16,7 +17,7 @@ namespace Wabbajack.Lib.CompilationSteps
_regex = new Regex(pattern);
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!_regex.IsMatch(source.Path)) return null;
var result = source.EvolveTo<IgnoredDirectly>();
@ -49,4 +50,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

7
Wabbajack.Lib/CompilationSteps/IgnoreStartsWith.cs
View File

@ -1,4 +1,5 @@
using Newtonsoft.Json;
using System.Threading.Tasks;
using Newtonsoft.Json;
namespace Wabbajack.Lib.CompilationSteps
{
@ -13,7 +14,7 @@ namespace Wabbajack.Lib.CompilationSteps
_reason = string.Format("Ignored because path starts with {0}", _prefix);
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (source.Path.StartsWith(_prefix))
{
@ -50,4 +51,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

3
Wabbajack.Lib/CompilationSteps/IgnoreVortex.cs
View File

@ -1,4 +1,5 @@
using System.IO;
using System.Threading.Tasks;
using Newtonsoft.Json;
namespace Wabbajack.Lib.CompilationSteps
@ -12,7 +13,7 @@ namespace Wabbajack.Lib.CompilationSteps
_vortex = (VortexCompiler) compiler;
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (Path.GetDirectoryName(source.AbsolutePath) != _vortex.DownloadsFolder) return null;
var result = source.EvolveTo<IgnoredDirectly>();

5
Wabbajack.Lib/CompilationSteps/IgnoreWabbajackInstallCruft.cs
View File

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Wabbajack.Common;
@ -18,7 +19,7 @@ namespace Wabbajack.Lib.CompilationSteps
};
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!_cruftFiles.Any(f => source.Path.StartsWith(f))) return null;
var result = source.EvolveTo<IgnoredDirectly>();
@ -40,4 +41,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

7
Wabbajack.Lib/CompilationSteps/IncludeAll.cs
View File

@ -1,4 +1,5 @@
using Alphaleonis.Win32.Filesystem;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
namespace Wabbajack.Lib.CompilationSteps
@ -9,7 +10,7 @@ namespace Wabbajack.Lib.CompilationSteps
{
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
var inline = source.EvolveTo<InlineFile>();
inline.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath));
@ -30,4 +31,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

7
Wabbajack.Lib/CompilationSteps/IncludeAllConfigs.cs
View File

@ -1,4 +1,5 @@
using Alphaleonis.Win32.Filesystem;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
using Wabbajack.Common;
@ -10,7 +11,7 @@ namespace Wabbajack.Lib.CompilationSteps
{
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!Consts.ConfigFileExtensions.Contains(Path.GetExtension(source.Path))) return null;
var result = source.EvolveTo<InlineFile>();
@ -32,4 +33,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

7
Wabbajack.Lib/CompilationSteps/IncludeDummyESPs.cs
View File

@ -1,4 +1,5 @@
using Alphaleonis.Win32.Filesystem;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
namespace Wabbajack.Lib.CompilationSteps
@ -9,7 +10,7 @@ namespace Wabbajack.Lib.CompilationSteps
{
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (Path.GetExtension(source.AbsolutePath) != ".esp" &&
Path.GetExtension(source.AbsolutePath) != ".esm") return null;
@ -42,4 +43,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

7
Wabbajack.Lib/CompilationSteps/IncludeLOOTFiles.cs
View File

@ -1,4 +1,5 @@
using Alphaleonis.Win32.Filesystem;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
using Wabbajack.Common;
@ -13,7 +14,7 @@ namespace Wabbajack.Lib.CompilationSteps
_prefix = Consts.LOOTFolderFilesDir + "\\";
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!source.Path.StartsWith(_prefix)) return null;
var result = source.EvolveTo<InlineFile>();
@ -35,4 +36,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

7
Wabbajack.Lib/CompilationSteps/IncludeModIniData.cs
View File

@ -1,4 +1,5 @@
using Alphaleonis.Win32.Filesystem;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
namespace Wabbajack.Lib.CompilationSteps
@ -9,7 +10,7 @@ namespace Wabbajack.Lib.CompilationSteps
{
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!source.Path.StartsWith("mods\\") || !source.Path.EndsWith("\\meta.ini")) return null;
var e = source.EvolveTo<InlineFile>();
@ -31,4 +32,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

5
Wabbajack.Lib/CompilationSteps/IncludeOtherProfiles.cs
View File

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
@ -19,7 +20,7 @@ namespace Wabbajack.Lib.CompilationSteps
.ToList();
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!source.Path.StartsWith("profiles\\")) return null;
if (_profiles.Any(profile => source.Path.StartsWith(profile))) return null;
@ -42,4 +43,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

3
Wabbajack.Lib/CompilationSteps/IncludePatches.cs
View File

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
using Wabbajack.Common;
@ -19,7 +20,7 @@ namespace Wabbajack.Lib.CompilationSteps
.ToDictionary(f => f.Key);
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!_indexed.TryGetValue(Path.GetFileName(source.File.Name.ToLower()), out var choices))
return null;

5
Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs
View File

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
@ -12,7 +13,7 @@ namespace Wabbajack.Lib.CompilationSteps
{
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
var files = new HashSet<string>
{
@ -52,4 +53,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

5
Wabbajack.Lib/CompilationSteps/IncludeRegex.cs
View File

@ -1,4 +1,5 @@
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
@ -15,7 +16,7 @@ namespace Wabbajack.Lib.CompilationSteps
_regex = new Regex(pattern);
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!_regex.IsMatch(source.Path)) return null;
@ -49,4 +50,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

3
Wabbajack.Lib/CompilationSteps/IncludeSteamWorkshopItems.cs
View File

@ -1,6 +1,7 @@
using System;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
@ -16,7 +17,7 @@ namespace Wabbajack.Lib.CompilationSteps
_game = steamGame;
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!_regex.IsMatch(source.Path))
return null;

5
Wabbajack.Lib/CompilationSteps/IncludeStubbedConfigfiles.cs
View File

@ -1,4 +1,5 @@
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
using Wabbajack.Common;
@ -14,7 +15,7 @@ namespace Wabbajack.Lib.CompilationSteps
_mo2Compiler = (MO2Compiler) compiler;
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
return Consts.ConfigFileExtensions.Contains(Path.GetExtension(source.Path)) ? RemapFile(source) : null;
}
@ -58,4 +59,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

5
Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs
View File

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
@ -24,7 +25,7 @@ namespace Wabbajack.Lib.CompilationSteps
}).Select(kv => $"mods\\{kv.Key}\\");
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!source.Path.StartsWith("mods")) return null;
foreach (var modpath in _includeDirectly)
@ -63,4 +64,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

5
Wabbajack.Lib/CompilationSteps/IncludeThisProfile.cs
View File

@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
@ -17,7 +18,7 @@ namespace Wabbajack.Lib.CompilationSteps
_correctProfiles = _mo2Compiler.SelectedProfiles.Select(p => Path.Combine("profiles", p) + "\\").ToList();
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (_correctProfiles.Any(p => source.Path.StartsWith(p)))
{
@ -56,4 +57,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

4
Wabbajack.Lib/CompilationSteps/IncludeVortexDeployment.cs
View File

@ -1,4 +1,5 @@
using System.IO;
using System.Threading.Tasks;
namespace Wabbajack.Lib.CompilationSteps
{
@ -8,9 +9,8 @@ namespace Wabbajack.Lib.CompilationSteps
{
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!source.Path.EndsWith("vortex.deployment.msgpack") &&
!source.Path.EndsWith("\\vortex.deployment.json") && Path.GetExtension(source.Path) != ".meta") return null;
var inline = source.EvolveTo<InlineFile>();

5
Wabbajack.Lib/CompilationSteps/PatchStockESMs.cs
View File

@ -1,4 +1,5 @@
using System.IO;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Wabbajack.Common;
using File = Alphaleonis.Win32.Filesystem.File;
@ -15,7 +16,7 @@ namespace Wabbajack.Lib.CompilationSteps
_mo2Compiler = (MO2Compiler) compiler;
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
var filename = Path.GetFileName(source.Path);
var gameFile = Path.Combine(_mo2Compiler.GamePath, "Data", filename);
@ -53,4 +54,4 @@ namespace Wabbajack.Lib.CompilationSteps
}
}
}
}
}

26
Wabbajack.Lib/MO2Compiler.cs
View File

@ -71,7 +71,7 @@ namespace Wabbajack.Lib
public HashSet<string> SelectedProfiles { get; set; } = new HashSet<string>();
protected override bool _Begin(CancellationToken cancel)
protected override async Task<bool> _Begin(CancellationToken cancel)
{
if (cancel.IsCancellationRequested) return false;
ConfigureProcessor(16);
@ -105,8 +105,8 @@ namespace Wabbajack.Lib
UpdateTracker.NextStep("Indexing folders");
if (cancel.IsCancellationRequested) return false;
VFS.AddRoots(roots);
VFS.WriteToFile(_vfsCacheName);
await VFS.AddRoots(roots);
await VFS.WriteToFile(_vfsCacheName);
if (Directory.Exists(lootPath))
{
@ -204,12 +204,12 @@ namespace Wabbajack.Lib
if (cancel.IsCancellationRequested) return false;
var stack = MakeStack();
UpdateTracker.NextStep("Running Compilation Stack");
var results = AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f)).ToList();
var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f));
// Add the extra files that were generated by the stack
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep($"Adding {ExtraFiles.Count} that were generated by the stack");
results = results.Concat(ExtraFiles).ToList();
results = results.Concat(ExtraFiles).ToArray();
var nomatch = results.OfType<NoMatch>();
Info($"No match for {nomatch.Count()} files");
@ -244,11 +244,11 @@ namespace Wabbajack.Lib
UpdateTracker.NextStep("Gathering Archives");
GatherArchives();
await GatherArchives();
UpdateTracker.NextStep("Including Archive Metadata");
IncludeArchiveMetadata();
UpdateTracker.NextStep("Building Patches");
BuildPatches();
await BuildPatches();
ModList = new ModList
{
@ -267,7 +267,7 @@ namespace Wabbajack.Lib
UpdateTracker.NextStep("Running Validation");
ValidateModlist.RunValidation(Queue, ModList);
await ValidateModlist.RunValidation(Queue, ModList);
UpdateTracker.NextStep("Generating Report");
GenerateReport();
@ -316,7 +316,7 @@ namespace Wabbajack.Lib
/// <summary>
/// Fills in the Patch fields in files that require them
/// </summary>
private void BuildPatches()
private async Task BuildPatches()
{
Info("Gathering patch files");
@ -335,21 +335,21 @@ namespace Wabbajack.Lib
Info($"Patching building patches from {groups.Count} archives");
var absolutePaths = AllFiles.ToDictionary(e => e.Path, e => e.AbsolutePath);
groups.PMap(Queue, group => BuildArchivePatches(group.Key, group, absolutePaths));
await groups.PMap(Queue, group => BuildArchivePatches(group.Key, group, absolutePaths));
if (InstallDirectives.OfType<PatchedFromArchive>().FirstOrDefault(f => f.PatchID == null) != null)
Error("Missing patches after generation, this should not happen");
}
private void BuildArchivePatches(string archiveSha, IEnumerable<PatchedFromArchive> group,
private async Task BuildArchivePatches(string archiveSha, IEnumerable<PatchedFromArchive> group,
Dictionary<string, string> absolutePaths)
{
using (var files = VFS.StageWith(group.Select(g => VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath))))
using (var files = await VFS.StageWith(group.Select(g => VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath))))
{
var byPath = files.GroupBy(f => string.Join("|", f.FilesInFullPath.Skip(1).Select(i => i.Name)))
.ToDictionary(f => f.Key, f => f.First());
// Now Create the patches
group.PMap(Queue, entry =>
await group.PMap(Queue, entry =>
{
Info($"Patching {entry.To}");
Status($"Patching {entry.To}");

46
Wabbajack.Lib/MO2Installer.cs
View File

@ -2,6 +2,7 @@
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Threading;
using System.Windows;
using Alphaleonis.Win32.Filesystem;
@ -36,10 +37,10 @@ namespace Wabbajack.Lib
{
}
protected override bool _Begin(CancellationToken cancel)
protected override async Task<bool> _Begin(CancellationToken cancel)
{
if (cancel.IsCancellationRequested) return false;
ConfigureProcessor(18, RecommendQueueSize());
ConfigureProcessor(18, await RecommendQueueSize());
var game = GameRegistry.Games[ModList.GameType];
if (GameFolder == null)
@ -61,7 +62,7 @@ namespace Wabbajack.Lib
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Validating Modlist");
ValidateModlist.RunValidation(Queue, ModList);
await ValidateModlist.RunValidation(Queue, ModList);
Directory.CreateDirectory(OutputFolder);
Directory.CreateDirectory(DownloadFolder);
@ -77,19 +78,19 @@ namespace Wabbajack.Lib
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Optimizing Modlist");
OptimizeModlist();
await OptimizeModlist();
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Hashing Archives");
HashArchives();
await HashArchives();
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Downloading Missing Archives");
DownloadArchives();
await DownloadArchives();
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Hashing Remaining Archives");
HashArchives();
await HashArchives();
var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
if (missing.Count > 0)
@ -112,23 +113,23 @@ namespace Wabbajack.Lib
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Installing Archives");
InstallArchives();
await InstallArchives();
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Installing Included files");
InstallIncludedFiles();
await InstallIncludedFiles();
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Installing Archive Metas");
InstallIncludedDownloadMetas();
await InstallIncludedDownloadMetas();
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Building BSAs");
BuildBSAs();
await BuildBSAs();
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Generating Merges");
zEditIntegration.GenerateMerges(this);
await zEditIntegration.GenerateMerges(this);
UpdateTracker.NextStep("Updating System-specific ini settings");
SetScreenSizeInPrefs();
@ -138,9 +139,9 @@ namespace Wabbajack.Lib
}
private void InstallIncludedDownloadMetas()
private async Task InstallIncludedDownloadMetas()
{
ModList.Directives
await ModList.Directives
.OfType<ArchiveMeta>()
.PMap(Queue, directive =>
{
@ -166,7 +167,7 @@ namespace Wabbajack.Lib
}
}
private void AskToEndorse()
private async Task AskToEndorse()
{
var mods = ModList.Archives
.Select(m => m.State)
@ -193,7 +194,7 @@ namespace Wabbajack.Lib
mods[b] = tmp;
}
mods.PMap(Queue, mod =>
await mods.PMap(Queue, mod =>
{
var er = new NexusApiClient().EndorseMod(mod);
Utils.Log($"Endorsed {mod.GameName} - {mod.ModID} - Result: {er.message}");
@ -201,19 +202,19 @@ namespace Wabbajack.Lib
Info("Done! You may now exit the application!");
}
private void BuildBSAs()
private async Task BuildBSAs()
{
var bsas = ModList.Directives.OfType<CreateBSA>().ToList();
Info($"Building {bsas.Count} bsa files");
bsas.Do(bsa =>
foreach (var bsa in bsas)
{
Status($"Building {bsa.To}");
var sourceDir = Path.Combine(OutputFolder, Consts.BSACreationDir, bsa.TempID);
using (var a = bsa.State.MakeBuilder())
{
bsa.FileStates.PMap(Queue, state =>
await bsa.FileStates.PMap(Queue, state =>
{
Status($"Adding {state.Path} to BSA");
using (var fs = File.OpenRead(Path.Combine(sourceDir, state.Path)))
@ -225,8 +226,7 @@ namespace Wabbajack.Lib
Info($"Writing {bsa.To}");
a.Build(Path.Combine(OutputFolder, bsa.To));
}
});
}
var bsaDir = Path.Combine(OutputFolder, Consts.BSACreationDir);
if (Directory.Exists(bsaDir))
@ -236,10 +236,10 @@ namespace Wabbajack.Lib
}
}
private void InstallIncludedFiles()
private async Task InstallIncludedFiles()
{
Info("Writing inline files");
ModList.Directives
await ModList.Directives
.OfType<InlineFile>()
.PMap(Queue, directive =>
{

12
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -1,4 +1,4 @@
using ReactiveUI;
using ReactiveUI;
using System;
using System.Collections.Generic;
using System.Diagnostics;
@ -355,7 +355,7 @@ namespace Wabbajack.Lib.NexusApi
set => _localCacheDir = value;
}
public void ClearUpdatedModsInCache()
public async Task ClearUpdatedModsInCache()
{
if (!UseLocalCache) return;
@ -374,7 +374,7 @@ namespace Wabbajack.Lib.NexusApi
Utils.Log($"Found {purge.Count} updated mods in the last month");
using (var queue = new WorkQueue())
{
var to_purge = Directory.EnumerateFiles(LocalCacheDir, "*.json")
var to_purge = (await Directory.EnumerateFiles(LocalCacheDir, "*.json")
.PMap(queue,f =>
{
Utils.Status("Cleaning Nexus cache for");
@ -389,16 +389,18 @@ namespace Wabbajack.Lib.NexusApi
return (should_remove, f);
}
// ToDo
// Can improve to not read the entire file to see if it starts with null
if (File.ReadAllText(f).StartsWith("null"))
return (true, f);
return (false, f);
})
}))
.Where(p => p.Item1)
.ToList();
Utils.Log($"Purging {to_purge.Count} cache entries");
to_purge.PMap(queue, f =>
await to_purge.PMap(queue, f =>
{
var uri = new Uri(Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(f.f).FromHex()));
Utils.Log($"Purging {uri}");

15
Wabbajack.Lib/Validation/ValidateModlist.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Path = Alphaleonis.Win32.Filesystem.Path;
@ -54,14 +55,14 @@ namespace Wabbajack.Lib.Validation
}
public static void RunValidation(WorkQueue queue, ModList modlist)
public static async Task RunValidation(WorkQueue queue, ModList modlist)
{
var validator = new ValidateModlist(queue);
validator.LoadListsFromGithub();
Utils.Log("Running validation checks");
var errors = validator.Validate(modlist);
var errors = await validator.Validate(modlist);
errors.Do(e => Utils.Log(e));
if (errors.Count() > 0)
{
@ -102,16 +103,16 @@ namespace Wabbajack.Lib.Validation
};
}
public IEnumerable<string> Validate(ModList modlist)
public async Task<IEnumerable<string>> Validate(ModList modlist)
{
ConcurrentStack<string> ValidationErrors = new ConcurrentStack<string>();
var nexus_mod_permissions = modlist.Archives
var nexus_mod_permissions = (await modlist.Archives
.Where(a => a.State is NexusDownloader.State)
.PMap(_queue, a => (a.Hash, FilePermissions((NexusDownloader.State)a.State), a))
.PMap(_queue, a => (a.Hash, FilePermissions((NexusDownloader.State)a.State), a)))
.ToDictionary(a => a.Hash, a => new { permissions = a.Item2, archive = a.a });
modlist.Directives
await modlist.Directives
.OfType<PatchedFromArchive>()
.PMap(_queue, p =>
{
@ -130,7 +131,7 @@ namespace Wabbajack.Lib.Validation
}
});
modlist.Directives
await modlist.Directives
.OfType<FromArchive>()
.PMap(_queue, p =>
{

25
Wabbajack.Lib/VortexCompiler.cs
View File

@ -4,6 +4,7 @@ using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using System.Threading;
using DynamicData;
using Microsoft.WindowsAPICodePack.Shell;
@ -69,8 +70,8 @@ namespace Wabbajack.Lib
ActiveArchives = new List<string>();
}
protected override bool _Begin(CancellationToken cancel)
protected override async Task<bool> _Begin(CancellationToken cancel)
{
if (cancel.IsCancellationRequested) return false;
@ -89,16 +90,16 @@ namespace Wabbajack.Lib
if (cancel.IsCancellationRequested) return false;
Info($"Indexing {StagingFolder}");
VFS.AddRoot(StagingFolder);
await VFS.AddRoot(StagingFolder);
Info($"Indexing {GamePath}");
VFS.AddRoot(GamePath);
await VFS.AddRoot(GamePath);
Info($"Indexing {DownloadsFolder}");
VFS.AddRoot(DownloadsFolder);
await VFS.AddRoot(DownloadsFolder);
if (cancel.IsCancellationRequested) return false;
AddExternalFolder();
await AddExternalFolder();
if (cancel.IsCancellationRequested) return false;
Info("Cleaning output folder");
@ -198,7 +199,7 @@ namespace Wabbajack.Lib
var stack = MakeStack();
Info("Running Compilation Stack");
var results = AllFiles.PMap(Queue, f => RunStack(stack.Where(s => s != null), f)).ToList();
var results = await AllFiles.PMap(Queue, f => RunStack(stack.Where(s => s != null), f));
IEnumerable<NoMatch> noMatch = results.OfType<NoMatch>().ToList();
Info($"No match for {noMatch.Count()} files");
@ -230,7 +231,7 @@ namespace Wabbajack.Lib
*/
if (cancel.IsCancellationRequested) return false;
GatherArchives();
await GatherArchives();
ModList = new ModList
{
@ -300,17 +301,17 @@ namespace Wabbajack.Lib
/// <summary>
/// Some have mods outside their game folder located
/// </summary>
private void AddExternalFolder()
private async Task AddExternalFolder()
{
var currentGame = GameRegistry.Games[Game];
if (currentGame.AdditionalFolders == null || currentGame.AdditionalFolders.Count == 0) return;
currentGame.AdditionalFolders.Do(f =>
foreach (var f in currentGame.AdditionalFolders)
{
var path = f.Replace("%documents%", KnownFolders.Documents.Path);
if (!Directory.Exists(path)) return;
Info($"Indexing {path}");
VFS.AddRoot(path);
});
await VFS.AddRoot(path);
}
}
private void CreateMetaFiles()

25
Wabbajack.Lib/VortexInstaller.cs
View File

@ -1,5 +1,6 @@
using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;
using System.Threading;
using System.Windows;
using Wabbajack.Common;
@ -30,7 +31,7 @@ namespace Wabbajack.Lib
GameInfo = GameRegistry.Games[ModList.GameType];
}
protected override bool _Begin(CancellationToken cancel)
protected override async Task<bool> _Begin(CancellationToken cancel)
{
if (cancel.IsCancellationRequested) return false;
MessageBox.Show(
@ -40,17 +41,17 @@ namespace Wabbajack.Lib
MessageBoxButton.OK);
if (cancel.IsCancellationRequested) return false;
ConfigureProcessor(10, RecommendQueueSize());
ConfigureProcessor(10, await RecommendQueueSize());
Directory.CreateDirectory(DownloadFolder);
if (cancel.IsCancellationRequested) return false;
HashArchives();
await HashArchives();
if (cancel.IsCancellationRequested) return false;
DownloadArchives();
await DownloadArchives();
if (cancel.IsCancellationRequested) return false;
HashArchives();
await HashArchives();
if (cancel.IsCancellationRequested) return false;
var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
@ -70,20 +71,20 @@ namespace Wabbajack.Lib
BuildFolderStructure();
if (cancel.IsCancellationRequested) return false;
InstallArchives();
await InstallArchives();
if (cancel.IsCancellationRequested) return false;
InstallIncludedFiles();
await InstallIncludedFiles();
if (cancel.IsCancellationRequested) return false;
InstallSteamWorkshopItems();
await InstallSteamWorkshopItems();
//InstallIncludedDownloadMetas();
Info("Installation complete! You may exit the program.");
return true;
}
private void InstallSteamWorkshopItems()
private async Task InstallSteamWorkshopItems()
{
//var currentLib = "";
SteamGame currentSteamGame = null;
@ -104,7 +105,7 @@ namespace Wabbajack.Lib
if (result != MessageBoxResult.Yes)
return;
ModList.Directives.OfType<SteamMeta>()
await ModList.Directives.OfType<SteamMeta>()
.PMap(Queue, item =>
{
Status("Extracting Steam meta file to temp folder");
@ -128,10 +129,10 @@ namespace Wabbajack.Lib
});
}
private void InstallIncludedFiles()
private async Task InstallIncludedFiles()
{
Info("Writing inline files");
ModList.Directives.OfType<InlineFile>()
await ModList.Directives.OfType<InlineFile>()
.PMap(Queue,directive =>
{
if (directive.To.EndsWith(".meta"))

7
Wabbajack.Lib/zEditIntegration.cs
View File

@ -9,6 +9,7 @@ using Wabbajack.Lib.CompilationSteps;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
using System.Threading.Tasks;
namespace Wabbajack.Lib
{
@ -70,7 +71,7 @@ namespace Wabbajack.Lib
m => m.First());
}
public override Directive Run(RawSourceFile source)
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!_mergesIndexed.TryGetValue(source.AbsolutePath, out var merge)) return null;
var result = source.EvolveTo<MergedPatch>();
@ -160,9 +161,9 @@ namespace Wabbajack.Lib
}
}
public static void GenerateMerges(MO2Installer installer)
public static async Task GenerateMerges(MO2Installer installer)
{
installer.ModList
await installer.ModList
.Directives
.OfType<MergedPatch>()
.PMap(installer.Queue, m =>

7
Wabbajack.Test.ListValidation/ListValidation.cs
View File

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common;
using Wabbajack.Lib;
@ -44,7 +45,7 @@ namespace Wabbajack.Test.ListValidation
[TestCategory("ListValidation")]
[DataTestMethod]
[DynamicData(nameof(GetModLists), DynamicDataSourceType.Method)]
public void ValidateModLists(string name, ModlistMetadata list)
public async Task ValidateModLists(string name, ModlistMetadata list)
{
Log($"Testing {list.Links.MachineURL} - {list.Title}");
var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ".wabbajack");
@ -67,12 +68,12 @@ namespace Wabbajack.Test.ListValidation
Log($"{installer.Archives.Count} archives to validate");
var invalids = installer.Archives
var invalids = (await installer.Archives
.PMap(Queue,archive =>
{
Log($"Validating: {archive.Name}");
return new {archive, is_valid = archive.State.Verify()};
})
}))
.Where(a => !a.is_valid)
.ToList();

21
Wabbajack.Test/ContentRightsManagementTests.cs
View File

@ -7,6 +7,7 @@ using Wabbajack.Lib;
using Wabbajack.Lib.Validation;
using Game = Wabbajack.Common.Game;
using Wabbajack.Common;
using System.Threading.Tasks;
namespace Wabbajack.Test
{
@ -127,7 +128,7 @@ namespace Wabbajack.Test
[TestMethod]
public void TestModValidation()
public async Task TestModValidation()
{
var modlist = new ModList
{
@ -160,7 +161,7 @@ namespace Wabbajack.Test
IEnumerable<string> errors;
// No errors, simple archive extraction
errors = validate.Validate(modlist);
errors = await validate.Validate(modlist);
Assert.AreEqual(errors.Count(), 0);
@ -171,7 +172,7 @@ namespace Wabbajack.Test
ArchiveHashPath = new[] {"DEADBEEF", "foo\\bar\\baz.pex"},
};
errors = validate.Validate(modlist);
errors = await validate.Validate(modlist);
Assert.AreEqual(errors.Count(), 1);
// Error due to extracted BSA file
@ -180,7 +181,7 @@ namespace Wabbajack.Test
ArchiveHashPath = new[] { "DEADBEEF", "foo.bsa", "foo\\bar\\baz.dds" },
};
errors = validate.Validate(modlist);
errors = await validate.Validate(modlist);
Assert.AreEqual(errors.Count(), 1);
// No error since we're just installing the .bsa, not extracting it
@ -189,7 +190,7 @@ namespace Wabbajack.Test
ArchiveHashPath = new[] { "DEADBEEF", "foo.bsa"},
};
errors = validate.Validate(modlist);
errors = await validate.Validate(modlist);
Assert.AreEqual(0, errors.Count());
// Error due to game conversion
@ -198,7 +199,7 @@ namespace Wabbajack.Test
{
ArchiveHashPath = new[] { "DEADBEEF", "foo\\bar\\baz.dds" },
};
errors = validate.Validate(modlist);
errors = await validate.Validate(modlist);
Assert.AreEqual(errors.Count(), 1);
// Error due to file downloaded from 3rd party
@ -208,7 +209,7 @@ namespace Wabbajack.Test
State = new HTTPDownloader.State() { Url = "https://somebadplace.com" },
Hash = "DEADBEEF"
};
errors = validate.Validate(modlist);
errors = await validate.Validate(modlist);
Assert.AreEqual(1, errors.Count());
// Ok due to file downloaded from whitelisted 3rd party
@ -218,7 +219,7 @@ namespace Wabbajack.Test
State = new HTTPDownloader.State { Url = "https://somegoodplace.com/baz.7z" },
Hash = "DEADBEEF"
};
errors = validate.Validate(modlist);
errors = await validate.Validate(modlist);
Assert.AreEqual(0, errors.Count());
@ -229,7 +230,7 @@ namespace Wabbajack.Test
State = new GoogleDriveDownloader.State { Id = "bleg"},
Hash = "DEADBEEF"
};
errors = validate.Validate(modlist);
errors = await validate.Validate(modlist);
Assert.AreEqual(errors.Count(), 1);
// Ok due to file downloaded from good google site
@ -239,7 +240,7 @@ namespace Wabbajack.Test
State = new GoogleDriveDownloader.State { Id = "googleDEADBEEF" },
Hash = "DEADBEEF"
};
errors = validate.Validate(modlist);
errors = await validate.Validate(modlist);
Assert.AreEqual(0, errors.Count());
}

44
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -31,10 +31,10 @@ namespace Wabbajack.VirtualFileSystem.Test
}
[TestMethod]
public void FilesAreIndexed()
public async Task FilesAreIndexed()
{
AddFile("test.txt", "This is a test");
AddTestRoot();
await AddTestRoot();
var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
Assert.IsNotNull(file);
@ -43,10 +43,10 @@ namespace Wabbajack.VirtualFileSystem.Test
Assert.AreEqual(file.Hash, "qX0GZvIaTKM=");
}
private void AddTestRoot()
private async Task AddTestRoot()
{
context.AddRoot(VFS_TEST_DIR_FULL);
context.WriteToFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
await context.AddRoot(VFS_TEST_DIR_FULL);
await context.WriteToFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
context.IntegrateFromFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
}
@ -56,7 +56,7 @@ namespace Wabbajack.VirtualFileSystem.Test
{
AddFile("archive/test.txt", "This is a test");
ZipUpFolder("archive", "test.zip");
AddTestRoot();
await AddTestRoot();
var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip");
var file = context.Index.ByFullPath[abs_path];
@ -79,7 +79,7 @@ namespace Wabbajack.VirtualFileSystem.Test
ZipUpFolder("archive", "test.zip");
AddFile("test.txt", "This is a test");
AddTestRoot();
await AddTestRoot();
var files = context.Index.ByHash["qX0GZvIaTKM="];
@ -90,7 +90,7 @@ namespace Wabbajack.VirtualFileSystem.Test
public async Task DeletedFilesAreRemoved()
{
AddFile("test.txt", "This is a test");
AddTestRoot();
await AddTestRoot();
var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
Assert.IsNotNull(file);
@ -100,21 +100,21 @@ namespace Wabbajack.VirtualFileSystem.Test
File.Delete(Path.Combine(VFS_TEST_DIR_FULL, "test.txt"));
AddTestRoot();
await AddTestRoot();
CollectionAssert.DoesNotContain(context.Index.ByFullPath, Path.Combine(VFS_TEST_DIR_FULL, "test.txt"));
}
[TestMethod]
public void UnmodifiedFilesAreNotReIndexed()
public async Task UnmodifiedFilesAreNotReIndexed()
{
AddFile("test.txt", "This is a test");
AddTestRoot();
await AddTestRoot();
var old_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
var old_time = old_file.LastAnalyzed;
AddTestRoot();
await AddTestRoot();
var new_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
@ -122,23 +122,23 @@ namespace Wabbajack.VirtualFileSystem.Test
}
[TestMethod]
public void CanStageSimpleArchives()
public async Task CanStageSimpleArchives()
{
AddFile("archive/test.txt", "This is a test");
ZipUpFolder("archive", "test.zip");
AddTestRoot();
await AddTestRoot();
var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip");
var file = context.Index.ByFullPath[abs_path + "|test.txt"];
var cleanup = context.Stage(new List<VirtualFile> {file});
var cleanup = await context.Stage(new List<VirtualFile> {file});
Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
cleanup();
}
[TestMethod]
public void CanStageNestedArchives()
public async Task CanStageNestedArchives()
{
AddFile("archive/test.txt", "This is a test");
ZipUpFolder("archive", "test.zip");
@ -148,11 +148,11 @@ namespace Wabbajack.VirtualFileSystem.Test
Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip"));
ZipUpFolder("archive", "test.zip");
AddTestRoot();
await AddTestRoot();
var files = context.Index.ByHash["qX0GZvIaTKM="];
var cleanup = context.Stage(files);
var cleanup = await context.Stage(files);
foreach (var file in files)
Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
@ -161,7 +161,7 @@ namespace Wabbajack.VirtualFileSystem.Test
}
[TestMethod]
public void CanRequestPortableFileTrees()
public async Task CanRequestPortableFileTrees()
{
AddFile("archive/test.txt", "This is a test");
ZipUpFolder("archive", "test.zip");
@ -171,7 +171,7 @@ namespace Wabbajack.VirtualFileSystem.Test
Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip"));
ZipUpFolder("archive", "test.zip");
AddTestRoot();
await AddTestRoot();
var files = context.Index.ByHash["qX0GZvIaTKM="];
var archive = context.Index.ByRootPath[Path.Combine(VFS_TEST_DIR_FULL, "test.zip")];
@ -180,12 +180,12 @@ namespace Wabbajack.VirtualFileSystem.Test
var new_context = new Context(Queue);
new_context.IntegrateFromPortable(state,
await new_context.IntegrateFromPortable(state,
new Dictionary<string, string> {{archive.Hash, archive.FullPath}});
var new_files = new_context.Index.ByHash["qX0GZvIaTKM="];
var close = new_context.Stage(new_files);
var close = await new_context.Stage(new_files);
foreach (var file in new_files)
Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));

34
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -46,7 +46,7 @@ namespace Wabbajack.VirtualFileSystem
return new TemporaryDirectory(Path.Combine(_stagingFolder, Guid.NewGuid().ToString()));
}
public IndexRoot AddRoot(string root)
public async Task<IndexRoot> AddRoot(string root)
{
if (!Path.IsPathRooted(root))
throw new InvalidDataException($"Path is not absolute: {root}");
@ -59,8 +59,8 @@ namespace Wabbajack.VirtualFileSystem
var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing {root}", filesToIndex.Count));
var allFiles= filesToIndex
.PMap(Queue, f =>
var allFiles = await filesToIndex
.PMap(Queue, async f =>
{
if (byPath.TryGetValue(f, out var found))
{
@ -69,7 +69,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return VirtualFile.Analyze(this, null, f, f);
return await VirtualFile.Analyze(this, null, f, f);
});
var newIndex = IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@ -82,7 +82,7 @@ namespace Wabbajack.VirtualFileSystem
return newIndex;
}
public IndexRoot AddRoots(List<string> roots)
public async Task<IndexRoot> AddRoots(List<string> roots)
{
if (!roots.All(p => Path.IsPathRooted(p)))
throw new InvalidDataException($"Paths are not absolute");
@ -95,8 +95,8 @@ namespace Wabbajack.VirtualFileSystem
var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing roots", filesToIndex.Count));
var allFiles = filesToIndex
.PMap(Queue, f =>
var allFiles = await filesToIndex
.PMap(Queue, async f =>
{
if (byPath.TryGetValue(f, out var found))
{
@ -105,7 +105,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return VirtualFile.Analyze(this, null, f, f);
return await VirtualFile.Analyze(this, null, f, f);
});
var newIndex = IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@ -137,7 +137,7 @@ namespace Wabbajack.VirtualFileSystem
});
}
public void WriteToFile(string filename)
public async Task WriteToFile(string filename)
{
using (var fs = File.OpenWrite(filename))
using (var bw = new BinaryWriter(fs, Encoding.UTF8, true))
@ -148,13 +148,13 @@ namespace Wabbajack.VirtualFileSystem
bw.Write(FileVersion);
bw.Write((ulong) Index.AllFiles.Count);
Index.AllFiles
(await Index.AllFiles
.PMap(Queue, f =>
{
var ms = new MemoryStream();
f.Write(ms);
return ms;
})
}))
.Do(ms =>
{
var size = ms.Position;
@ -202,7 +202,7 @@ namespace Wabbajack.VirtualFileSystem
}
}
public Action Stage(IEnumerable<VirtualFile> files)
public async Task<Action> Stage(IEnumerable<VirtualFile> files)
{
var grouped = files.SelectMany(f => f.FilesInFullPath)
.Distinct()
@ -216,7 +216,7 @@ namespace Wabbajack.VirtualFileSystem
foreach (var group in grouped)
{
var tmpPath = Path.Combine(_stagingFolder, Guid.NewGuid().ToString());
FileExtractor.ExtractAll(Queue, group.Key.StagedPath, tmpPath);
await FileExtractor.ExtractAll(Queue, group.Key.StagedPath, tmpPath);
paths.Add(tmpPath);
foreach (var file in group)
file.StagedPath = Path.Combine(tmpPath, file.Name);
@ -245,11 +245,11 @@ namespace Wabbajack.VirtualFileSystem
}).ToList();
}
public void IntegrateFromPortable(List<PortableFile> state, Dictionary<string, string> links)
public async Task IntegrateFromPortable(List<PortableFile> state, Dictionary<string, string> links)
{
var indexedState = state.GroupBy(f => f.ParentHash)
.ToDictionary(f => f.Key ?? "", f => (IEnumerable<PortableFile>) f);
var parents = indexedState[""]
var parents = await indexedState[""]
.PMap(Queue,f => VirtualFile.CreateFromPortable(this, indexedState, links, f));
var newIndex = Index.Integrate(parents);
@ -259,9 +259,9 @@ namespace Wabbajack.VirtualFileSystem
}
}
public DisposableList<VirtualFile> StageWith(IEnumerable<VirtualFile> files)
public async Task<DisposableList<VirtualFile>> StageWith(IEnumerable<VirtualFile> files)
{
return new DisposableList<VirtualFile>(Stage(files), files);
return new DisposableList<VirtualFile>(await Stage(files), files);
}

12
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -4,6 +4,7 @@ using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.CSP;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
@ -128,7 +129,7 @@ namespace Wabbajack.VirtualFileSystem
}
}
public static VirtualFile Analyze(Context context, VirtualFile parent, string abs_path,
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, string abs_path,
string rel_path)
{
var fi = new FileInfo(abs_path);
@ -148,11 +149,12 @@ namespace Wabbajack.VirtualFileSystem
using (var tempFolder = context.GetTemporaryFolder())
{
FileExtractor.ExtractAll(context.Queue, abs_path, tempFolder.FullName);
await FileExtractor.ExtractAll(context.Queue, abs_path, tempFolder.FullName);
self.Children = Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories)
.PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName)))
.ToImmutableList();
var list = await Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories)
.PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName)));
self.Children = list.ToImmutableList();
}
}

2
Wabbajack/View Models/ModListMetadataVM.cs
View File

@ -81,7 +81,7 @@ namespace Wabbajack
var sub = queue.Status.Select(i => i.ProgressPercent)
.Subscribe(percent => ProgressPercent = percent);
TaskCompletionSource<bool> tcs = new TaskCompletionSource<bool>();
queue.QueueTask(() =>
queue.QueueTask(async () =>
{
var downloader = DownloadDispatcher.ResolveArchive(Metadata.Links.Download);
downloader.Download(new Archive{ Name = Metadata.Title, Size = Metadata.DownloadMetadata?.Size ?? 0}, Location);