diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 63a99525..5c508b62 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Data.HashFunction.xxHash; using System.Diagnostics; @@ -611,6 +611,44 @@ namespace Wabbajack.Common return await Task.WhenAll(tasks); } + public static async Task PMap(this IEnumerable coll, WorkQueue queue, + Func f) + { + var colllst = coll.ToList(); + + var remainingTasks = colllst.Count; + + var tasks = colllst.Select(i => + { + var tc = new TaskCompletionSource(); + queue.QueueTask(async () => + { + try + { + await f(i); + tc.SetResult(true); + } + catch (Exception ex) + { + tc.SetException(ex); + } + Interlocked.Decrement(ref remainingTasks); + }); + return tc.Task; + }).ToList(); + + // To avoid thread starvation, we'll start to help out in the work queue + if (WorkQueue.WorkerThread) + while (remainingTasks > 0) + if (queue.Queue.TryTake(out var a, 500)) + { + WorkQueue.AsyncLocalCurrentQueue.Value = WorkQueue.ThreadLocalCurrentQueue.Value; + await a(); + } + + await Task.WhenAll(tasks); + } + public static async Task PMap(this IEnumerable coll, WorkQueue queue, Action f) { await coll.PMap(queue, i => diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index f93985eb..be8b7da7 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -117,7 +117,7 @@ namespace Wabbajack.Lib if (Directory.Exists(lootPath)) { - lootFiles = Directory.EnumerateFiles(lootPath, "userlist.yaml", SearchOption.AllDirectories) + lootFiles = Directory.EnumerateFiles(lootPath, "userlist.yaml", SearchOption.AllDirectories) .Where(p => p.FileExists()) .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p]) { Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath)) }); @@ -415,8 +415,9 @@ namespace Wabbajack.Lib var absolutePaths = AllFiles.ToDictionary(e => e.Path, e => e.AbsolutePath); await groups.PMap(Queue, group => BuildArchivePatches(group.Key, group, absolutePaths)); - if (InstallDirectives.OfType().FirstOrDefault(f => f.PatchID == null) != null) - Error("Missing patches after generation, this should not happen"); + var firstFailedPatch = InstallDirectives.OfType().FirstOrDefault(f => f.PatchID == null); + if (firstFailedPatch != null) + Error($"Missing patches after generation, this should not happen. First failure: {firstFailedPatch.FullPath}"); } private async Task BuildArchivePatches(string archiveSha, IEnumerable group,