diff --git a/Wabbajack.Common.Test/WorkQueueTests.cs b/Wabbajack.Common.Test/WorkQueueTests.cs index ec1761a5..ee5be9ef 100644 --- a/Wabbajack.Common.Test/WorkQueueTests.cs +++ b/Wabbajack.Common.Test/WorkQueueTests.cs @@ -1,15 +1,71 @@ using System; +using System.Collections.Concurrent; +using System.Collections.Generic; using System.Linq; using System.Reactive.Linq; using System.Reactive.Subjects; using System.Threading; using System.Threading.Tasks; +using Splat; +using Wabbajack; +using Wabbajack.Common; using Xunit; +using Xunit.Abstractions; namespace Wabbajack.Common.Test { - public class WorkQueueTests + public class WorkQueueTests : IAsyncLifetime { + private ITestOutputHelper _output; + private IDisposable _sub; + + #region OrderTests + + [Fact] + public async Task WorkerQueuesRunDepthFirst() + { + ConcurrentQueue list = new ConcurrentQueue(); + + using var queue = new WorkQueue(1); + + await Enumerable.Range(1, 2).PMap(queue, async i => + { + Utils.Log(i.ToString()); + list.Enqueue(i); + await Enumerable.Range(i * 10, 2).PMap(queue, i2 => + { + Utils.Log(i2.ToString()); + list.Enqueue(i2); + }); + }); + + Assert.Equal(6,list.Count); + Assert.Equal(new List {2, 21, 20, 1, 11, 10}, list.ToArray()); + } + + [Fact] + public async Task TasksRunOnce() + { + ConcurrentQueue list = new ConcurrentQueue(); + + using var queue = new WorkQueue(1); + + await Enumerable.Range(1, 2).PMap(queue, async i => + { + Utils.Log($"A {i}"); + list.Enqueue(i); + await Enumerable.Range(i * 10, 2).PMap(queue, i2 => + { + Utils.Log($"B {i2}"); + list.Enqueue(i2); + }); + }); + + Assert.Equal(list.ToArray(), list.Distinct().ToArray()); + } + + #endregion + #region DynamicNumThreads const int Large = 8; const int Medium = 6; @@ -186,5 +242,20 @@ namespace Wabbajack.Common.Test Assert.Equal(completed, task); } #endregion + + + public WorkQueueTests(ITestOutputHelper output) + { + _output = output; + } + public async Task InitializeAsync() + { + _sub = Utils.LogMessages.Subscribe(msg => _output.WriteLine(msg.ToString())); + } + + public async Task DisposeAsync() + { + _sub.Dispose(); + } } } diff --git a/Wabbajack.Common/AsyncBlockingCollection.cs b/Wabbajack.Common/AsyncBlockingCollection.cs index 15e53625..27225081 100644 --- a/Wabbajack.Common/AsyncBlockingCollection.cs +++ b/Wabbajack.Common/AsyncBlockingCollection.cs @@ -9,12 +9,14 @@ namespace Wabbajack.Common { public class AsyncBlockingCollection : IDisposable { - private readonly ConcurrentQueue _collection = new ConcurrentQueue(); + private readonly ConcurrentStack _collection = new ConcurrentStack(); private bool isDisposed = false; + + public int Count => _collection.Count; public void Add(T val) { - _collection.Enqueue(val); + _collection.Push(val); } public async ValueTask<(bool found, T val)> TryTake(TimeSpan timeout, CancellationToken token) @@ -22,7 +24,7 @@ namespace Wabbajack.Common var startTime = DateTime.Now; while (true) { - if (_collection.TryDequeue(out T result)) + if (_collection.TryPop(out T result)) { return (true, result); } diff --git a/Wabbajack.Common/Util/TempFolder.cs b/Wabbajack.Common/Util/TempFolder.cs index 9cd37842..0d66f842 100644 --- a/Wabbajack.Common/Util/TempFolder.cs +++ b/Wabbajack.Common/Util/TempFolder.cs @@ -11,9 +11,22 @@ namespace Wabbajack.Common { public AbsolutePath Dir { get; } public bool DeleteAfter = true; + private static Task _cleanTask; + + static TempFolder() + { + _cleanTask = "tmp_files".RelativeTo(AbsolutePath.EntryPoint).DeleteDirectory(); + } + + public static async Task EnsureInited() + { + Utils.Log("Cleaning temp files"); + await _cleanTask; + } public TempFolder(bool deleteAfter = true) { + _cleanTask.Wait(); Dir = Path.Combine("tmp_files", Guid.NewGuid().ToString()).RelativeTo(AbsolutePath.EntryPoint); if (!Dir.Exists) Dir.CreateDirectory(); @@ -31,6 +44,7 @@ namespace Wabbajack.Common } public async ValueTask DisposeAsync() { + Utils.Log($"Deleting {Dir}"); if (DeleteAfter && Dir.Exists) { await Utils.DeleteDirectory(Dir); diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index c88e95fc..160b4875 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -527,13 +527,17 @@ namespace Wabbajack.Common // To avoid thread starvation, we'll start to help out in the work queue if (WorkQueue.WorkerThread) { - while (remainingTasks > 0) + while (true) { var (got, a) = await queue.Queue.TryTake(TimeSpan.FromMilliseconds(100), CancellationToken.None); if (got) { await a(); } + else + { + break; + } } } @@ -605,7 +609,7 @@ namespace Wabbajack.Common }); return tc.Task; }).ToList(); - + // To avoid thread starvation, we'll start to help out in the work queue if (WorkQueue.WorkerThread) { @@ -999,7 +1003,7 @@ namespace Wabbajack.Common Status($"Deleting: {p.Line}"); }); - await process.Start(); + var exitCode = await process.Start(); await result; } diff --git a/Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs b/Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs index d2d8a06f..7635b477 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs @@ -9,7 +9,7 @@ namespace Wabbajack.Lib.CompilationSteps { public class IncludeTaggedMods : ACompilationStep { - private readonly IEnumerable _includeDirectly; + private readonly IEnumerable _includeDirectly; private readonly string _tag; private readonly MO2Compiler _mo2Compiler; @@ -23,7 +23,7 @@ namespace Wabbajack.Lib.CompilationSteps if (general.notes != null && general.notes.Contains(_tag)) return true; return general.comments != null && general.comments.Contains(_tag); - }).Select(kv => $"mods\\{kv.Key}\\"); + }).Select(kv => kv.Key); } public override async ValueTask Run(RawSourceFile source) @@ -31,7 +31,7 @@ namespace Wabbajack.Lib.CompilationSteps if (!source.Path.StartsWith(Consts.MO2ModFolderName)) return null; foreach (var modpath in _includeDirectly) { - if (!source.Path.StartsWith(modpath)) continue; + if (!source.AbsolutePath.InFolder(modpath)) continue; var result = source.EvolveTo(); result.SourceDataID = await _compiler.IncludeFile(source.AbsolutePath); return result; diff --git a/Wabbajack.Test/SanityTests.cs b/Wabbajack.Test/SanityTests.cs index 7baeeda7..fe503611 100644 --- a/Wabbajack.Test/SanityTests.cs +++ b/Wabbajack.Test/SanityTests.cs @@ -260,6 +260,25 @@ namespace Wabbajack.Test Assert.NotNull(directive); Assert.IsAssignableFrom(directive); } + + [Fact] + public async Task NoMatchIncludeIncludesNonMatchingFiles() + { + + var profile = utils.AddProfile(); + var mod = utils.AddMod(); + var testPex = utils.AddModFile(mod, @"Data\scripts\test.pex", 10); + + await utils.Configure(); + + await utils.AddModFile(mod, "meta.ini").WriteAllLinesAsync(new[] + { + "[General]", "notes= fsdaf WABBAJACK_NOMATCH_INCLUDE fadsfsad", + }); + await CompileAndInstall(profile); + + utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex"); + } } } diff --git a/Wabbajack/App.xaml.cs b/Wabbajack/App.xaml.cs index 3c40d560..d0b62a49 100644 --- a/Wabbajack/App.xaml.cs +++ b/Wabbajack/App.xaml.cs @@ -19,6 +19,7 @@ namespace Wabbajack { public App() { + TempFolder.EnsureInited(); RenderOptions.ProcessRenderMode = RenderMode.SoftwareOnly; CLIOld.ParseOptions(Environment.GetCommandLineArgs()); if (CLIArguments.Help)