Temp folder fixes, fix for NOMATCH_INCLUDE

Wabbajack.Common.Test/WorkQueueTests.cs

@@ -1,15 +1,71 @@
 using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
 using System.Linq;
 using System.Reactive.Linq;
 using System.Reactive.Subjects;
 using System.Threading;
 using System.Threading.Tasks;
+using Splat;
+using Wabbajack;
+using Wabbajack.Common;
 using Xunit;
+using Xunit.Abstractions;
 
 namespace Wabbajack.Common.Test
 {
-    public class WorkQueueTests
+    public class WorkQueueTests : IAsyncLifetime
     {
+        private ITestOutputHelper _output;
+        private IDisposable _sub;
+
+        #region OrderTests
+
+        [Fact]
+        public async Task WorkerQueuesRunDepthFirst()
+        {
+            ConcurrentQueue<int> list = new ConcurrentQueue<int>();
+            
+            using var queue = new WorkQueue(1);
+
+            await Enumerable.Range(1, 2).PMap(queue, async i =>
+            {
+                Utils.Log(i.ToString());
+                list.Enqueue(i);
+                await Enumerable.Range(i * 10, 2).PMap(queue, i2 =>
+                {
+                    Utils.Log(i2.ToString());
+                    list.Enqueue(i2);
+                });
+            });
+
+            Assert.Equal(6,list.Count);
+            Assert.Equal(new List<int> {2, 21, 20, 1, 11, 10}, list.ToArray());
+        }
+
+        [Fact]
+        public async Task TasksRunOnce()
+        {
+            ConcurrentQueue<int> list = new ConcurrentQueue<int>();
+            
+            using var queue = new WorkQueue(1);
+
+            await Enumerable.Range(1, 2).PMap(queue, async i =>
+            {
+                Utils.Log($"A {i}");
+                list.Enqueue(i);
+                await Enumerable.Range(i * 10, 2).PMap(queue, i2 =>
+                {
+                    Utils.Log($"B {i2}");
+                    list.Enqueue(i2);
+                });
+            });
+            
+            Assert.Equal(list.ToArray(), list.Distinct().ToArray());
+        }
+
+        #endregion
+        
         #region DynamicNumThreads
         const int Large = 8;
         const int Medium = 6;
@@ -186,5 +242,20 @@ namespace Wabbajack.Common.Test
             Assert.Equal(completed, task);
         }
         #endregion
+
+
+        public WorkQueueTests(ITestOutputHelper output)
+        {
+            _output = output;
+        }
+        public async Task InitializeAsync()
+        {
+            _sub = Utils.LogMessages.Subscribe(msg => _output.WriteLine(msg.ToString()));
+        }
+
+        public async Task DisposeAsync()
+        {
+            _sub.Dispose();
+        }
     }
 }

Wabbajack.Common/AsyncBlockingCollection.cs

@@ -9,12 +9,14 @@ namespace Wabbajack.Common
 {
     public class AsyncBlockingCollection<T> : IDisposable
     {
-        private readonly ConcurrentQueue<T> _collection = new ConcurrentQueue<T>();
+        private readonly ConcurrentStack<T> _collection = new ConcurrentStack<T>();
         private bool isDisposed = false;
+
+        public int Count => _collection.Count;
         
         public void Add(T val)
         {
-            _collection.Enqueue(val);
+            _collection.Push(val);
         }
 
         public async ValueTask<(bool found, T val)> TryTake(TimeSpan timeout, CancellationToken token)
@@ -22,7 +24,7 @@ namespace Wabbajack.Common
             var startTime = DateTime.Now;
             while (true)
             {
-                if (_collection.TryDequeue(out T result))
+                if (_collection.TryPop(out T result))
                 {
                     return (true, result);
                 }

Wabbajack.Common/Util/TempFolder.cs

@@ -11,9 +11,22 @@ namespace Wabbajack.Common
     {
         public AbsolutePath Dir { get; }
         public bool DeleteAfter = true;
+        private static Task _cleanTask;
+
+        static TempFolder()
+        {
+            _cleanTask = "tmp_files".RelativeTo(AbsolutePath.EntryPoint).DeleteDirectory();
+        }
+
+        public static async Task EnsureInited()
+        {
+            Utils.Log("Cleaning temp files");
+            await _cleanTask;
+        }
 
         public TempFolder(bool deleteAfter = true)
         {
+            _cleanTask.Wait();
             Dir = Path.Combine("tmp_files", Guid.NewGuid().ToString()).RelativeTo(AbsolutePath.EntryPoint);
             if (!Dir.Exists) 
                 Dir.CreateDirectory();
@@ -31,6 +44,7 @@ namespace Wabbajack.Common
         }
         public async ValueTask DisposeAsync()
         {
+            Utils.Log($"Deleting {Dir}");
             if (DeleteAfter && Dir.Exists)
             {
                 await Utils.DeleteDirectory(Dir);

Wabbajack.Common/Utils.cs

@@ -527,13 +527,17 @@ namespace Wabbajack.Common
             // To avoid thread starvation, we'll start to help out in the work queue
             if (WorkQueue.WorkerThread)
             {
-                while (remainingTasks > 0)
+                while (true)
                 {
                     var (got, a) = await queue.Queue.TryTake(TimeSpan.FromMilliseconds(100), CancellationToken.None);
                     if (got)
                     {
                         await a();
                     }
+                    else
+                    {
+                        break;
+                    }
                 }
             }
 
@@ -605,7 +609,7 @@ namespace Wabbajack.Common
                 });
                 return tc.Task;
             }).ToList();
-
+            
             // To avoid thread starvation, we'll start to help out in the work queue
             if (WorkQueue.WorkerThread)
             {
@@ -999,7 +1003,7 @@ namespace Wabbajack.Common
                     Status($"Deleting: {p.Line}");
                 });
 
-            await process.Start();
+            var exitCode = await process.Start();
             await result;
         }
 

Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs

@@ -9,7 +9,7 @@ namespace Wabbajack.Lib.CompilationSteps
 {
     public class IncludeTaggedMods : ACompilationStep
     {
-        private readonly IEnumerable<string> _includeDirectly;
+        private readonly IEnumerable<AbsolutePath> _includeDirectly;
         private readonly string _tag;
         private readonly MO2Compiler _mo2Compiler;
 
@@ -23,7 +23,7 @@ namespace Wabbajack.Lib.CompilationSteps
                 if (general.notes != null && general.notes.Contains(_tag))
                     return true;
                 return general.comments != null && general.comments.Contains(_tag);
-            }).Select(kv => $"mods\\{kv.Key}\\");
+            }).Select(kv => kv.Key);
         }
 
         public override async ValueTask<Directive?> Run(RawSourceFile source)
@@ -31,7 +31,7 @@ namespace Wabbajack.Lib.CompilationSteps
             if (!source.Path.StartsWith(Consts.MO2ModFolderName)) return null;
             foreach (var modpath in _includeDirectly)
             {
-                if (!source.Path.StartsWith(modpath)) continue;
+                if (!source.AbsolutePath.InFolder(modpath)) continue;
                 var result = source.EvolveTo<InlineFile>();
                 result.SourceDataID = await _compiler.IncludeFile(source.AbsolutePath);
                 return result;

Wabbajack.Test/SanityTests.cs

@@ -260,6 +260,25 @@ namespace Wabbajack.Test
             Assert.NotNull(directive);
             Assert.IsAssignableFrom<PatchedFromArchive>(directive);
         }
+        
+        [Fact]
+        public async Task NoMatchIncludeIncludesNonMatchingFiles() 
+        {
+
+            var profile = utils.AddProfile();
+            var mod = utils.AddMod();
+            var testPex = utils.AddModFile(mod, @"Data\scripts\test.pex", 10);
+
+            await utils.Configure();
+
+            await utils.AddModFile(mod, "meta.ini").WriteAllLinesAsync(new[]
+            {
+                "[General]", "notes= fsdaf WABBAJACK_NOMATCH_INCLUDE fadsfsad",
+            });
+            await CompileAndInstall(profile);
+
+            utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex");
+        }
 
     }
 }

Wabbajack/App.xaml.cs

@@ -19,6 +19,7 @@ namespace Wabbajack
     {
         public App()
         {
+            TempFolder.EnsureInited();
             RenderOptions.ProcessRenderMode = RenderMode.SoftwareOnly;
             CLIOld.ParseOptions(Environment.GetCommandLineArgs());
             if (CLIArguments.Help)