Merge pull request #305 from Noggog/patch-freeze-investigation

Patch freeze investigation

Wabbajack.Common/FileExtractor.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
@@ -208,6 +208,7 @@ namespace Wabbajack.Common
 
             if (p.ExitCode == 0)
             {
+                Utils.Status($"Extracting {name} - 100%", 100, alsoLog: true);
                 return;
             }
             Utils.Log(new _7zipReturnError(p.ExitCode, source, dest, p.StandardOutput.ReadToEnd()));

Wabbajack.Common/Utils.cs

@@ -597,12 +597,15 @@ namespace Wabbajack.Common
 
             // To avoid thread starvation, we'll start to help out in the work queue
             if (WorkQueue.WorkerThread)
+            {
                 while (remainingTasks > 0)
+                {
                     if (queue.Queue.TryTake(out var a, 500))
                     {
-                        WorkQueue.AsyncLocalCurrentQueue.Value = WorkQueue.ThreadLocalCurrentQueue.Value;
                         await a();
                     }
+                }
+            }
 
             return await Task.WhenAll(tasks);
         }
@@ -634,12 +637,15 @@ namespace Wabbajack.Common
 
             // To avoid thread starvation, we'll start to help out in the work queue
             if (WorkQueue.WorkerThread)
+            {
                 while (remainingTasks > 0)
+                {
                     if (queue.Queue.TryTake(out var a, 500))
                     {
-                        WorkQueue.AsyncLocalCurrentQueue.Value = WorkQueue.ThreadLocalCurrentQueue.Value;
                         await a();
                     }
+                }
+            }
 
             return await Task.WhenAll(tasks);
         }
@@ -672,12 +678,15 @@ namespace Wabbajack.Common
 
             // To avoid thread starvation, we'll start to help out in the work queue
             if (WorkQueue.WorkerThread)
+            {
                 while (remainingTasks > 0)
+                {
                     if (queue.Queue.TryTake(out var a, 500))
                     {
-                        WorkQueue.AsyncLocalCurrentQueue.Value = WorkQueue.ThreadLocalCurrentQueue.Value;
                         await a();
                     }
+                }
+            }
 
             await Task.WhenAll(tasks);
         }

Wabbajack.Common/WorkQueue.cs

@@ -20,8 +20,8 @@ namespace Wabbajack.Common
         private static readonly AsyncLocal<int> _cpuId = new AsyncLocal<int>();
         public int CpuId => _cpuId.Value;
 
-        internal static bool WorkerThread => ThreadLocalCurrentQueue.Value != null;
+        public static bool WorkerThread => AsyncLocalCurrentQueue.Value != null;
-        internal static readonly ThreadLocal<WorkQueue> ThreadLocalCurrentQueue = new ThreadLocal<WorkQueue>();
+        public bool IsWorkerThread => WorkerThread;
         internal static readonly AsyncLocal<WorkQueue> AsyncLocalCurrentQueue = new AsyncLocal<WorkQueue>();
 
         private readonly Subject<CPUStatus> _Status = new Subject<CPUStatus>();
@@ -61,7 +61,6 @@ namespace Wabbajack.Common
         private async Task ThreadBody(int idx)
         {
             _cpuId.Value = idx;
-            ThreadLocalCurrentQueue.Value = this;
             AsyncLocalCurrentQueue.Value = this;
 
             try

Wabbajack.Lib/ACompiler.cs

@@ -221,13 +221,15 @@ namespace Wabbajack.Lib
 
         public async Task<Archive> ResolveArchive(IndexedArchive archive)
         {
+            Utils.Status($"Checking link for {archive.Name}", alsoLog: true);
+
             if (archive.IniData == null)
                 Error(
                     $"No download metadata found for {archive.Name}, please use MO2 to query info or add a .meta file and try again.");
 
             var result = new Archive
             {
-                State = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(archive.IniData)
+                State = await DownloadDispatcher.ResolveArchive(archive.IniData)
             };
 
             if (result.State == null)
@@ -238,8 +240,6 @@ namespace Wabbajack.Lib
             result.Meta = archive.Meta;
             result.Size = archive.File.Size;
 
-            Info($"Checking link for {archive.Name}");
-
             if (result.State != null && !await result.State.Verify())
                 Error(
                     $"Unable to resolve link for {archive.Name}. If this is hosted on the Nexus the file may have been removed.");

Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs

@@ -65,10 +65,7 @@ namespace Wabbajack.Lib.CompilationSteps
 
             var id = Guid.NewGuid().ToString();
 
-            var matches = await sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e)
+            var matches = await sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e, Path.Combine(Consts.BSACreationDir, id, e.Name))));
-            {
-                Path = Path.Combine(Consts.BSACreationDir, id, e.Name)
-            }));
 
 
             foreach (var match in matches)

Wabbajack.Lib/Data.cs

@@ -11,11 +11,14 @@ namespace Wabbajack.Lib
 {
     public class RawSourceFile
     {
+        // ToDo
+        // Make readonly
         public string Path;
 
-        public RawSourceFile(VirtualFile file)
+        public RawSourceFile(VirtualFile file, string path)
         {
             File = file;
+            Path = path;
         }
 
         public string AbsolutePath => File.StagedPath;

Wabbajack.Lib/MO2Compiler.cs

@@ -119,8 +119,7 @@ namespace Wabbajack.Lib
             {
                 lootFiles = Directory.EnumerateFiles(lootPath, "userlist.yaml", SearchOption.AllDirectories)
                     .Where(p => p.FileExists())
-                    .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
+                    .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath))));
-                        { Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath)) });
             }
 
             if (cancel.IsCancellationRequested) return false;
@@ -158,12 +157,11 @@ namespace Wabbajack.Lib
 
             var mo2Files = Directory.EnumerateFiles(MO2Folder, "*", SearchOption.AllDirectories)
                 .Where(p => p.FileExists())
-                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p]) { Path = p.RelativeTo(MO2Folder) });
+                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(MO2Folder)));
 
             var gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
                 .Where(p => p.FileExists())
-                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
+                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath))));
-                { Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) });
 
             
             ModMetas = Directory.EnumerateDirectories(Path.Combine(MO2Folder, "mods"))
@@ -229,12 +227,12 @@ namespace Wabbajack.Lib
             UpdateTracker.NextStep($"Adding {ExtraFiles.Count} that were generated by the stack");
             results = results.Concat(ExtraFiles).ToArray();
 
-            var nomatch = results.OfType<NoMatch>();
+            var nomatch = results.OfType<NoMatch>().ToArray();
-            Info($"No match for {nomatch.Count()} files");
+            Info($"No match for {nomatch.Length} files");
-            foreach (var file in nomatch)
-                Info($"     {file.To}");
             if (nomatch.Any())
             {
+                foreach (var file in nomatch)
+                    Info($"     {file.To}");
                 if (IgnoreMissingFiles)
                 {
                     Info("Continuing even though files were missing at the request of the user.");

Wabbajack.Lib/VortexCompiler.cs

@@ -117,18 +117,15 @@ namespace Wabbajack.Lib
             UpdateTracker.NextStep("Finding Install Files");
             var vortexStagingFiles = Directory.EnumerateFiles(StagingFolder, "*", SearchOption.AllDirectories)
                 .Where(p => p.FileExists() && p != StagingMarkerName && !p.Contains(Consts.ManualGameFilesDir))
-                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
+                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(StagingFolder)));
-                    {Path = p.RelativeTo(StagingFolder)});
             
             var vortexDownloads = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.AllDirectories)
                 .Where(p => p.FileExists() && p != DownloadMarkerName)
-                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
+                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(DownloadsFolder)));
-                    {Path = p.RelativeTo(DownloadsFolder)});
 
             var gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
                 .Where(p => p.FileExists())
-                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
+                .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath))));
-                    { Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) });
 
             Info("Indexing Archives");
             IndexedArchives = Directory.EnumerateFiles(DownloadsFolder)

Wabbajack.Test/MO2Tests.cs

@@ -80,7 +80,7 @@ namespace Wabbajack.Test
                 var downloadsFolder = Path.Combine(tempDir.Dir.FullName, "downloads");
                 Directory.CreateDirectory(downloadsFolder);
                 File.Create(Path.Combine(tempDir.Dir.FullName, $"downloads/someFile.txt"));
-                Assert.IsFalse(MO2Installer.CheckValidInstallPath(tempDir.Dir.FullName, downloadFolder: downloadsFolder).Succeeded);
+                Assert.IsTrue(MO2Installer.CheckValidInstallPath(tempDir.Dir.FullName, downloadFolder: downloadsFolder).Succeeded);
             }
         }
         #endregion

Wabbajack.Test/PMapTests.cs

@@ -0,0 +1,220 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Wabbajack.Common;
+
+namespace Wabbajack.Test
+{
+    [TestClass]
+    public class PMapTests
+    {
+        const int TypicalThreadCount = 2;
+        const int TypicalDelayMS = 50;
+
+        [TestMethod]
+        public async Task Typical_Action()
+        {
+            using (var queue = new WorkQueue(TypicalThreadCount))
+            {
+                var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
+                var output = new List<int>();
+                await Enumerable.Range(0, TypicalThreadCount * 2)
+                    .PMap(queue, (item) =>
+                    {
+                        Assert.IsTrue(WorkQueue.WorkerThread);
+                        Thread.Sleep(TypicalDelayMS);
+                        lock (output)
+                        {
+                            output.Add(item);
+                        }
+                    });
+                Assert.IsTrue(input.SequenceEqual(output.OrderBy(i => i)));
+            }
+        }
+
+        [TestMethod]
+        public async Task Typical_Func()
+        {
+            using (var queue = new WorkQueue(TypicalThreadCount))
+            {
+                var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
+                var results = await Enumerable.Range(0, TypicalThreadCount * 2)
+                    .PMap(queue, (item) =>
+                    {
+                        Assert.IsTrue(WorkQueue.WorkerThread);
+                        Thread.Sleep(TypicalDelayMS);
+                        return item.ToString();
+                    });
+                Assert.IsTrue(input.Select(i => i.ToString()).SequenceEqual(results));
+            }
+        }
+
+        [TestMethod]
+        public async Task Typical_Task()
+        {
+            using (var queue = new WorkQueue(TypicalThreadCount))
+            {
+                var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
+                var output = new List<int>();
+                await Enumerable.Range(0, TypicalThreadCount * 2)
+                    .PMap(queue, async (item) =>
+                    {
+                        Assert.IsTrue(WorkQueue.WorkerThread);
+                        await Task.Delay(TypicalDelayMS);
+                        lock (output)
+                        {
+                            output.Add(item);
+                        }
+                    });
+                Assert.IsTrue(input.SequenceEqual(output.OrderBy(i => i)));
+            }
+        }
+
+        [TestMethod]
+        public async Task Typical_TaskReturn()
+        {
+            using (var queue = new WorkQueue(TypicalThreadCount))
+            {
+                var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
+                var results = await Enumerable.Range(0, TypicalThreadCount * 2)
+                    .PMap(queue, async (item) =>
+                    {
+                        Assert.IsTrue(WorkQueue.WorkerThread);
+                        await Task.Delay(TypicalDelayMS);
+                        return item.ToString();
+                    });
+                Assert.IsTrue(input.Select(i => i.ToString()).SequenceEqual(results));
+            }
+        }
+
+        [TestMethod]
+        public async Task NestedAction()
+        {
+            using (var queue = new WorkQueue(TypicalThreadCount))
+            {
+                var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
+                var inputConstructedResults = input.SelectMany(i => Enumerable.Range(i * 100, TypicalThreadCount * 2));
+                var output = new List<int>();
+                await Enumerable.Range(0, TypicalThreadCount * 2)
+                    .PMap(queue, async (item) =>
+                    {
+                        Assert.IsTrue(WorkQueue.WorkerThread);
+                        await Enumerable.Range(item * 100, TypicalThreadCount * 2)
+                            .PMap(queue, async (subItem) =>
+                            {
+                                Assert.IsTrue(WorkQueue.WorkerThread);
+                                Thread.Sleep(TypicalDelayMS);
+                                lock (output)
+                                {
+                                    output.Add(subItem);
+                                }
+                            });
+                    });
+                Assert.IsTrue(inputConstructedResults.SequenceEqual(output.OrderBy(i => i)));
+            }
+        }
+
+        [TestMethod]
+        public async Task Nested_Func()
+        {
+            using (var queue = new WorkQueue(TypicalThreadCount))
+            {
+                var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
+                var inputConstructedResults = input.SelectMany(i => Enumerable.Range(i * 100, TypicalThreadCount * 2));
+                var results = await Enumerable.Range(0, TypicalThreadCount * 2)
+                    .PMap(queue, async (item) =>
+                    {
+                        Assert.IsTrue(WorkQueue.WorkerThread);
+                        return await Enumerable.Range(item * 100, TypicalThreadCount * 2)
+                            .PMap(queue, (subItem) =>
+                            {
+                                Assert.IsTrue(WorkQueue.WorkerThread);
+                                Thread.Sleep(TypicalDelayMS);
+                                return subItem;
+                            });
+                    });
+                Assert.IsTrue(inputConstructedResults.SequenceEqual(results.SelectMany(i => i)));
+            }
+        }
+
+        [TestMethod]
+        public async Task Nested_Task()
+        {
+            using (var queue = new WorkQueue(TypicalThreadCount))
+            {
+                var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
+                var inputConstructedResults = input.SelectMany(i => Enumerable.Range(i * 100, TypicalThreadCount * 2));
+                var output = new List<int>();
+                await Enumerable.Range(0, TypicalThreadCount * 2)
+                    .PMap(queue, async (item) =>
+                    {
+                        Assert.IsTrue(WorkQueue.WorkerThread);
+                        await Enumerable.Range(item * 100, TypicalThreadCount * 2)
+                            .PMap(queue, async (subItem) =>
+                            {
+                                Assert.IsTrue(WorkQueue.WorkerThread);
+                                await Task.Delay(TypicalDelayMS);
+                                lock (output)
+                                {
+                                    output.Add(subItem);
+                                }
+                            });
+                    });
+                Assert.IsTrue(inputConstructedResults.SequenceEqual(output.OrderBy(i => i)));
+            }
+        }
+
+        [TestMethod]
+        public async Task Nested_TaskReturn()
+        {
+            using (var queue = new WorkQueue(TypicalThreadCount))
+            {
+                var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
+                var inputConstructedResults = input.SelectMany(i => Enumerable.Range(i * 100, TypicalThreadCount * 2));
+                var results = await Enumerable.Range(0, TypicalThreadCount * 2)
+                    .PMap(queue, async (item) =>
+                    {
+                        Assert.IsTrue(WorkQueue.WorkerThread);
+                        return await Enumerable.Range(item * 100, TypicalThreadCount * 2)
+                            .PMap(queue, async (subItem) =>
+                            {
+                                Assert.IsTrue(WorkQueue.WorkerThread);
+                                await Task.Delay(TypicalDelayMS);
+                                return subItem;
+                            });
+                    });
+                Assert.IsTrue(inputConstructedResults.SequenceEqual(results.SelectMany(i => i)));
+            }
+        }
+
+        [TestMethod]
+        public async Task Nested_BackgroundThreadsInvolved()
+        {
+            using (var queue = new WorkQueue(TypicalThreadCount))
+            {
+                var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
+                var inputConstructedResults = input.SelectMany(i => Enumerable.Range(i * 100, TypicalThreadCount * 2));
+                var results = await Enumerable.Range(0, TypicalThreadCount * 2)
+                    .PMap(queue, async (item) =>
+                    {
+                        Assert.IsTrue(WorkQueue.WorkerThread);
+                        return await Enumerable.Range(item * 100, TypicalThreadCount * 2)
+                            .PMap(queue, async (subItem) =>
+                            {
+                                return await Task.Run(async () =>
+                                {
+                                    Assert.IsTrue(WorkQueue.WorkerThread);
+                                    await Task.Delay(TypicalDelayMS);
+                                    return subItem;
+                                });
+                            });
+                    });
+                Assert.IsTrue(inputConstructedResults.SequenceEqual(results.SelectMany(i => i)));
+            }
+        }
+    }
+}

Wabbajack.Test/SanityTests.cs

@@ -128,8 +128,8 @@ namespace Wabbajack.Test
 
             // Update the file and verify that it throws an error.
             utils.GenerateRandomFileData(game_file, 20);
-            var exception = Assert.ThrowsException<AggregateException>(() => Install(compiler));
+            var exception = await Assert.ThrowsExceptionAsync<InvalidGameESMError>(async () => await Install(compiler));
-            Assert.IsInstanceOfType(exception.InnerExceptions.First(), typeof(InvalidGameESMError));
+            Assert.IsInstanceOfType(exception, typeof(InvalidGameESMError));
         }
 
         [TestMethod]

Wabbajack.Test/Wabbajack.Test.csproj

@@ -110,6 +110,7 @@
     <Compile Include="MiscTests.cs" />
     <Compile Include="MO2Tests.cs" />
     <Compile Include="ModlistMetadataTests.cs" />
+    <Compile Include="PMapTests.cs" />
     <Compile Include="RestartingDownloadsTests.cs" />
     <Compile Include="SimpleHTTPServer.cs" />
     <Compile Include="TestUtils.cs" />

Wabbajack.VirtualFileSystem/Context.cs

@@ -98,6 +98,7 @@ namespace Wabbajack.VirtualFileSystem
             var allFiles = await filesToIndex
                 .PMap(Queue, async f =>
                 {
+                    Utils.Status($"Indexing {Path.GetFileName(f)}");
                     if (byPath.TryGetValue(f, out var found))
                     {
                         var fi = new FileInfo(f);