Merge pull request #657 from wabbajack-tools/fixes-for-1.1.4.0

Fixes and changes to get into 1.1.4.0

CHANGELOG.md

@@ -1,7 +1,10 @@
 ### Changelog
 
-#### Version - Next
+#### Version - 3/30/2020
 * Added support for Morrowind on GOG
+* Fix a bug in the Author file uploader (Sync Error)
+* Include symbols in the launcher
+* Fix a small race condition/deadlock in the worker queue code
 
 #### Version - 1.1.3.0 - 3/23/2020
 * Fix for a lack of VC++ Redist dlls on newly installed Windows machines.

Wabbajack.BuildServer/Controllers/Heartbeat.cs

@@ -3,10 +3,12 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using MongoDB.Bson;
+using MongoDB.Driver;
 using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.BuildServer.Models;
 using Wabbajack.Common;
@@ -65,5 +67,23 @@ namespace Wabbajack.BuildServer.Controllers
             }
             return Ok(string.Join("\n", lst));
         }
+
+        [HttpPost("export_inis")]
+        [Authorize]
+        public async Task<IActionResult> ExportInis()
+        {
+            if (!Directory.Exists("exported_inis"))
+                Directory.CreateDirectory("exported_inis");
+
+            var loaded = 0;
+            foreach (var ini in await Db.DownloadStates.AsQueryable().ToListAsync())
+            {
+                var file = Path.Combine("exported_inis", ini.Hash.FromBase64().ToHex() + ".ini");
+                Alphaleonis.Win32.Filesystem.File.WriteAllLines(file, ini.State.GetMetaIni());
+                loaded += 1;
+            }
+
+            return Ok(loaded);
+        }
     }
 }

Wabbajack.BuildServer/Wabbajack.BuildServer.csproj

@@ -7,8 +7,8 @@
         <PublishReadyToRun>true</PublishReadyToRun>
         <PublishSingleFile>true</PublishSingleFile>
         <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
-        <AssemblyVersion>1.1.2.0</AssemblyVersion>
+        <AssemblyVersion>1.1.4.0</AssemblyVersion>
-        <FileVersion>1.1.2.0</FileVersion>
+        <FileVersion>1.1.4.0</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Server component for Wabbajack</Description>
         <AssemblyName>BuildServer</AssemblyName>

Wabbajack.CLI/Wabbajack.CLI.csproj

@@ -6,8 +6,8 @@
         <AssemblyName>wabbajack-cli</AssemblyName>
         <Company>Wabbajack</Company>
         <Platforms>x64</Platforms>
-        <AssemblyVersion>1.1.2.0</AssemblyVersion>
+        <AssemblyVersion>1.1.4.0</AssemblyVersion>
-        <FileVersion>1.1.2.0</FileVersion>
+        <FileVersion>1.1.4.0</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>An automated ModList installer</Description>
         <PublishReadyToRun>true</PublishReadyToRun>        

Wabbajack.Common/AsyncBlockingConnection.cs

@@ -0,0 +1,46 @@
+using System;
+using System.Collections;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Wabbajack.Common
+{
+    public class AsyncBlockingCollection<T> : IDisposable
+    {
+        private readonly ConcurrentQueue<T> _collection;
+        private bool isDisposed = false;
+
+        public AsyncBlockingCollection()
+        {
+            _collection = new ConcurrentQueue<T>();
+        }
+        
+        public void Add(T val)
+        {
+            _collection.Enqueue(val);
+        }
+
+        public async ValueTask<(bool found, T val)> TryTake(TimeSpan timeout, CancellationToken token)
+        {
+            var startTime = DateTime.Now;
+            while (true)
+            {
+                if (_collection.TryDequeue(out T result))
+                {
+                    return (true, result);
+                }
+
+                if (DateTime.Now - startTime > timeout || token.IsCancellationRequested || isDisposed)
+                    return (false, default);
+                await Task.Delay(100);
+            }
+        }
+
+        public void Dispose()
+        {
+            isDisposed = true;
+        }
+    }
+}

Wabbajack.Common/Utils.cs

@@ -759,7 +759,8 @@ namespace Wabbajack.Common
             {
                 while (remainingTasks > 0)
                 {
-                    if (queue.Queue.TryTake(out var a, 500))
+                    var (got, a) = await queue.Queue.TryTake(TimeSpan.FromMilliseconds(200), CancellationToken.None);
+                    if (got)
                     {
                         await a();
                     }
@@ -799,7 +800,8 @@ namespace Wabbajack.Common
             {
                 while (remainingTasks > 0)
                 {
-                    if (queue.Queue.TryTake(out var a, 500))
+                    var (got, a) = await queue.Queue.TryTake(TimeSpan.FromMilliseconds(200), CancellationToken.None);
+                    if (got)
                     {
                         await a();
                     }
@@ -840,7 +842,8 @@ namespace Wabbajack.Common
             {
                 while (remainingTasks > 0)
                 {
-                    if (queue.Queue.TryTake(out var a, 500))
+                    var (got, a) = await queue.Queue.TryTake(TimeSpan.FromMilliseconds(200), CancellationToken.None);
+                    if (got)
                     {
                         await a();
                     }

Wabbajack.Common/WorkQueue.cs

@@ -16,7 +16,7 @@ namespace Wabbajack.Common
 {
     public class WorkQueue : IDisposable
     {
-        internal BlockingCollection<Func<Task>> Queue = new BlockingCollection<Func<Task>>(new ConcurrentStack<Func<Task>>());
+        internal AsyncBlockingCollection<Func<Task>> Queue = new AsyncBlockingCollection<Func<Task>>();
 
         public const int UnassignedCpuId = 0;
 
@@ -50,7 +50,7 @@ namespace Wabbajack.Common
 
         private readonly Subject<IObservable<int>> _activeNumThreadsObservable = new Subject<IObservable<int>>();
 
-        internal const int PollMS = 200;
+        public TimeSpan PollMS = TimeSpan.FromMilliseconds(200);
 
         /// <summary>
         /// Creates a WorkQueue with the given number of threads
@@ -124,7 +124,7 @@ namespace Wabbajack.Common
                     bool got;
                     try
                     {
-                        got = Queue.TryTake(out f, PollMS, _shutdown.Token);
+                        (got, f) = await Queue.TryTake(PollMS, _shutdown.Token);
                     }
                     catch (Exception)
                     {

Wabbajack.Launcher/Wabbajack.Launcher.csproj

@@ -4,8 +4,8 @@
         <OutputType>WinExe</OutputType>
         <TargetFramework>netcoreapp3.1</TargetFramework>
         <UseWPF>true</UseWPF>
-        <AssemblyVersion>1.1.2.0</AssemblyVersion>
+        <AssemblyVersion>1.1.4.0</AssemblyVersion>
-        <FileVersion>1.1.2.0</FileVersion>
+        <FileVersion>1.1.4.0</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Application Launcher</Description>
         <PublishReadyToRun>true</PublishReadyToRun>

Wabbajack.Lib/FileUploader/AuthorAPI.cs

@@ -83,16 +83,16 @@ namespace Wabbajack.Lib.FileUploader
                             await fs.ReadAsync(data, 0, data.Length);
 
                             
-                            response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
+                            var putResponse = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
                                 new ByteArrayContent(data));
 
-                            if (!response.IsSuccessStatusCode)
+                            if (!putResponse.IsSuccessStatusCode)
                             {
-                                tcs.SetException(new Exception($"Put Error: {response.StatusCode} {response.ReasonPhrase}"));
+                                tcs.SetException(new Exception($"Put Error: {putResponse.StatusCode} {putResponse.ReasonPhrase}"));
                                 return;
                             }
 
-                            var val = long.Parse(await response.Content.ReadAsStringAsync());
+                            var val = long.Parse(await putResponse.Content.ReadAsStringAsync());
                             if (val != block_offset + data.Length)
                             {
                                 tcs.SetResult($"Sync Error {val} vs {block_offset + data.Length}");

Wabbajack.Test/WorkQueueTests.cs

@@ -18,7 +18,7 @@ namespace Wabbajack.Test
         const int Large = 8;
         const int Medium = 6;
         const int Small = 4;
-        public int PollMS => WorkQueue.PollMS * 5;
+        public TimeSpan PollMS => TimeSpan.FromSeconds(1);
 
         [TestMethod]
         public void DynamicNumThreads_Typical()