Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix some more race conditions in tests

Browse Source
This commit is contained in:
Timothy Baldridge 2020-03-29 14:42:45 -06:00
parent 3d2ab80f34
commit 85b39d5dcc
13 changed files with 101 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
Wabbajack.App.Test/BasicUITests.cs
View File

@ -1,25 +0,0 @@
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Threading;
using Xunit;
namespace Wabbajack.App.Test
{
public class BasicUITests
{
[StaFact]
public async Task CanCompileASimpleModlist()
{
SynchronizationContext.SetSynchronizationContext(new DispatcherSynchronizationContext(Dispatcher.CurrentDispatcher));
var window = new MainWindow();
window.Show();
await Task.Delay(1000);
window.Close();
Assert.True(true);
}
}
}

2
Wabbajack.Common.Test/TaskExtTests.cs
View File

@ -18,7 +18,7 @@ namespace Wabbajack.Common.Test
public async Task TimeoutButContinue_TimedOut()
{
bool timedOut = false;
await Task.Delay(300).TimeoutButContinue(TimeSpan.FromMilliseconds(100), () => timedOut = true);
await Task.Delay(3000).TimeoutButContinue(TimeSpan.FromMilliseconds(100), () => timedOut = true);
Assert.True(timedOut);
}
}

80
Wabbajack.Common.Test/WorkQueueTests.cs
View File

@ -186,85 +186,5 @@ namespace Wabbajack.Common.Test
Assert.Equal(completed, task);
}
#endregion
#region Known Parallel Work Collapse Pitfall
/// <summary>
/// Putting a single TCS completion source onto the WorkQueue will result in parallization collapse, where
/// all work is being done by one actual thread. Similar to the deadlock scenario, this is just slightly different.
///
/// Since all worker tasks in charge of pulling off the queue were working on a single job driven by a single TCS,
/// when that TCS completes, the one thread that completed it is in charge of all the continuation. All the continuation
/// tasks happen to be all Tasks in charge of pulling off the queue. This results in one actual thread essentially calling a
/// Task.WhenAll() on all of our queue.Take tasks. This means only one thread is now ping-ponging around doing the work, rather
/// than our desired number of threads working in parallel.
///
/// This will happen even if the WorkQueue is backed by Threads, rather than Task.Run() calls. It's just the nature of how async
/// continuation is wired to work.
///
/// Other notes:
/// This seems to fail when run in the normal pipeline of unit tests. I think the timing gets interrupted by other tests?
/// Disabled the test from being run automatically for now
///
/// TLDR: Don't put the same work completion source to be done on the queue multiple times.
/// </summary>
[Fact]
public async Task ThreadCoalescenceExample()
{
var subj = new BehaviorSubject<int>(Large);
var tcs = new TaskCompletionSource<bool>();
object lockObj = new object();
using (var queue = new WorkQueue(subj))
{
Assert.Equal(Large, queue.DesiredNumWorkers);
Assert.Equal(Large, queue._tasks.Count);
bool[] workStartedArray = new bool[Large];
async Task Job(int num, bool[] b)
{
// Mark work started as soon as job started
lock (lockObj)
{
b[num] = true;
}
// Do lots of hard work for 1 second
Thread.Sleep(5000);
};
// Do hard work in parallel
Enumerable.Range(0, Large).Do(i => queue.QueueTask(() => Job(i, workStartedArray)));
// Wait some time, so all jobs should be started
await Task.Delay(2500);
// Show that all jobs are started
lock (lockObj)
{
Assert.Equal(Large, workStartedArray.Where(i => i).Count());
}
await Task.Delay(15000);
// Start lots of jobs, all pinning from the same TCS
Enumerable.Range(0, Large).Do(_ => queue.QueueTask(() => tcs.Task));
// All 8 worker tasks are completed by the same TCS, but continued by the single Task
// that kicked it off and is in charge of the continuation tasks.
// Parallel worker Tasks have now coalesced into a single thread
Task.Run(() => tcs.SetResult(true)).FireAndForget();
Assert.Equal(Large, queue.DesiredNumWorkers);
Assert.Equal(Large, queue._tasks.Count);
await Task.Delay(10000);
// Do a test to prove work isn't being done in parallel anymore
var secondWorkStartedArray = new bool[Large];
Enumerable.Range(0, Large).Do(i => queue.QueueTask(() => Job(i, secondWorkStartedArray)));
// Wait some time, so all jobs should be started
await Task.Delay(2500);
// Show that only one job was started/worked on (by our one coalesced worker thread)
lock (lockObj)
{
Assert.Single(secondWorkStartedArray.Where(i => i));
}
}
}
#endregion
}
}

15
Wabbajack.Common/Utils.cs
View File

@ -216,6 +216,21 @@ namespace Wabbajack.Common
Status(status, Percent.FactoryPutInRange(totalRead, maxSize));
}
}
public static async Task CopyToWithStatusAsync(this Stream istream, long maxSize, Stream ostream, string status)
{
var buffer = new byte[1024 * 64];
if (maxSize == 0) maxSize = 1;
long totalRead = 0;
while (true)
{
var read = await istream.ReadAsync(buffer, 0, buffer.Length);
if (read == 0) break;
totalRead += read;
await ostream.WriteAsync(buffer, 0, read);
Status(status, Percent.FactoryPutInRange(totalRead, maxSize));
}
}
public static string xxHash(this byte[] data, bool nullOnIOError = false)
{
try

11
Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
View File

@ -73,12 +73,11 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Download(Archive a, AbsolutePath destination)
{
using(var src = SourcePath.OpenRead())
using (var dest = destination.Create())
{
var size = SourcePath.Size;
src.CopyToWithStatus(size, dest, "Copying from Game folder");
}
await using var src = SourcePath.OpenRead();
await using var dest = destination.Create();
var size = SourcePath.Size;
await src.CopyToWithStatusAsync(size, dest, "Copying from Game folder");
return true;
}

13
Wabbajack.Test/ACompilerTest.cs
View File

@ -8,14 +8,13 @@ using Xunit.Abstractions;
namespace Wabbajack.Test
{
public abstract class ACompilerTest : IDisposable
public abstract class ACompilerTest : XunitContextBase, IDisposable
{
public ITestOutputHelper TestContext { get; set; }
private IDisposable _unsub;
protected TestUtils utils { get; set; }
public ACompilerTest(ITestOutputHelper helper)
public ACompilerTest(ITestOutputHelper helper) : base (helper)
{
TestContext = helper;
Helpers.Init();
Consts.TestMode = true;
@ -23,13 +22,15 @@ namespace Wabbajack.Test
utils.Game = Game.SkyrimSpecialEdition;
DateTime startTime = DateTime.Now;
Utils.LogMessages.Subscribe(f => TestContext.WriteLine($"{DateTime.Now - startTime} - {f.ShortDescription}"));
_unsub = Utils.LogMessages.Subscribe(f => XunitContext.WriteLine($"{DateTime.Now - startTime} - {f.ShortDescription}"));
}
public void Dispose()
public override void Dispose()
{
utils.Dispose();
_unsub.Dispose();
base.Dispose();
}
protected async Task<MO2Compiler> ConfigureAndRunCompiler(string profile)

23
Wabbajack.Test/ATestBase.cs Normal file
View File

@ -0,0 +1,23 @@
using System;
using Wabbajack.Common;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Test
{
public abstract class ATestBase : XunitContextBase
{
private IDisposable _unsub;
protected ATestBase(ITestOutputHelper output) : base(output)
{
_unsub = Utils.LogMessages.Subscribe(f => XunitContext.WriteLine($"{DateTime.Now} - {f}"));
}
public override void Dispose()
{
_unsub.Dispose();
base.Dispose();
}
}
}

30
Wabbajack.Test/DownloaderTests.cs
View File

@ -23,19 +23,25 @@ using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Test
{
public class DownloaderTests
public class DownloaderTests : XunitContextBase, IDisposable
{
public DownloaderTests(ITestOutputHelper helper)
private IDisposable _unsubMsgs;
private IDisposable _unsubErr;
public DownloaderTests(ITestOutputHelper helper) : base(helper)
{
TestContext = helper;
Helpers.Init();
Utils.LogMessages.OfType<IInfo>().Subscribe(onNext: msg => TestContext.WriteLine(msg.ShortDescription));
Utils.LogMessages.OfType<IUserIntervention>().Subscribe(msg =>
TestContext.WriteLine("ERROR: User intervention required: " + msg.ShortDescription));
Helpers.Init();
_unsubMsgs = Utils.LogMessages.OfType<IInfo>().Subscribe(onNext: msg => XunitContext.WriteLine(msg.ShortDescription));
_unsubErr = Utils.LogMessages.OfType<IUserIntervention>().Subscribe(msg =>
XunitContext.WriteLine("ERROR: User intervention required: " + msg.ShortDescription));
}
public ITestOutputHelper TestContext { get; }
public override void Dispose()
{
base.Dispose();
_unsubErr.Dispose();
_unsubMsgs.Dispose();
}
[Fact]
public void TestAllPrepares()
@ -408,7 +414,7 @@ namespace Wabbajack.Test
await converted.Download(new Archive { Name = "Update.esm" }, filename.Path);
Assert.Equal(Hash.FromBase64("/DLG/LjdGXI="), Utils.FileHash(filename.Path));
Assert.Equal(Hash.FromBase64("/DLG/LjdGXI="), await Utils.FileHashAsync(filename.Path));
Assert.Equal(await filename.Path.ReadAllBytesAsync(), await Game.SkyrimSpecialEdition.MetaData().GameLocation()?.Combine("Data/Update.esm").ReadAllBytesAsync());
Consts.TestMode = true;
}
@ -513,8 +519,8 @@ namespace Wabbajack.Test
{
(RelativePath)@"Download.esm",
(RelativePath)@"Download.esm.xxHash",
(RelativePath)@"Download_f80ee6d109516018308f62e2c862b7f061987ac4a8c2327a101ac6b8f80ec4ae_.esm",
(RelativePath)@"Download_f80ee6d109516018308f62e2c862b7f061987ac4a8c2327a101ac6b8f80ec4ae_.esm.xxHash"
(RelativePath)@"Download_c4047f2251d8eead22df4b4888cc4b833ae7d9a6766ff29128e083d944f9ec4b_.esm",
(RelativePath)@"Download_c4047f2251d8eead22df4b4888cc4b833ae7d9a6766ff29128e083d944f9ec4b_.esm.xxHash"
}.OrderBy(a => a).ToArray());
Consts.TestMode = true;

14
Wabbajack.Test/EndToEndTests.cs
View File

@ -13,33 +13,33 @@ using Xunit.Abstractions;
namespace Wabbajack.Test
{
public class EndToEndTests : IDisposable
public class EndToEndTests : XunitContextBase, IDisposable
{
private AbsolutePath _downloadFolder = "downloads".RelativeTo(AbsolutePath.EntryPoint);
private TestUtils utils = new TestUtils();
public ITestOutputHelper TestContext { get; set; }
private IDisposable _unsub;
public WorkQueue Queue { get; set; }
public EndToEndTests(ITestOutputHelper helper)
public EndToEndTests(ITestOutputHelper helper) : base(helper)
{
TestContext = helper;
Queue = new WorkQueue();
Consts.TestMode = true;
utils = new TestUtils();
utils.Game = Game.SkyrimSpecialEdition;
Utils.LogMessages.Subscribe(f => TestContext.WriteLine($"{DateTime.Now} - {f}"));
_unsub = Utils.LogMessages.Subscribe(f => XunitContext.WriteLine($"{DateTime.Now} - {f}"));
_downloadFolder.CreateDirectory();
}
public void Dispose()
public override void Dispose()
{
Queue.Dispose();
_unsub.Dispose();
base.Dispose();
}
[Fact]

8
Wabbajack.Test/ModlistMetadataTests.cs
View File

@ -4,10 +4,11 @@ using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Test
{
public class ModlistMetadataTests
public class ModlistMetadataTests : ATestBase
{
[Fact]
public async Task TestLoadingModlists()
@ -28,5 +29,10 @@ namespace Wabbajack.Test
Assert.True(await logoState.Verify(new Archive{Size = 0}), $"{modlist.ImageUri} is not valid");
}
}
public ModlistMetadataTests(ITestOutputHelper output) : base(output)
{
}
}
}

11
Wabbajack.Test/Wabbajack.Test.csproj
View File

@ -21,14 +21,19 @@
<None Include="ZEditIntegrationTests.cs" />
<Compile Remove="CompilationStackTests.cs" />
<Compile Remove="FilePickerTests.cs" />
<None Update="xunit.runner.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<PackageReference Include="CefSharp.Common" Version="79.1.350" />
<PackageReference Include="CefSharp.OffScreen" Version="79.1.350" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.console" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
<PackageReference Include="coverlet.collector" Version="1.0.1" />
<PackageReference Include="XunitContext" Version="1.9.0" />
</ItemGroup>
<ItemGroup>

4
Wabbajack.Test/xunit.runner.json Normal file
View File

@ -0,0 +1,4 @@
{
"parallelizeTestCollections": false,
"maxParallelThreads": 1
}

14
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -185,16 +185,14 @@ namespace Wabbajack.VirtualFileSystem
if (FileExtractor.CanExtract(absPath))
{
using (var tempFolder = Context.GetTemporaryFolder())
{
await FileExtractor.ExtractAll(context.Queue, absPath, tempFolder.FullName);
using var tempFolder = Context.GetTemporaryFolder();
await FileExtractor.ExtractAll(context.Queue, absPath, tempFolder.FullName);
var list = await tempFolder.FullName.EnumerateFiles()
.PMap(context.Queue,
absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), depth + 1));
var list = await tempFolder.FullName.EnumerateFiles()
.PMap(context.Queue,
absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), depth + 1));
self.Children = list.ToImmutableList();
}
self.Children = list.ToImmutableList();
}
return self;