Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #305 from Noggog/patch-freeze-investigation

Browse Source 
Patch freeze investigation
This commit is contained in:
Timothy Baldridge 2019-12-22 21:17:00 -08:00 committed by GitHub
commit b1b4ac5829
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 259 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Wabbajack.Common/FileExtractor.cs
View File

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
@ -208,6 +208,7 @@ namespace Wabbajack.Common
if (p.ExitCode == 0)
{
Utils.Status($"Extracting {name} - 100%", 100, alsoLog: true);
return;
}
Utils.Log(new _7zipReturnError(p.ExitCode, source, dest, p.StandardOutput.ReadToEnd()));

15
Wabbajack.Common/Utils.cs
View File

@ -597,12 +597,15 @@ namespace Wabbajack.Common
// To avoid thread starvation, we'll start to help out in the work queue
if (WorkQueue.WorkerThread)
{
while (remainingTasks > 0)
{
if (queue.Queue.TryTake(out var a, 500))
{
WorkQueue.AsyncLocalCurrentQueue.Value = WorkQueue.ThreadLocalCurrentQueue.Value;
await a();
}
}
}
return await Task.WhenAll(tasks);
}
@ -634,12 +637,15 @@ namespace Wabbajack.Common
// To avoid thread starvation, we'll start to help out in the work queue
if (WorkQueue.WorkerThread)
{
while (remainingTasks > 0)
{
if (queue.Queue.TryTake(out var a, 500))
{
WorkQueue.AsyncLocalCurrentQueue.Value = WorkQueue.ThreadLocalCurrentQueue.Value;
await a();
}
}
}
return await Task.WhenAll(tasks);
}
@ -672,12 +678,15 @@ namespace Wabbajack.Common
// To avoid thread starvation, we'll start to help out in the work queue
if (WorkQueue.WorkerThread)
{
while (remainingTasks > 0)
{
if (queue.Queue.TryTake(out var a, 500))
{
WorkQueue.AsyncLocalCurrentQueue.Value = WorkQueue.ThreadLocalCurrentQueue.Value;
await a();
}
}
}
await Task.WhenAll(tasks);
}

5
Wabbajack.Common/WorkQueue.cs
View File

@ -20,8 +20,8 @@ namespace Wabbajack.Common
private static readonly AsyncLocal<int> _cpuId = new AsyncLocal<int>();
public int CpuId => _cpuId.Value;
internal static bool WorkerThread => ThreadLocalCurrentQueue.Value != null;
internal static readonly ThreadLocal<WorkQueue> ThreadLocalCurrentQueue = new ThreadLocal<WorkQueue>();
public static bool WorkerThread => AsyncLocalCurrentQueue.Value != null;
public bool IsWorkerThread => WorkerThread;
internal static readonly AsyncLocal<WorkQueue> AsyncLocalCurrentQueue = new AsyncLocal<WorkQueue>();
private readonly Subject<CPUStatus> _Status = new Subject<CPUStatus>();
@ -61,7 +61,6 @@ namespace Wabbajack.Common
private async Task ThreadBody(int idx)
{
_cpuId.Value = idx;
ThreadLocalCurrentQueue.Value = this;
AsyncLocalCurrentQueue.Value = this;
try

6
Wabbajack.Lib/ACompiler.cs
View File

@ -221,13 +221,15 @@ namespace Wabbajack.Lib
public async Task<Archive> ResolveArchive(IndexedArchive archive)
{
Utils.Status($"Checking link for {archive.Name}", alsoLog: true);
if (archive.IniData == null)
Error(
$"No download metadata found for {archive.Name}, please use MO2 to query info or add a .meta file and try again.");
var result = new Archive
{
State = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(archive.IniData)
State = await DownloadDispatcher.ResolveArchive(archive.IniData)
};
if (result.State == null)
@ -238,8 +240,6 @@ namespace Wabbajack.Lib
result.Meta = archive.Meta;
result.Size = archive.File.Size;
Info($"Checking link for {archive.Name}");
if (result.State != null && !await result.State.Verify())
Error(
$"Unable to resolve link for {archive.Name}. If this is hosted on the Nexus the file may have been removed.");

5
Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs
View File

@ -65,10 +65,7 @@ namespace Wabbajack.Lib.CompilationSteps
var id = Guid.NewGuid().ToString();
var matches = await sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e)
{
Path = Path.Combine(Consts.BSACreationDir, id, e.Name)
}));
var matches = await sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e, Path.Combine(Consts.BSACreationDir, id, e.Name))));
foreach (var match in matches)

5
Wabbajack.Lib/Data.cs
View File

@ -11,11 +11,14 @@ namespace Wabbajack.Lib
{
public class RawSourceFile
{
// ToDo
// Make readonly
public string Path;
public RawSourceFile(VirtualFile file)
public RawSourceFile(VirtualFile file, string path)
{
File = file;
Path = path;
}
public string AbsolutePath => File.StagedPath;

16
Wabbajack.Lib/MO2Compiler.cs
View File

@ -119,8 +119,7 @@ namespace Wabbajack.Lib
{
lootFiles = Directory.EnumerateFiles(lootPath, "userlist.yaml", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{ Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath)) });
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath))));
}
if (cancel.IsCancellationRequested) return false;
@ -158,12 +157,11 @@ namespace Wabbajack.Lib
var mo2Files = Directory.EnumerateFiles(MO2Folder, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p]) { Path = p.RelativeTo(MO2Folder) });
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(MO2Folder)));
var gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{ Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) });
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath))));
ModMetas = Directory.EnumerateDirectories(Path.Combine(MO2Folder, "mods"))
@ -229,12 +227,12 @@ namespace Wabbajack.Lib
UpdateTracker.NextStep($"Adding {ExtraFiles.Count} that were generated by the stack");
results = results.Concat(ExtraFiles).ToArray();
var nomatch = results.OfType<NoMatch>();
Info($"No match for {nomatch.Count()} files");
foreach (var file in nomatch)
Info($" {file.To}");
var nomatch = results.OfType<NoMatch>().ToArray();
Info($"No match for {nomatch.Length} files");
if (nomatch.Any())
{
foreach (var file in nomatch)
Info($" {file.To}");
if (IgnoreMissingFiles)
{
Info("Continuing even though files were missing at the request of the user.");

9
Wabbajack.Lib/VortexCompiler.cs
View File

@ -117,18 +117,15 @@ namespace Wabbajack.Lib
UpdateTracker.NextStep("Finding Install Files");
var vortexStagingFiles = Directory.EnumerateFiles(StagingFolder, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists() && p != StagingMarkerName && !p.Contains(Consts.ManualGameFilesDir))
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{Path = p.RelativeTo(StagingFolder)});
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(StagingFolder)));
var vortexDownloads = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists() && p != DownloadMarkerName)
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{Path = p.RelativeTo(DownloadsFolder)});
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], p.RelativeTo(DownloadsFolder)));
var gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{ Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) });
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath))));
Info("Indexing Archives");
IndexedArchives = Directory.EnumerateFiles(DownloadsFolder)

2
Wabbajack.Test/MO2Tests.cs
View File

@ -80,7 +80,7 @@ namespace Wabbajack.Test
var downloadsFolder = Path.Combine(tempDir.Dir.FullName, "downloads");
Directory.CreateDirectory(downloadsFolder);
File.Create(Path.Combine(tempDir.Dir.FullName, $"downloads/someFile.txt"));
Assert.IsFalse(MO2Installer.CheckValidInstallPath(tempDir.Dir.FullName, downloadFolder: downloadsFolder).Succeeded);
Assert.IsTrue(MO2Installer.CheckValidInstallPath(tempDir.Dir.FullName, downloadFolder: downloadsFolder).Succeeded);
}
}
#endregion

220
Wabbajack.Test/PMapTests.cs Normal file
View File

@ -0,0 +1,220 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common;
namespace Wabbajack.Test
{
[TestClass]
public class PMapTests
{
const int TypicalThreadCount = 2;
const int TypicalDelayMS = 50;
[TestMethod]
public async Task Typical_Action()
{
using (var queue = new WorkQueue(TypicalThreadCount))
{
var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
var output = new List<int>();
await Enumerable.Range(0, TypicalThreadCount * 2)
.PMap(queue, (item) =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
Thread.Sleep(TypicalDelayMS);
lock (output)
{
output.Add(item);
}
});
Assert.IsTrue(input.SequenceEqual(output.OrderBy(i => i)));
}
}
[TestMethod]
public async Task Typical_Func()
{
using (var queue = new WorkQueue(TypicalThreadCount))
{
var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
var results = await Enumerable.Range(0, TypicalThreadCount * 2)
.PMap(queue, (item) =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
Thread.Sleep(TypicalDelayMS);
return item.ToString();
});
Assert.IsTrue(input.Select(i => i.ToString()).SequenceEqual(results));
}
}
[TestMethod]
public async Task Typical_Task()
{
using (var queue = new WorkQueue(TypicalThreadCount))
{
var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
var output = new List<int>();
await Enumerable.Range(0, TypicalThreadCount * 2)
.PMap(queue, async (item) =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
await Task.Delay(TypicalDelayMS);
lock (output)
{
output.Add(item);
}
});
Assert.IsTrue(input.SequenceEqual(output.OrderBy(i => i)));
}
}
[TestMethod]
public async Task Typical_TaskReturn()
{
using (var queue = new WorkQueue(TypicalThreadCount))
{
var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
var results = await Enumerable.Range(0, TypicalThreadCount * 2)
.PMap(queue, async (item) =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
await Task.Delay(TypicalDelayMS);
return item.ToString();
});
Assert.IsTrue(input.Select(i => i.ToString()).SequenceEqual(results));
}
}
[TestMethod]
public async Task NestedAction()
{
using (var queue = new WorkQueue(TypicalThreadCount))
{
var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
var inputConstructedResults = input.SelectMany(i => Enumerable.Range(i * 100, TypicalThreadCount * 2));
var output = new List<int>();
await Enumerable.Range(0, TypicalThreadCount * 2)
.PMap(queue, async (item) =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
await Enumerable.Range(item * 100, TypicalThreadCount * 2)
.PMap(queue, async (subItem) =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
Thread.Sleep(TypicalDelayMS);
lock (output)
{
output.Add(subItem);
}
});
});
Assert.IsTrue(inputConstructedResults.SequenceEqual(output.OrderBy(i => i)));
}
}
[TestMethod]
public async Task Nested_Func()
{
using (var queue = new WorkQueue(TypicalThreadCount))
{
var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
var inputConstructedResults = input.SelectMany(i => Enumerable.Range(i * 100, TypicalThreadCount * 2));
var results = await Enumerable.Range(0, TypicalThreadCount * 2)
.PMap(queue, async (item) =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
return await Enumerable.Range(item * 100, TypicalThreadCount * 2)
.PMap(queue, (subItem) =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
Thread.Sleep(TypicalDelayMS);
return subItem;
});
});
Assert.IsTrue(inputConstructedResults.SequenceEqual(results.SelectMany(i => i)));
}
}
[TestMethod]
public async Task Nested_Task()
{
using (var queue = new WorkQueue(TypicalThreadCount))
{
var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
var inputConstructedResults = input.SelectMany(i => Enumerable.Range(i * 100, TypicalThreadCount * 2));
var output = new List<int>();
await Enumerable.Range(0, TypicalThreadCount * 2)
.PMap(queue, async (item) =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
await Enumerable.Range(item * 100, TypicalThreadCount * 2)
.PMap(queue, async (subItem) =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
await Task.Delay(TypicalDelayMS);
lock (output)
{
output.Add(subItem);
}
});
});
Assert.IsTrue(inputConstructedResults.SequenceEqual(output.OrderBy(i => i)));
}
}
[TestMethod]
public async Task Nested_TaskReturn()
{
using (var queue = new WorkQueue(TypicalThreadCount))
{
var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
var inputConstructedResults = input.SelectMany(i => Enumerable.Range(i * 100, TypicalThreadCount * 2));
var results = await Enumerable.Range(0, TypicalThreadCount * 2)
.PMap(queue, async (item) =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
return await Enumerable.Range(item * 100, TypicalThreadCount * 2)
.PMap(queue, async (subItem) =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
await Task.Delay(TypicalDelayMS);
return subItem;
});
});
Assert.IsTrue(inputConstructedResults.SequenceEqual(results.SelectMany(i => i)));
}
}
[TestMethod]
public async Task Nested_BackgroundThreadsInvolved()
{
using (var queue = new WorkQueue(TypicalThreadCount))
{
var input = Enumerable.Range(0, TypicalThreadCount * 2).ToArray();
var inputConstructedResults = input.SelectMany(i => Enumerable.Range(i * 100, TypicalThreadCount * 2));
var results = await Enumerable.Range(0, TypicalThreadCount * 2)
.PMap(queue, async (item) =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
return await Enumerable.Range(item * 100, TypicalThreadCount * 2)
.PMap(queue, async (subItem) =>
{
return await Task.Run(async () =>
{
Assert.IsTrue(WorkQueue.WorkerThread);
await Task.Delay(TypicalDelayMS);
return subItem;
});
});
});
Assert.IsTrue(inputConstructedResults.SequenceEqual(results.SelectMany(i => i)));
}
}
}
}

4
Wabbajack.Test/SanityTests.cs
View File

@ -128,8 +128,8 @@ namespace Wabbajack.Test
// Update the file and verify that it throws an error.
utils.GenerateRandomFileData(game_file, 20);
var exception = Assert.ThrowsException<AggregateException>(() => Install(compiler));
Assert.IsInstanceOfType(exception.InnerExceptions.First(), typeof(InvalidGameESMError));
var exception = await Assert.ThrowsExceptionAsync<InvalidGameESMError>(async () => await Install(compiler));
Assert.IsInstanceOfType(exception, typeof(InvalidGameESMError));
}
[TestMethod]

1
Wabbajack.Test/Wabbajack.Test.csproj
View File

@ -110,6 +110,7 @@
<Compile Include="MiscTests.cs" />
<Compile Include="MO2Tests.cs" />
<Compile Include="ModlistMetadataTests.cs" />
<Compile Include="PMapTests.cs" />
<Compile Include="RestartingDownloadsTests.cs" />
<Compile Include="SimpleHTTPServer.cs" />
<Compile Include="TestUtils.cs" />

1
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -98,6 +98,7 @@ namespace Wabbajack.VirtualFileSystem
var allFiles = await filesToIndex
.PMap(Queue, async f =>
{
Utils.Status($"Indexing {Path.GetFileName(f)}");
if (byPath.TryGetValue(f, out var found))
{
var fi = new FileInfo(f);