Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into wabbajack-lib-nullability

Browse Source
This commit is contained in:
Justin Swanson 2020-04-12 00:40:49 -05:00
commit 01588dbcc9
44 changed files with 540 additions and 476 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Wabbajack.BuildServer.Test/BasicServerTests.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Controllers;
using Wabbajack.Common;
using Xunit;
using Xunit.Abstractions;
@ -15,8 +16,8 @@ namespace Wabbajack.BuildServer.Test
[Fact]
public async Task CanGetHeartbeat()
{
var heartbeat = (await _client.GetStringAsync(MakeURL("heartbeat"))).FromJsonString<string>();
Assert.True(TimeSpan.Parse(heartbeat) > TimeSpan.Zero);
var heartbeat = (await _client.GetStringAsync(MakeURL("heartbeat"))).FromJsonString<Heartbeat.HeartbeatResult>();
Assert.True(heartbeat.Uptime > TimeSpan.Zero);
}
[Fact]

7
Wabbajack.BuildServer.Test/JobQueueTests.cs
View File

@ -23,11 +23,14 @@ namespace Wabbajack.BuildServer.Test
var found = await sqlService.GetJob();
Assert.NotNull(found);
Assert.IsAssignableFrom<GetNexusUpdatesJob>(found.Payload);
found.Result = JobResult.Success();
await sqlService.FinishJob(found);
}
[Fact]
public async Task PriorityMatters()
{
await ClearJobQueue();
var sqlService = Fixture.GetService<SqlService>();
var priority = new List<Job.JobPriority>
{
@ -41,6 +44,10 @@ namespace Wabbajack.BuildServer.Test
var found = await sqlService.GetJob();
Assert.NotNull(found);
Assert.Equal(pri, found.Priority);
found.Result = JobResult.Success();
// Finish the job so the next can run
await sqlService.FinishJob(found);
}
}

61
Wabbajack.BuildServer.Test/JobTests.cs Normal file
View File

@ -0,0 +1,61 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class JobTests : ABuildServerSystemTest
{
public JobTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanRunNexusUpdateJob()
{
var sql = Fixture.GetService<SqlService>();
var oldRecords = await NexusUpdatesFeeds.GetUpdates();
foreach (var record in oldRecords)
{
await sql.AddNexusModInfo(record.Game, record.ModId, DateTime.UtcNow - TimeSpan.FromDays(1),
new ModInfo());
await sql.AddNexusModFiles(record.Game, record.ModId, DateTime.UtcNow - TimeSpan.FromDays(1),
new NexusApiClient.GetModFilesResponse());
Assert.NotNull(await sql.GetModFiles(record.Game, record.ModId));
Assert.NotNull(await sql.GetNexusModInfoString(record.Game, record.ModId));
}
Utils.Log($"Ingested {oldRecords.Count()} nexus records");
// We know this will load the same records as above, but the date will be more recent, so the above records
// should no longer exist in SQL after this job is run
await sql.EnqueueJob(new Job {Payload = new GetNexusUpdatesJob()});
await RunAllJobs();
foreach (var record in oldRecords)
{
Assert.Null(await sql.GetModFiles(record.Game, record.ModId));
Assert.Null(await sql.GetNexusModInfoString(record.Game, record.ModId));
}
}
[Fact]
public async Task CanPrimeTheNexusCache()
{
var sql = Fixture.GetService<SqlService>();
Assert.True(await GetNexusUpdatesJob.UpdateNexusCacheFast(sql) > 0);
Assert.True(await GetNexusUpdatesJob.UpdateNexusCacheFast(sql) == 0);
}
}
}

5
Wabbajack.BuildServer.Test/ModListValidationTests.cs
View File

@ -93,8 +93,9 @@ namespace Wabbajack.BuildServer.Test
Assert.IsType<UpdateModLists>(job.Payload);
var jobResult = await job.Payload.Execute(sql, settings);
Assert.Equal(JobResultType.Success, jobResult.ResultType);
job.Result = await job.Payload.Execute(sql, settings);
await sql.FinishJob(job);
Assert.Equal(JobResultType.Success, job.Result.ResultType);
}
private async Task CheckListFeeds(int failed, int passed)

1
Wabbajack.BuildServer.Test/sql/wabbajack_db.sql
View File

@ -204,6 +204,7 @@ GO
CREATE TABLE [dbo].[Jobs](
[Id] [bigint] IDENTITY(1,1) NOT NULL,
[Priority] [int] NOT NULL,
[PrimaryKeyString] [nvarchar](max) NULL,
[Started] [datetime] NULL,
[Ended] [datetime] NULL,
[Created] [datetime] NOT NULL,

18
Wabbajack.BuildServer/Controllers/Heartbeat.cs
View File

@ -4,7 +4,10 @@ using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Org.BouncyCastle.Asn1.Cms;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Common.StatusFeed;
namespace Wabbajack.BuildServer.Controllers
@ -36,9 +39,20 @@ namespace Wabbajack.BuildServer.Controllers
}
[HttpGet]
public async Task<TimeSpan> GetHeartbeat()
public async Task<IActionResult> GetHeartbeat()
{
return DateTime.Now - _startTime;
return Ok(new HeartbeatResult
{
Uptime = DateTime.Now - _startTime,
LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync
});
}
[JsonName("HeartbeatResult")]
public class HeartbeatResult
{
public TimeSpan Uptime { get; set; }
public TimeSpan LastNexusUpdate { get; set; }
}
[HttpGet("only-authenticated")]

12
Wabbajack.BuildServer/Controllers/NexusCache.cs
View File

@ -49,10 +49,10 @@ namespace Wabbajack.BuildServer.Controllers
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{ModId}.json";
var body = await api.Get<ModInfo>(path);
await SQL.AddNexusModInfo(game, ModId, DateTime.Now, body);
result = await api.GetModInfo(game, ModId, false);
await SQL.AddNexusModInfo(game, ModId, DateTime.UtcNow, result);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);
}
@ -78,10 +78,8 @@ namespace Wabbajack.BuildServer.Controllers
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files.json";
var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
await SQL.AddNexusModFiles(game, ModId, DateTime.Now, body);
result = await api.GetModFiles(game, ModId, false);
await SQL.AddNexusModFiles(game, ModId, DateTime.UtcNow, result);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);

28
Wabbajack.BuildServer/GOLIVE.md Normal file
View File

@ -0,0 +1,28 @@
Deployment Plan for 2.0 go-live
1. Release 2.0 to authors and let them rebuild their lists
1. Save old configs so the don't get overwritten
1. Backup SQL server data
1. Update SQL Tables
1. Nexus Mod Files
1. Nexus Mod Infos
1. Job Queue
1. Api Keys
1. Mod Lists
1. Download States
1. Uploaded Files
1. Export Download Inis from server
1. Export all cache files from server
1. Hand insert all API keys
1. Copy over new server binaries
1. Disable background jobs on server
1. Start new server
1. Load data
1. Import downloaded Inis
1. Import all cache files
1. Stop server
1. Enable backend jobs
1. Start server
1. Verify that list validation triggers
1. ???
1. Profit?

13
Wabbajack.BuildServer/JobManager.cs
View File

@ -10,6 +10,7 @@ using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer
{
@ -74,6 +75,9 @@ namespace Wabbajack.BuildServer
Utils.LogMessages.Subscribe(Heartbeat.AddToLog);
Utils.LogMessages.OfType<IUserIntervention>().Subscribe(u => u.Cancel());
if (!Settings.JobScheduler) return;
var task = RunNexusCacheLoop();
while (true)
{
await KillOrphanedJobs();
@ -86,6 +90,15 @@ namespace Wabbajack.BuildServer
}
}
private async Task RunNexusCacheLoop()
{
while (true)
{
await GetNexusUpdatesJob.UpdateNexusCacheFast(Sql);
await Task.Delay(TimeSpan.FromMinutes(1));
}
}
private async Task KillOrphanedJobs()
{
try

4
Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs
View File

@ -30,6 +30,10 @@ namespace Wabbajack.BuildServer.Models.JobQueue
public virtual bool UsesNexus { get; } = false;
public abstract Task<JobResult> Execute(SqlService sql,AppSettings settings);
protected abstract IEnumerable<object> PrimaryKey { get; }
public string PrimaryKeyString => string.Join("|", PrimaryKey.Cons(this.GetType().Name).Select(i => i.ToString()));
static AJobPayload()
{

3
Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using System.Linq;
using Wabbajack.BuildServer.Model.Models;
@ -38,6 +39,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => new object[0];
private static async Task EnqueueFromList(SqlService sql, ModlistMetadata list, WorkQueue queue)
{
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);

5
Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs
View File

@ -1,4 +1,5 @@
using System.Linq;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
@ -63,5 +64,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
}
protected override IEnumerable<object> PrimaryKey => new object[0];
}
}

42
Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Models.JobQueue;
@ -69,6 +70,45 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => new object[0];
public static DateTime LastNexusSync { get; set; } = DateTime.Now;
public static async Task<long> UpdateNexusCacheFast(SqlService sql)
{
var results = await NexusUpdatesFeeds.GetUpdates();
NexusApiClient client = null;
long updated = 0;
foreach (var result in results)
{
var purgedMods = await sql.DeleteNexusModFilesUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
var purgedFiles = await sql.DeleteNexusModInfosUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
var totalPurged = purgedFiles + purgedMods;
if (totalPurged > 0)
Utils.Log($"Purged {totalPurged} cache items");
if (await sql.GetNexusModInfoString(result.Game, result.ModId) != null) continue;
// Lazily create the client
client ??= await NexusApiClient.Get();
// Cache the info
var files = await client.GetModFiles(result.Game, result.ModId, false);
await sql.AddNexusModFiles(result.Game, result.ModId, result.TimeStamp, files);
var modInfo = await client.GetModInfo(result.Game, result.ModId);
await sql.AddNexusModInfo(result.Game, result.ModId, result.TimeStamp, modInfo);
updated++;
}
if (updated > 0)
Utils.Log($"Primed {updated} nexus cache entries");
LastNexusSync = DateTime.Now;
return updated;
}
}

3
Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
@ -60,5 +61,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => new object[0];
}
}

1
Wabbajack.BuildServer/Models/Jobs/IndexJob.cs
View File

@ -63,6 +63,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => Archive.State.PrimaryKey;
}
}

2
Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs
View File

@ -69,5 +69,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
}
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => new object[0];
}
}

9
Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Model.Models;
@ -43,8 +44,10 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
private async Task ValidateList(SqlService sql, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
protected override IEnumerable<object> PrimaryKey => new object[0];
private async Task ValidateList(SqlService sql, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
{
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
@ -68,7 +71,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
Utils.Log($"{installer.Archives.Count} archives to validate");
DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
await DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
var validated = (await installer.Archives

3
Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
@ -68,6 +69,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => new object[] {FileId};
public class Progress : IProgress<FluentFTP.FtpProgress>
{
private RelativePath _name;

3
Wabbajack.BuildServer/Models/PatchArchive.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
@ -74,6 +75,8 @@ namespace Wabbajack.BuildServer.Models
}
protected override IEnumerable<object> PrimaryKey => new object[] {Src, DestPK};
public static AbsolutePath CdnPath(Hash srcHash, Hash destHash)
{
return $"updates/{srcHash.ToHex()}_{destHash.ToHex()}".RelativeTo(AbsolutePath.EntryPoint);

15
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -181,9 +181,10 @@ namespace Wabbajack.BuildServer.Model.Models
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"INSERT INTO dbo.Jobs (Created, Priority, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @Payload, @OnSuccess)",
@"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)",
new {
job.Priority,
PrimaryKeyString = job.Payload.PrimaryKeyString,
Payload = job.Payload.ToJson(),
OnSuccess = job.OnSuccess?.ToJson() ?? null});
}
@ -217,7 +218,11 @@ namespace Wabbajack.BuildServer.Model.Models
{
await using var conn = await Open();
var result = await conn.QueryAsync<Job>(
@"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy WHERE ID in (SELECT TOP(1) ID FROM Jobs WHERE Started is NULL ORDER BY Priority DESC, Created);
@"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy
WHERE ID in (SELECT TOP(1) ID FROM Jobs
WHERE Started is NULL
AND PrimaryKeyString NOT IN (SELECT PrimaryKeyString from jobs WHERE Started IS NOT NULL and Ended IS NULL)
ORDER BY Priority DESC, Created);
SELECT TOP(1) * FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC",
new {RunBy = Guid.NewGuid().ToString()});
return result.FirstOrDefault();
@ -573,7 +578,7 @@ namespace Wabbajack.BuildServer.Model.Models
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date
@"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
return deleted;
@ -583,9 +588,9 @@ namespace Wabbajack.BuildServer.Model.Models
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date
@"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
return deleted;
}

2
Wabbajack.CLI/Verbs/DownloadUrl.cs
View File

@ -24,7 +24,7 @@ namespace Wabbajack.CLI.Verbs
if (state == null)
return CLIUtils.Exit($"Could not find download source for URL {Url}", ExitCode.Error);
DownloadDispatcher.PrepareAll(new []{state});
await DownloadDispatcher.PrepareAll(new []{state});
using var queue = new WorkQueue();
queue.Status

47
Wabbajack.Common/Json.cs
View File

@ -27,11 +27,14 @@ namespace Wabbajack.Common
};
public static JsonSerializerSettings JsonSettings =>
new JsonSerializerSettings {
TypeNameHandling = TypeNameHandling.Objects,
SerializationBinder = new JsonNameSerializationBinder(),
Converters = Converters};
new JsonSerializerSettings {
TypeNameHandling = TypeNameHandling.Objects,
SerializationBinder = new JsonNameSerializationBinder(),
Converters = Converters};
public static JsonSerializerSettings GenericJsonSettings =>
new JsonSerializerSettings { };
public static void ToJson<T>(this T obj, string filename)
{
@ -69,12 +72,15 @@ namespace Wabbajack.Common
return JsonConvert.DeserializeObject<T>(data, JsonSettings)!;
}
public static T FromJson<T>(this Stream stream)
public static T FromJson<T>(this Stream stream, bool genericReader = false)
{
using var tr = new StreamReader(stream, Encoding.UTF8, leaveOpen: true);
using var reader = new JsonTextReader(tr);
var ser = JsonSerializer.Create(JsonSettings);
return ser.Deserialize<T>(reader)!;
var ser = JsonSerializer.Create(genericReader ? GenericJsonSettings : JsonSettings);
var result = ser.Deserialize<T>(reader);
if (result == null)
throw new JsonException("Type deserialized into null");
return result;
}
private class HashJsonConverter : JsonConverter<Hash>
@ -232,7 +238,7 @@ namespace Wabbajack.Common
public class JsonNameSerializationBinder : ISerializationBinder
public class JsonNameSerializationBinder : DefaultSerializationBinder
{
private static Dictionary<string, Type> _nameToType = new Dictionary<string, Type>();
private static Dictionary<Type, string> _typeToName = new Dictionary<Type, string>();
@ -274,7 +280,7 @@ namespace Wabbajack.Common
}
public Type BindToType(string? assemblyName, string typeName)
public override Type BindToType(string? assemblyName, string typeName)
{
if (typeName.EndsWith("[]"))
{
@ -289,22 +295,17 @@ namespace Wabbajack.Common
if (val != null)
return val;
if (assemblyName != null)
{
var assembly = AppDomain.CurrentDomain.Load(assemblyName);
if (assembly != null)
{
var result = assembly.GetType(typeName);
if (result != null) return result;
}
}
throw new InvalidDataException($"No Binding name for {typeName}");
return base.BindToType(assemblyName, typeName);
}
public void BindToName(Type serializedType, out string? assemblyName, out string? typeName)
public override void BindToName(Type serializedType, out string? assemblyName, out string? typeName)
{
if (serializedType.FullName?.StartsWith("System.") ?? false)
{
base.BindToName(serializedType, out assemblyName, out typeName);
return;
}
if (!_typeToName.ContainsKey(serializedType))
{
throw new InvalidDataException($"No Binding name for {serializedType}");

9
Wabbajack.Common/Paths.cs
View File

@ -145,13 +145,16 @@ namespace Wabbajack.Common
}
}
/// <summary>
/// Returns the full path the folder that contains Wabbajack.Common. This will almost always be
/// where all the binaries for the project reside.
/// </summary>
/// <exception cref="ArgumentException"></exception>
public static AbsolutePath EntryPoint
{
get
{
var location = Assembly.GetEntryAssembly()?.Location ?? null;
if (location == null)
location = Assembly.GetExecutingAssembly().Location ?? null;
var location = Assembly.GetExecutingAssembly().Location ?? null;
if (location == null)
throw new ArgumentException("Could not find entry point.");
return ((AbsolutePath)location).Parent;

26
Wabbajack.Common/ProcessHelper.cs
View File

@ -16,24 +16,35 @@ namespace Wabbajack.Common
Error,
}
public string Path { get; set; } = string.Empty;
public AbsolutePath Path { get; set; }
public IEnumerable<object> Arguments { get; set; } = Enumerable.Empty<object>();
public bool LogError { get; set; } = true;
public readonly Subject<(StreamType Type, string Line)> Output = new Subject<(StreamType Type, string)>();
public readonly Subject<(StreamType Type, string Line)> Output = new Subject<(StreamType Type, string)>();
public bool ThrowOnNonZeroExitCode { get; set; } = false;
public ProcessHelper()
{
}
public async Task<int> Start()
{
var args = Arguments.Select(arg =>
{
return arg switch
{
AbsolutePath abs => $"\"{abs}\"",
RelativePath rel => $"\"{rel}\"",
_ => arg.ToString()
};
});
var info = new ProcessStartInfo
{
FileName = (string)Path,
Arguments = string.Join(" ", Arguments),
Arguments = string.Join(" ", args),
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
@ -65,7 +76,7 @@ namespace Wabbajack.Common
if (string.IsNullOrEmpty(data.Data)) return;
Output.OnNext((StreamType.Error, data.Data));
if (LogError)
Utils.Log($"{AlphaPath.GetFileName(Path)} ({p.Id}) StdErr: {data.Data}");
Utils.Log($"{Path.FileName} ({p.Id}) StdErr: {data.Data}");
};
p.ErrorDataReceived += ErrorEventHandler;
@ -92,6 +103,9 @@ namespace Wabbajack.Common
p.Exited -= Exited;
Output.OnCompleted();
if (result != 0 && ThrowOnNonZeroExitCode)
throw new Exception($"Error executing {Path} - Exit Code {result} - Check the log for more information");
return result;
}

5
Wabbajack.Common/Utils.cs
View File

@ -517,7 +517,6 @@ namespace Wabbajack.Common
return await Task.WhenAll(tasks);
}
public static async Task<TR[]> PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue,
Func<TI, Task<TR>> f)
{
@ -956,7 +955,7 @@ namespace Wabbajack.Common
{
var process = new ProcessHelper
{
Path = "cmd.exe",
Path = ((RelativePath)"cmd.exe").RelativeToSystemDirectory(),
Arguments = new object[] {"/c", "del", "/f", "/q", "/s", $"\"{(string)path}\"", "&&", "rmdir", "/q", "/s", $"\"{(string)path}\""},
};
var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
@ -999,7 +998,7 @@ namespace Wabbajack.Common
var encoded = ProtectedData.Protect(bytes, Encoding.UTF8.GetBytes(key), DataProtectionScope.LocalMachine);
Consts.LocalAppDataPath.CreateDirectory();
Consts.LocalAppDataPath.Combine(key).WriteAllBytes(bytes);
Consts.LocalAppDataPath.Combine(key).WriteAllBytes(encoded);
}
public static byte[] FromEncryptedData(string key)
{

2
Wabbajack.Lib/AInstaller.cs
View File

@ -181,7 +181,7 @@ namespace Wabbajack.Lib
});
Status("Unstaging files");
onFinish();
await onFinish();
// Now patch all the files from this archive
await grouping.OfType<PatchedFromArchive>()

4
Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs
View File

@ -106,8 +106,9 @@ namespace Wabbajack.Lib.Downloaders
result.ToEcryptedJson(DataName);
return result;
}
catch (Exception)
catch (Exception ex)
{
Utils.Error(ex, "Could not save Bethesda.NET login info");
return null;
}
}
@ -369,6 +370,7 @@ namespace Wabbajack.Lib.Downloaders
}
[JsonName("BethesdaNetData")]
public class BethesdaNetData
{
public string body { get; set; } = string.Empty;

7
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -58,7 +58,6 @@ namespace Wabbajack.Lib.Downloaders
public static T GetInstance<T>() where T : IDownloader
{
var inst = (T)IndexedDownloaders[typeof(T)];
inst.Prepare();
return inst;
}
@ -79,11 +78,11 @@ namespace Wabbajack.Lib.Downloaders
return Downloaders.OfType<IUrlDownloader>().Select(d => d.GetDownloaderState(url)).FirstOrDefault(result => result != null);
}
public static void PrepareAll(IEnumerable<AbstractDownloadState> states)
public static async Task PrepareAll(IEnumerable<AbstractDownloadState> states)
{
states.Select(s => s.GetDownloader().GetType())
await Task.WhenAll(states.Select(s => s.GetDownloader().GetType())
.Distinct()
.Do(t => Downloaders.First(d => d.GetType() == t).Prepare());
.Select(t => Downloaders.First(d => d.GetType() == t).Prepare()));
}
public static async Task<bool> DownloadWithPossibleUpgrade(Archive archive, AbsolutePath destination)

82
Wabbajack.Lib/Downloaders/YouTubeDownloader.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Diagnostics;
using System.IO.Compression;
using System.Linq;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
@ -149,71 +150,46 @@ namespace Wabbajack.Lib.Downloaders
}
private const string FFMpegPath = "Downloaders/Converters/ffmpeg.exe";
private const string xWMAEncodePath = "Downloaders/Converters/xWMAEncode.exe";
private async Task ExtractTrack(AbsolutePath source, AbsolutePath dest_folder, Track track)
private AbsolutePath FFMpegPath => "Downloaders/Converters/ffmpeg.exe".RelativeTo(AbsolutePath.EntryPoint);
private AbsolutePath xWMAEncodePath = "Downloaders/Converters/xWMAEncode.exe".RelativeTo(AbsolutePath.EntryPoint);
private Extension WAVExtension = new Extension(".wav");
private Extension XWMExtension = new Extension(".xwm");
private async Task ExtractTrack(AbsolutePath source, AbsolutePath destFolder, Track track)
{
var info = new ProcessStartInfo
var process = new ProcessHelper
{
FileName = FFMpegPath,
Arguments =
$"-threads 1 -i \"{source}\" -ss {track.Start} -t {track.End - track.Start} \"{dest_folder}\\{track.Name}.wav\"",
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
Path = FFMpegPath,
Arguments = new object[] {"-threads", 1, "-i", source, "-ss", track.Start, "-t", track.End - track.Start, track.Name.RelativeTo(destFolder).WithExtension(WAVExtension)},
ThrowOnNonZeroExitCode = true
};
var ffmpegLogs = process.Output.Where(arg => arg.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(val =>
{
Utils.Status($"Extracting {track.Name} - {val.Line}");
});
var p = new Process {StartInfo = info};
p.Start();
ChildProcessTracker.AddProcess(p);
await p.StandardError.ReadToEndAsync();
try
{
p.PriorityClass = ProcessPriorityClass.BelowNormal;
}
catch (Exception e)
{
Utils.Error(e, "Error while setting process priority level for ffmpeg.exe");
}
p.WaitForExit();
await process.Start();
if (track.Format == Track.FormatEnum.WAV) return;
info = new ProcessStartInfo
process = new ProcessHelper()
{
FileName = xWMAEncodePath,
Arguments =
$"-b 192000 \"{dest_folder}\\{track.Name}.wav\" \"{dest_folder}\\{track.Name}.xwm\"",
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
Path = xWMAEncodePath,
Arguments = new object[] {"-b", 192000, track.Name.RelativeTo(destFolder).WithExtension(WAVExtension), track.Name.RelativeTo(destFolder).WithExtension(XWMExtension)},
ThrowOnNonZeroExitCode = true
};
p = new Process {StartInfo = info};
var xwmLogs = process.Output.Where(arg => arg.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(val =>
{
Utils.Status($"Encoding {track.Name} - {val.Line}");
});
p.Start();
ChildProcessTracker.AddProcess(p);
var output2 = await p.StandardError.ReadToEndAsync();
try
{
p.PriorityClass = ProcessPriorityClass.BelowNormal;
}
catch (Exception e)
{
Utils.Error(e, "Error while setting process priority level for ffmpeg.exe");
}
p.WaitForExit();
await process.Start();
if (File.Exists($"{dest_folder}\\{track.Name}.wav"))
File.Delete($"{dest_folder}\\{track.Name}.wav");
if (File.Exists($"{destFolder}\\{track.Name}.wav"))
File.Delete($"{destFolder}\\{track.Name}.wav");
}

2
Wabbajack.Lib/MO2Compiler.cs
View File

@ -447,7 +447,7 @@ namespace Wabbajack.Lib
private async Task BuildArchivePatches(Hash archiveSha, IEnumerable<PatchedFromArchive> group,
Dictionary<RelativePath, AbsolutePath> absolutePaths)
{
using var files = await VFS.StageWith(@group.Select(g => VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath)));
await using var files = await VFS.StageWith(@group.Select(g => VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath)));
var byPath = files.GroupBy(f => string.Join("|", f.FilesInFullPath.Skip(1).Select(i => i.Name)))
.ToDictionary(f => f.Key, f => f.First());
// Now Create the patches

16
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -251,7 +251,7 @@ namespace Wabbajack.Lib.NexusApi
await using var stream = await response.Content.ReadAsStreamAsync();
return stream.FromJson<T>();
return stream.FromJson<T>(genericReader:true);
}
catch (TimeoutException)
{
@ -321,10 +321,11 @@ namespace Wabbajack.Lib.NexusApi
public List<NexusFileInfo> files { get; set; } = new List<NexusFileInfo>();
}
public async Task<GetModFilesResponse> GetModFiles(Game game, long modid)
public async Task<GetModFilesResponse> GetModFiles(Game game, long modid, bool useCache = true)
{
var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modid}/files.json";
var result = await GetCached<GetModFilesResponse>(url);
var result = useCache ? await GetCached<GetModFilesResponse>(url) : await Get<GetModFilesResponse>(url);
if (result.files == null)
throw new InvalidOperationException("Got Null data from the Nexus while finding mod files");
return result;
@ -336,10 +337,15 @@ namespace Wabbajack.Lib.NexusApi
return await Get<List<MD5Response>>(url);
}
public async Task<ModInfo> GetModInfo(Game game, long modId)
public async Task<ModInfo> GetModInfo(Game game, long modId, bool useCache = true)
{
var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modId}.json";
return await GetCached<ModInfo>(url);
if (useCache)
{
return await GetCached<ModInfo>(url);
}
return await Get<ModInfo>(url);
}
private class DownloadLink

80
Wabbajack.Lib/NexusApi/NexusUpdatesFeeds.cs Normal file
View File

@ -0,0 +1,80 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.ServiceModel.Syndication;
using System.Threading.Tasks;
using System.Xml;
using Wabbajack.Common;
namespace Wabbajack.Lib.NexusApi
{
public class NexusUpdatesFeeds
{
public static async Task<List<UpdateRecord>> GetUpdates()
{
var updated = GetFeed(new Uri("https://www.nexusmods.com/rss/updatedtoday"));
var newToday = GetFeed(new Uri("https://www.nexusmods.com/rss/newtoday"));
var sorted = (await updated).Concat(await newToday).OrderByDescending(f => f.TimeStamp);
var deduped = sorted.GroupBy(g => (g.Game, g.ModId)).Select(g => g.First()).ToList();
return deduped;
}
private static bool TryParseGameUrl(SyndicationLink link, out Game game, out long modId)
{
var parts = link.Uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
var foundGame = GameRegistry.GetByFuzzyName(parts[0]);
if (foundGame == null)
{
game = Game.Oblivion;
modId = 0;
return false;
}
if (long.TryParse(parts[2], out modId))
{
game = foundGame.Game;
return true;
}
game = Game.Oblivion;
modId = 0;
return false;
}
private static async Task<IEnumerable<UpdateRecord>> GetFeed(Uri uri)
{
var client = new Common.Http.Client();
var data = await client.GetStringAsync(uri);
var reader = XmlReader.Create(new StringReader(data));
var results = SyndicationFeed.Load(reader);
return results.Items
.Select(itm =>
{
if (TryParseGameUrl(itm.Links.First(), out var game, out var modId))
{
return new UpdateRecord
{
TimeStamp = itm.PublishDate.UtcDateTime,
Game = game,
ModId = modId
};
}
return null;
}).Where(v => v != null);
}
public class UpdateRecord
{
public Game Game { get; set; }
public long ModId { get; set; }
public DateTime TimeStamp { get; set; }
}
}
}

3
Wabbajack.Lib/Wabbajack.Lib.csproj
View File

@ -52,6 +52,9 @@
<PackageReference Include="System.Net.Http">
<Version>4.3.4</Version>
</PackageReference>
<PackageReference Include="System.ServiceModel.Syndication">
<Version>4.7.0</Version>
</PackageReference>
<PackageReference Include="WebSocketSharp-netstandard">
<Version>1.0.1</Version>
</PackageReference>

2
Wabbajack.Test/ACompilerTest.cs
View File

@ -28,7 +28,7 @@ namespace Wabbajack.Test
public override void Dispose()
{
utils.Dispose();
utils.DisposeAsync().AsTask().Wait();
_unsub.Dispose();
base.Dispose();
}

4
Wabbajack.Test/DownloaderTests.cs
View File

@ -44,9 +44,9 @@ namespace Wabbajack.Test
}
[Fact]
public void TestAllPrepares()
public async Task TestAllPrepares()
{
DownloadDispatcher.Downloaders.Do(d => d.Prepare());
await Task.WhenAll(DownloadDispatcher.Downloaders.Select(d => d.Prepare()));
}
[Fact]

1
Wabbajack.Test/EndToEndTests.cs
View File

@ -39,6 +39,7 @@ namespace Wabbajack.Test
{
Queue.Dispose();
_unsub.Dispose();
utils.DisposeAsync().AsTask().Wait();
base.Dispose();
}

32
Wabbajack.Test/NexusTests.cs Normal file
View File

@ -0,0 +1,32 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Test
{
public class NexusTests : ATestBase
{
[Fact]
public async Task CanGetNexusRSSUpdates()
{
var results = (await NexusUpdatesFeeds.GetUpdates()).ToArray();
Assert.NotEmpty(results);
Utils.Log($"Loaded {results.Length} updates from the Nexus");
foreach (var result in results)
{
Assert.True(DateTime.UtcNow - result.TimeStamp < TimeSpan.FromDays(1));
}
}
public NexusTests(ITestOutputHelper output) : base(output)
{
}
}
}

13
Wabbajack.Test/TestUtils.cs
View File

@ -14,7 +14,7 @@ using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Test
{
public class TestUtils : IDisposable
public class TestUtils : IAsyncDisposable
{
private static Random _rng = new Random();
public TestUtils()
@ -119,13 +119,14 @@ namespace Wabbajack.Test
return arr;
}
public void Dispose()
public async ValueTask DisposeAsync()
{
var exts = new [] {".md", ".exe"};
WorkingDirectory.Combine(ID).DeleteDirectory();
var exts = new[] { ".md", ".exe" };
await WorkingDirectory.Combine(ID).DeleteDirectory();
Profiles.Do(p =>
{
foreach (var ext in exts) {
foreach (var ext in exts)
{
var path = Path.Combine(Directory.GetCurrentDirectory(), p + ext);
if (File.Exists(path))
File.Delete(path);
@ -246,7 +247,5 @@ namespace Wabbajack.Test
GenerateRandomFileData(fullPath, i);
return fullPath;
}
}
}

37
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -9,7 +9,7 @@ using Xunit.Abstractions;
namespace Wabbajack.VirtualFileSystem.Test
{
public class VFSTests
public class VFSTests : IAsyncLifetime
{
private static readonly AbsolutePath VFS_TEST_DIR = "vfs_test_dir".ToPath().RelativeToEntryPoint();
private static readonly AbsolutePath TEST_ZIP = "test.zip".RelativeTo(VFS_TEST_DIR);
@ -18,18 +18,26 @@ namespace Wabbajack.VirtualFileSystem.Test
private Context context;
private readonly ITestOutputHelper _helper;
private WorkQueue Queue { get; }
private WorkQueue Queue { get; } = new WorkQueue();
public VFSTests(ITestOutputHelper helper)
{
_helper = helper;
Utils.LogMessages.Subscribe(f => _helper.WriteLine(f.ShortDescription));
VFS_TEST_DIR.DeleteDirectory();
VFS_TEST_DIR.CreateDirectory();
Queue = new WorkQueue();
context = new Context(Queue);
}
public async Task InitializeAsync()
{
await VFS_TEST_DIR.DeleteDirectory();
VFS_TEST_DIR.CreateDirectory();
}
public async Task DisposeAsync()
{
await VFS_TEST_DIR.DeleteDirectory();
}
[Fact]
public async Task FilesAreIndexed()
{
@ -51,12 +59,11 @@ namespace Wabbajack.VirtualFileSystem.Test
await context.IntegrateFromFile( "vfs_cache.bin".RelativeTo(VFS_TEST_DIR));
}
[Fact]
public async Task ArchiveContentsAreIndexed()
{
await AddFile(ARCHIVE_TEST_TXT, "This is a test");
ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await AddTestRoot();
var absPath = "test.zip".RelativeTo(VFS_TEST_DIR);
@ -78,7 +85,7 @@ namespace Wabbajack.VirtualFileSystem.Test
public async Task DuplicateFileHashes()
{
await AddFile(ARCHIVE_TEST_TXT, "This is a test");
ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await AddFile(TEST_TXT, "This is a test");
await AddTestRoot();
@ -127,7 +134,7 @@ namespace Wabbajack.VirtualFileSystem.Test
public async Task CanStageSimpleArchives()
{
await AddFile(ARCHIVE_TEST_TXT, "This is a test");
ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await AddTestRoot();
var res = new FullPath(TEST_ZIP, new[] {(RelativePath)"test.txt"});
@ -136,19 +143,19 @@ namespace Wabbajack.VirtualFileSystem.Test
var cleanup = await context.Stage(new List<VirtualFile> {file});
Assert.Equal("This is a test", await file.StagedPath.ReadAllTextAsync());
cleanup();
await cleanup();
}
[Fact]
public async Task CanStageNestedArchives()
{
await AddFile(ARCHIVE_TEST_TXT, "This is a test");
ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
var inner_dir = @"archive\other\dir".RelativeTo(VFS_TEST_DIR);
inner_dir.CreateDirectory();
TEST_ZIP.MoveTo( @"archive\other\dir\nested.zip".RelativeTo(VFS_TEST_DIR));
ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await AddTestRoot();
@ -159,7 +166,7 @@ namespace Wabbajack.VirtualFileSystem.Test
foreach (var file in files)
Assert.Equal("This is a test", await file.StagedPath.ReadAllTextAsync());
cleanup();
await cleanup();
}
private static async Task AddFile(AbsolutePath filename, string text)
@ -168,10 +175,10 @@ namespace Wabbajack.VirtualFileSystem.Test
await filename.WriteAllTextAsync(text);
}
private static void ZipUpFolder(AbsolutePath folder, AbsolutePath output)
private static async Task ZipUpFolder(AbsolutePath folder, AbsolutePath output)
{
ZipFile.CreateFromDirectory((string)folder, (string)output);
folder.DeleteDirectory();
await folder.DeleteDirectory();
}
}
}

31
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -195,7 +195,7 @@ namespace Wabbajack.VirtualFileSystem
}
}
public async Task<Action> Stage(IEnumerable<VirtualFile> files)
public async Task<Func<Task>> Stage(IEnumerable<VirtualFile> files)
{
var grouped = files.SelectMany(f => f.FilesInFullPath)
.Distinct()
@ -215,18 +215,18 @@ namespace Wabbajack.VirtualFileSystem
file.StagedPath = file.RelativeName.RelativeTo(tmpPath);
}
return () =>
return async () =>
{
paths.Do(p =>
foreach (var p in paths)
{
p.DeleteDirectory();
});
await p.DeleteDirectory();
}
};
}
public async Task<DisposableList<VirtualFile>> StageWith(IEnumerable<VirtualFile> files)
public async Task<AsyncDisposableList<VirtualFile>> StageWith(IEnumerable<VirtualFile> files)
{
return new DisposableList<VirtualFile>(await Stage(files), files);
return new AsyncDisposableList<VirtualFile>(await Stage(files), files);
}
@ -275,7 +275,7 @@ namespace Wabbajack.VirtualFileSystem
_knownFiles = new List<HashRelativePath>();
}
#endregion
}
@ -294,6 +294,21 @@ namespace Wabbajack.VirtualFileSystem
}
}
public class AsyncDisposableList<T> : List<T>, IAsyncDisposable
{
private Func<Task> _unstage;
public AsyncDisposableList(Func<Task> unstage, IEnumerable<T> files) : base(files)
{
_unstage = unstage;
}
public async ValueTask DisposeAsync()
{
await _unstage();
}
}
public class IndexRoot
{
public static IndexRoot Empty = new IndexRoot();

207
Wabbajack.VirtualFileSystem/FileExtractor.cs
View File

@ -1,6 +1,7 @@
using System;
using System.Diagnostics;
using System.Linq;
using System.Reactive.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Compression.BSA;
@ -25,9 +26,9 @@ namespace Wabbajack.VirtualFileSystem
else if (source.Extension == Consts.OMOD)
ExtractAllWithOMOD(source, dest);
else if (source.Extension == Consts.EXE)
ExtractAllEXE(source, dest);
await ExtractAllExe(source, dest);
else
ExtractAllWith7Zip(source, dest);
await ExtractAllWith7Zip(source, dest);
}
catch (Exception ex)
{
@ -35,71 +36,40 @@ namespace Wabbajack.VirtualFileSystem
}
}
private static void ExtractAllEXE(AbsolutePath source, AbsolutePath dest)
private static async Task ExtractAllExe(AbsolutePath source, AbsolutePath dest)
{
var isArchive = TestWith7z(source);
var isArchive = await TestWith7z(source);
if (isArchive)
{
ExtractAllWith7Zip(source, dest);
await ExtractAllWith7Zip(source, dest);
return;
}
Utils.Log($"Extracting {(string)source.FileName}");
var info = new ProcessStartInfo
var process = new ProcessHelper
{
FileName = @"Extractors\innounp.exe",
Arguments = $"-x -y -b -d\"{(string)dest}\" \"{(string)source}\"",
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
Path = @"Extractors\innounp.exe".RelativeTo(AbsolutePath.EntryPoint),
Arguments = new object[] {"-x", "-y", "-b", $"-d\"{dest}\"", source}
};
var p = new Process {StartInfo = info};
p.Start();
ChildProcessTracker.AddProcess(p);
try
{
p.PriorityClass = ProcessPriorityClass.BelowNormal;
}
catch (Exception e)
{
Utils.Error(e, "Error while setting process priority level for innounp.exe");
}
var name = source.FileName;
try
{
while (!p.HasExited)
var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p =>
{
var line = p.StandardOutput.ReadLine();
var (_, line) = p;
if (line == null)
break;
return;
if (line.Length <= 4 || line[3] != '%')
continue;
return;
int.TryParse(line.Substring(0, 3), out var percentInt);
Utils.Status($"Extracting {(string)name} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d));
}
}
catch (Exception e)
{
Utils.Error(e, "Error while reading StandardOutput for innounp.exe");
}
p.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Extracting {(string)name}");
if (p.ExitCode == 0)
return;
Utils.Log(p.StandardOutput.ReadToEnd());
Utils.Log($"Extraction error extracting {source}");
}
Utils.Status($"Extracting {source.FileName} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d));
});
await process.Start();
}
private class OMODProgress : ICodeProgress
{
@ -159,60 +129,42 @@ namespace Wabbajack.VirtualFileSystem
}
}
private static void ExtractAllWith7Zip(AbsolutePath source, AbsolutePath dest)
private static async Task ExtractAllWith7Zip(AbsolutePath source, AbsolutePath dest)
{
Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}", $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe"));
var info = new ProcessStartInfo
var process = new ProcessHelper
{
FileName = @"Extractors\7z.exe",
Arguments = $"x -bsp1 -y -o\"{(string)dest}\" \"{(string)source}\" -mmt=off",
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),
Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest}\"", source, "-mmt=off"}
};
var p = new Process {StartInfo = info};
p.Start();
ChildProcessTracker.AddProcess(p);
try
{
p.PriorityClass = ProcessPriorityClass.BelowNormal;
}
catch (Exception)
{
}
var name = source.FileName;
try
{
while (!p.HasExited)
var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p =>
{
var line = p.StandardOutput.ReadLine();
var (_, line) = p;
if (line == null)
break;
return;
if (line.Length <= 4 || line[3] != '%') continue;
if (line.Length <= 4 || line[3] != '%') return;
int.TryParse(line.Substring(0, 3), out var percentInt);
Utils.Status($"Extracting {(string)name} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d));
}
}
catch (Exception)
{
}
Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d));
});
p.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Extracting {name}");
var exitCode = await process.Start();
if (p.ExitCode == 0)
if (exitCode != 0)
{
Utils.Status($"Extracting {name} - 100%", Percent.One, alsoLog: true);
return;
Utils.Error(new _7zipReturnError(exitCode, source, dest, ""));
}
else
{
Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true);
}
Utils.Error(new _7zipReturnError(p.ExitCode, source, dest, p.StandardOutput.ReadToEnd()));
}
/// <summary>
@ -220,92 +172,35 @@ namespace Wabbajack.VirtualFileSystem
/// </summary>
/// <param name="v"></param>
/// <returns></returns>
public static bool CanExtract(AbsolutePath v)
public static async Task<bool> CanExtract(AbsolutePath v)
{
var ext = v.Extension;
if(ext != _exeExtension && !Consts.TestArchivesBeforeExtraction.Contains(ext))
return Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext);
var isArchive = TestWith7z(v);
var isArchive = await TestWith7z(v);
if (isArchive)
return true;
var info = new ProcessStartInfo
var process = new ProcessHelper
{
FileName = @"Extractors\innounp.exe",
Arguments = $"-t \"{v}\" ",
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
Path = @"Extractors\innounp.exe".RelativeTo(AbsolutePath.EntryPoint),
Arguments = new object[] {"-t", v},
};
var p = new Process {StartInfo = info};
p.Start();
ChildProcessTracker.AddProcess(p);
var name = v.FileName;
while (!p.HasExited)
{
var line = p.StandardOutput.ReadLine();
if (line == null)
break;
if (line[0] != '#')
continue;
Utils.Status($"Testing {(string)name} - {line.Trim()}");
}
p.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Testing {name}");
return p.ExitCode == 0;
return await process.Start() == 0;
}
public static bool TestWith7z(AbsolutePath file)
public static async Task<bool> TestWith7z(AbsolutePath file)
{
var testInfo = new ProcessStartInfo
var process = new ProcessHelper()
{
FileName = @"Extractors\7z.exe",
Arguments = $"t \"{file}\"",
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),
Arguments = new object[] {"t", file},
};
var testP = new Process {StartInfo = testInfo};
testP.Start();
ChildProcessTracker.AddProcess(testP);
try
{
testP.PriorityClass = ProcessPriorityClass.BelowNormal;
}
catch (Exception)
{
return false;
}
try
{
while (!testP.HasExited)
{
var line = testP.StandardOutput.ReadLine();
if (line == null)
break;
}
}
catch (Exception)
{
return false;
}
testP.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Can Extract Check {file}");
return testP.ExitCode == 0;
return await process.Start() == 0;
}
private static Extension _exeExtension = new Extension(".exe");

2
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -180,7 +180,7 @@ namespace Wabbajack.VirtualFileSystem
if (context.UseExtendedHashes)
self.ExtendedHashes = ExtendedHashes.FromFile(absPath);
if (FileExtractor.CanExtract(absPath))
if (await FileExtractor.CanExtract(absPath))
{
await using var tempFolder = Context.GetTemporaryFolder();
await FileExtractor.ExtractAll(context.Queue, absPath, tempFolder.FullName);

147
Wabbajack/Views/Compilers/VortexCompilerConfigView.xaml
View File

@ -1,147 +0,0 @@
<UserControl
x:Class="Wabbajack.VortexCompilerConfigView"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:local="clr-namespace:Wabbajack"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
d:DesignHeight="450"
d:DesignWidth="800"
mc:Ignorable="d">
<Grid>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="Auto" />
<ColumnDefinition Width="20" />
<ColumnDefinition Width="*" />
<ColumnDefinition Width="30" />
<ColumnDefinition Width="Auto" />
<ColumnDefinition Width="20" />
<ColumnDefinition Width="*" />
</Grid.ColumnDefinitions>
<Grid.RowDefinitions>
<RowDefinition Height="40" />
<RowDefinition Height="40" />
<RowDefinition Height="40" />
</Grid.RowDefinitions>
<TextBlock
Grid.Row="0"
Grid.Column="0"
HorizontalAlignment="Right"
VerticalAlignment="Center"
FontSize="14"
Text="Game"
TextAlignment="Center"
ToolTip="The game you wish to target" />
<ComboBox
Grid.Row="0"
Grid.Column="2"
Height="30"
VerticalAlignment="Center"
VerticalContentAlignment="Center"
FontSize="14"
ItemsSource="{Binding GameOptions}"
SelectedValue="{Binding SelectedGame}"
ToolTip="The game you wish to target">
<ComboBox.ItemTemplate>
<DataTemplate>
<TextBlock Margin="6,2" Text="{Binding DisplayName}" />
</DataTemplate>
</ComboBox.ItemTemplate>
</ComboBox>
<TextBlock
Grid.Row="1"
Grid.Column="0"
HorizontalAlignment="Right"
VerticalAlignment="Center"
FontSize="14"
Text="Game Folder"
TextAlignment="Center"
ToolTip="The install folder for the game" />
<local:FilePicker
Grid.Row="1"
Grid.Column="2"
Height="30"
VerticalAlignment="Center"
PickerVM="{Binding GameLocation}"
FontSize="14"
ToolTip="The install folder for the game" />
<Grid
Grid.Row="2"
Grid.Column="2"
Height="28"
HorizontalAlignment="Left"
VerticalAlignment="Top">
<Grid.ColumnDefinitions>
<ColumnDefinition Width="Auto" />
<ColumnDefinition Width="Auto" />
</Grid.ColumnDefinitions>
<Button
Grid.Column="0"
Margin="0,0,5,0"
Background="Transparent"
Command="{Binding FindGameInSteamCommand}"
Style="{StaticResource CircleButtonStyle}"
ToolTip="Attempt to locate the game in Steam">
<Image Margin="1" Source="../../Resources/Icons/steam.png" />
</Button>
<Button
Grid.Column="1"
Background="Transparent"
Command="{Binding FindGameInGogCommand}"
Style="{StaticResource CircleButtonStyle}"
ToolTip="Attempt to locate game in GoG">
<Image Margin="1" Source="../../Resources/Icons/gog.png" />
</Button>
</Grid>
<TextBlock
Grid.Row="0"
Grid.Column="4"
HorizontalAlignment="Right"
VerticalAlignment="Center"
FontSize="14"
Text="Download Location"
TextAlignment="Center"
ToolTip="The folder to download your mods" />
<local:FilePicker
Grid.Row="0"
Grid.Column="6"
Height="30"
VerticalAlignment="Center"
PickerVM="{Binding DownloadsLocation}"
FontSize="14"
ToolTip="The folder to download your mods" />
<TextBlock
Grid.Row="1"
Grid.Column="4"
HorizontalAlignment="Right"
VerticalAlignment="Center"
FontSize="14"
Text="Staging Location"
TextAlignment="Center" />
<local:FilePicker
Grid.Row="1"
Grid.Column="6"
Height="30"
VerticalAlignment="Center"
PickerVM="{Binding StagingLocation}"
FontSize="14" />
<TextBlock
Grid.Row="2"
Grid.Column="4"
HorizontalAlignment="Right"
VerticalAlignment="Center"
FontSize="14"
Text="Output Location"
TextAlignment="Center"
ToolTip="The folder to place the resulting modlist.wabbajack file" />
<local:FilePicker
Grid.Row="2"
Grid.Column="6"
Height="30"
VerticalAlignment="Center"
PickerVM="{Binding Parent.OutputLocation}"
FontSize="14"
ToolTip="The folder to place the resulting modlist.wabbajack file" />
</Grid>
</UserControl>

15
Wabbajack/Views/Compilers/VortexCompilerConfigView.xaml.cs
View File

@ -1,15 +0,0 @@
using System.Windows.Controls;
namespace Wabbajack
{
/// <summary>
/// Interaction logic for VortexCompilerConfigView.xaml
/// </summary>
public partial class VortexCompilerConfigView : UserControl
{
public VortexCompilerConfigView()
{
InitializeComponent();
}
}
}