Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Bug fixes encountered during the 2.0 release

Browse Source
This commit is contained in:
Timothy Baldridge 2020-04-25 23:13:42 -06:00
parent 803a6c9e03
commit 1327745115
12 changed files with 104 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Wabbajack.BuildServer.Test/MetricsTests.cs
View File

@ -30,5 +30,12 @@ namespace Wabbajack.BuildServer.Test
Assert.Equal(subject, result);
}
[Fact]
public async Task CanLoadMetricsFromSQL()
{
var sql = Fixture.GetService<SqlService>();
var results = await sql.MetricsReport("finish_install");
}
}
}

14
Wabbajack.BuildServer/BackendServices/ListIngest.cs
View File

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO.Compression;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.Common;
using Wabbajack.Lib;
@ -42,14 +43,23 @@ namespace Wabbajack.BuildServer.BackendServices
ModList modlist;
await using (var fs = modlistPath.OpenRead())
using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
await using (var entry = zip.GetEntry("modlist.json")?.Open())
await using (var entry = zip.GetEntry("modlist")?.Open())
{
if (entry == null)
{
Utils.Log($"Bad Modlist {list.Links.MachineURL}");
continue;
}
modlist = entry.FromJson<ModList>();
try
{
modlist = entry.FromJson<ModList>();
}
catch (JsonReaderException ex)
{
Utils.Log($"Bad JSON format for {list.Links.MachineURL}");
continue;
}
}
newData = true;

3
Wabbajack.BuildServer/Controllers/GraphQL.cs
View File

@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer.GraphQL;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
namespace Wabbajack.BuildServer.Controllers
{
@ -35,7 +36,7 @@ namespace Wabbajack.BuildServer.Controllers
if (result.Errors?.Count > 0)
{
return BadRequest();
return BadRequest(result.Errors);
}
return Ok(result);

25
Wabbajack.BuildServer/Controllers/IndexedFiles.cs
View File

@ -52,19 +52,25 @@ namespace Wabbajack.BuildServer.Controllers
var fullPath = folder.RelativeTo((AbsolutePath)_settings.TempFolder);
Utils.Log($"Ingesting Inis from {fullPath}");
int loadCount = 0;
foreach (var file in fullPath.EnumerateFiles().Where(f => f.Extension == Consts.IniExtension))
{
var loaded = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile(), true));
if (loaded == null)
using var queue = new WorkQueue();
await fullPath.EnumerateFiles().Where(f => f.Extension == Consts.IniExtension)
.PMap(queue, async file => {
try
{
Utils.Log($"Unsupported Ini {file}");
continue;
var loaded =
(AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile(), true));
var hash = Hash.FromHex(((string)file.FileNameWithoutExtension).Split("_").First());
await SQL.AddDownloadState(hash, loaded);
}
catch (Exception ex)
{
Utils.Log($"Failure for {file}");
}
var hash = Hash.FromHex(((string)file.FileNameWithoutExtension).Split("_").First());
await SQL.AddDownloadState(hash, loaded);
loadCount += 1;
}
});
return Ok(loadCount);
}
@ -82,7 +88,6 @@ namespace Wabbajack.BuildServer.Controllers
{
await using var ins = entry.Open();
var iniString = Encoding.UTF8.GetString(await ins.ReadAllAsync());
Utils.Log(iniString);
var data = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(iniString.LoadIniString(), true));
if (data == null)

8
Wabbajack.BuildServer/Controllers/ListValidation.cs
View File

@ -68,7 +68,10 @@ namespace Wabbajack.BuildServer.Controllers
var (_, result) = ValidateArchive(data, archive);
if (result == ArchiveStatus.InValid)
{
return await TryToFix(data, archive);
var fixResult = await TryToFix(data, archive);
return fixResult;
}
return (archive, result);
@ -105,7 +108,7 @@ namespace Wabbajack.BuildServer.Controllers
});
var cacheOptions = new MemoryCacheEntryOptions().SetSlidingExpiration(TimeSpan.FromMinutes(1));
var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1));
Cache.Set(ModListSummariesKey, results, cacheOptions);
return results;
}
@ -142,6 +145,7 @@ namespace Wabbajack.BuildServer.Controllers
var result = await _updater.GetAlternative(archive.Hash.ToHex());
return result switch
{
OkObjectResult ok => (archive, ArchiveStatus.Updated),
OkResult ok => (archive, ArchiveStatus.Updated),
AcceptedResult accept => (archive, ArchiveStatus.Updating),
_ => (archive, ArchiveStatus.InValid)

2
Wabbajack.BuildServer/Controllers/ModlistUpdater.cs
View File

@ -216,7 +216,7 @@ namespace Wabbajack.BuildServer.Controllers
private async Task<Archive> FindNexusAlternative(NexusDownloader.State state, Hash srcHash)
{
var origSize = _settings.PathForArchive(srcHash).Size;
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var api = await NexusApiClient.Get(Request?.Headers["apikey"].FirstOrDefault());
var allMods = await api.GetModFiles(state.Game, state.ModID);
var archive = allMods.files.Where(m => !string.IsNullOrEmpty(m.category_name))
.OrderBy(s => Math.Abs((long)s.size - origSize))

3
Wabbajack.BuildServer/Controllers/NexusCache.cs
View File

@ -41,7 +41,6 @@ namespace Wabbajack.BuildServer.Controllers
[Route("{GameName}/mods/{ModId}.json")]
public async Task<ModInfo> GetModInfo(string GameName, long ModId)
{
Utils.Log($"Nexus Mod Info {GameName} {ModId}");
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var result = await SQL.GetNexusModInfoString(game, ModId);
@ -69,8 +68,6 @@ namespace Wabbajack.BuildServer.Controllers
[Route("{GameName}/mods/{ModId}/files.json")]
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, long ModId)
{
Utils.Log($"Nexus Mod Files {GameName} {ModId}");
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var result = await SQL.GetModFiles(game, ModId);

3
Wabbajack.BuildServer/GraphQL/Query.cs
View File

@ -8,6 +8,7 @@ namespace Wabbajack.BuildServer.GraphQL
{
public Query(SqlService sql)
{
/*
FieldAsync<ListGraphType<ModListStatusType>>("modLists",
arguments: new QueryArguments(new QueryArgument<ArchiveEnumFilterType>
{
@ -28,7 +29,7 @@ namespace Wabbajack.BuildServer.GraphQL
}
});
*/
FieldAsync<ListGraphType<MetricResultType>>("dailyUniqueMetrics",
arguments: new QueryArguments(
new QueryArgument<MetricEnum> {Name = "metric_type", Description = "The grouping of metric data to query"}

8
Wabbajack.BuildServer/JobManager.cs
View File

@ -1,9 +1,11 @@
using System;
using System.Linq;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.BuildServer.BackendServices;
using Wabbajack.BuildServer.Controllers;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
@ -77,6 +79,7 @@ namespace Wabbajack.BuildServer
if (!Settings.JobScheduler) return;
var task = RunNexusCacheLoop();
var listIngest = (new ListIngest(Sql, Settings)).RunLoop(CancellationToken.None);
while (true)
{
@ -99,6 +102,7 @@ namespace Wabbajack.BuildServer
}
}
private async Task KillOrphanedJobs()
{
try
@ -126,7 +130,7 @@ namespace Wabbajack.BuildServer
if (!Settings.RunFrontEndJobs && typeof(T).ImplementsInterface(typeof(IFrontEndJob))) return;
try
{
var jobs = (await Sql.GetUnfinishedJobs())
var jobs = (await Sql.GetAllJobs(span))
.Where(j => j.Payload is T)
.OrderByDescending(j => j.Created)
.Take(10);
@ -134,7 +138,7 @@ namespace Wabbajack.BuildServer
foreach (var job in jobs)
{
if (job.Started == null || job.Ended == null) return;
if (DateTime.Now - job.Ended < span) return;
if (DateTime.UtcNow - job.Ended < span) return;
}
await Sql.EnqueueJob(new Job
{

67
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -167,7 +167,7 @@ namespace Wabbajack.BuildServer.Model.Models
AND DATEADD(DAY, number+1, dbo.MinMetricDate()) < dbo.MaxMetricDate()) as d
ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action
WHERE d.Action = @action
AND d.Date >= DATEADD(month, -1, GETDATE())
AND d.Date >= DATEADD(month, -1, GETUTCDATE())
group by d.Date, d.GroupingSubject, d.Action
ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action}))
.ToList();
@ -184,7 +184,7 @@ namespace Wabbajack.BuildServer.Model.Models
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)",
@"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETUTCDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)",
new {
job.Priority,
PrimaryKeyString = job.Payload.PrimaryKeyString,
@ -201,7 +201,7 @@ namespace Wabbajack.BuildServer.Model.Models
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"UPDATE dbo.Jobs SET Ended = GETDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id",
@"UPDATE dbo.Jobs SET Ended = GETUTCDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id",
new {
job.Id,
Success = job.Result.ResultType == JobResultType.Success,
@ -221,7 +221,7 @@ namespace Wabbajack.BuildServer.Model.Models
{
await using var conn = await Open();
var result = await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>(
@"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy
@"UPDATE jobs SET Started = GETUTCDATE(), RunBy = @RunBy
WHERE ID in (SELECT TOP(1) ID FROM Jobs
WHERE Started is NULL
AND PrimaryKeyString NOT IN (SELECT PrimaryKeyString from jobs WHERE Started IS NOT NULL and Ended IS NULL)
@ -229,14 +229,14 @@ namespace Wabbajack.BuildServer.Model.Models
SELECT TOP(1) Id, Started, Ended, Created, Payload, Priority FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC",
new {RunBy = Guid.NewGuid().ToString()});
return result.Select(k =>
new Job {
Id = k.Item1,
Started = k.Item2,
Ended = k.Item3,
Created = k.Item4,
Payload = k.Item5,
Priority = (Job.JobPriority)k.Item6
}).FirstOrDefault();
new Job {
Id = k.Item1,
Started = k.Item2,
Ended = k.Item3,
Created = k.Item4,
Payload = k.Item5,
Priority = (Job.JobPriority)k.Item6
}).FirstOrDefault();
}
@ -244,8 +244,16 @@ namespace Wabbajack.BuildServer.Model.Models
{
await using var conn = await Open();
var results =
await conn.QueryAsync<Job>("SELECT * from dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL ");
return results;
await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority FROM dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL ");
return results.Select(k =>
new Job {
Id = k.Item1,
Started = k.Item2,
Ended = k.Item3,
Created = k.Item4,
Payload = k.Item5,
Priority = (Job.JobPriority)k.Item6
});
}
@ -253,8 +261,16 @@ namespace Wabbajack.BuildServer.Model.Models
{
await using var conn = await Open();
var results =
await conn.QueryAsync<Job>("SELECT * from dbo.Jobs WHERE Ended IS NULL ");
return results;
await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority from dbo.Jobs WHERE Ended IS NULL ");
return results.Select(k =>
new Job {
Id = k.Item1,
Started = k.Item2,
Ended = k.Item3,
Created = k.Item4,
Payload = k.Item5,
Priority = (Job.JobPriority)k.Item6
});
}
@ -353,7 +369,7 @@ namespace Wabbajack.BuildServer.Model.Models
public async Task<IEnumerable<UploadedFile>> AllUploadedFiles()
{
await using var conn = await Open();
return await conn.QueryAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles ORDER BY UploadDate DESC");
return await conn.QueryAsync<UploadedFile>("SELECT Id, Name, Size, UploadedBy as Uploader, Hash, UploadDate, CDNName FROM dbo.UploadedFiles ORDER BY UploadDate DESC");
}
public async Task DeleteUploadedFile(Guid dupId)
@ -853,5 +869,22 @@ namespace Wabbajack.BuildServer.Model.Models
#endregion
public async Task<IEnumerable<Job>> GetAllJobs(TimeSpan from)
{
await using var conn = await Open();
var results =
await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority from dbo.Jobs WHERE Created >= @FromTime ",
new {FromTime = DateTime.UtcNow - from});
return results.Select(k =>
new Job {
Id = k.Item1,
Started = k.Item2,
Ended = k.Item3,
Created = k.Item4,
Payload = k.Item5,
Priority = (Job.JobPriority)k.Item6
});
}
}
}

2
Wabbajack.Common/Json.cs
View File

@ -98,7 +98,7 @@ namespace Wabbajack.Common
}
}
private class RelativePathConverter : JsonConverter<RelativePath>
public class RelativePathConverter : JsonConverter<RelativePath>
{
public override void WriteJson(JsonWriter writer, RelativePath value, JsonSerializer serializer)
{

2
Wabbajack.Common/Paths.cs
View File

@ -9,6 +9,7 @@ using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
@ -405,6 +406,7 @@ namespace Wabbajack.Common
}
}
[JsonConverter(typeof(Utils.RelativePathConverter))]
public struct RelativePath : IPath, IEquatable<RelativePath>, IComparable<RelativePath>
{
private readonly string? _nullable_path;