Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix file extractor tests, convert metrics to SQL

Browse Source
This commit is contained in:
Timothy Baldridge 2020-01-30 06:07:16 -07:00
parent 4d65bacbf3
commit b7ef339978
7 changed files with 90 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Wabbajack.BuildServer/Controllers/GraphQL.cs
View File

@ -4,6 +4,7 @@ using GraphQL.Types;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer.GraphQL;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
namespace Wabbajack.BuildServer.Controllers
@ -12,15 +13,18 @@ namespace Wabbajack.BuildServer.Controllers
[ApiController]
public class GraphQL : AControllerBase<GraphQL>
{
public GraphQL(ILogger<GraphQL> logger, DBContext db) : base(logger, db)
private SqlService _sql;
public GraphQL(ILogger<GraphQL> logger, DBContext db, SqlService sql) : base(logger, db)
{
_sql = sql;
}
[HttpPost]
public async Task<IActionResult> Post([FromBody] GraphQLQuery query)
{
var inputs = query.Variables.ToInputs();
var schema = new Schema {Query = new Query(Db), Mutation = new Mutation(Db)};
var schema = new Schema {Query = new Query(Db, _sql), Mutation = new Mutation(Db)};
var result = await new DocumentExecuter().ExecuteAsync(_ =>
{

20
Wabbajack.BuildServer/Controllers/MetricsController.cs
View File

@ -1,11 +1,14 @@
using System;
using System.Collections.Generic;
using System.Data.SqlTypes;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
using Wabbajack.Lib.ModListRegistry;
@ -16,8 +19,11 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/metrics")]
public class MetricsController : AControllerBase<MetricsController>
{
public MetricsController(ILogger<MetricsController> logger, DBContext db) : base(logger, db)
private SqlService _sql;
public MetricsController(ILogger<MetricsController> logger, DBContext db, SqlService sql) : base(logger, db)
{
_sql = sql;
}
[HttpGet]
@ -29,10 +35,20 @@ namespace Wabbajack.BuildServer.Controllers
return new Result { Timestamp = date};
}
[Authorize]
[HttpGet]
[Route("transfer")]
public async Task<string> Transfer()
{
var all_metrics = await Db.Metrics.AsQueryable().ToListAsync();
await _sql.IngestAllMetrics(all_metrics);
return "done";
}
private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
{
Logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}");
await Db.Metrics.InsertOneAsync(new Metric
await _sql.IngestMetric(new Metric
{
Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey
});

15
Wabbajack.BuildServer/GraphQL/Query.cs
View File

@ -1,10 +1,13 @@
using System;
using System.Linq;
using System.Collections.Generic;
using System.Data.SqlTypes;
using GraphQL;
using GraphQL.Types;
using GraphQLParser.AST;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
@ -12,7 +15,7 @@ namespace Wabbajack.BuildServer.GraphQL
{
public class Query : ObjectGraphType
{
public Query(DBContext db)
public Query(DBContext db, SqlService sql)
{
Field<ListGraphType<JobType>>("unfinishedJobs", resolve: context =>
{
@ -68,7 +71,15 @@ namespace Wabbajack.BuildServer.GraphQL
resolve: async context =>
{
var group = context.GetArgument<string>("metric_type");
return await Metric.Report(db, group);
var data = (await sql.MetricsReport(group))
.GroupBy(m => m.Subject)
.Select(g => new MetricResult
{
SeriesName = g.Key,
Labels = g.Select(m => m.Date.ToString()).ToList(),
Values = g.Select(m => m.Count).ToList()
});
return data;
});
}
}

10
Wabbajack.BuildServer/Models/Metric.cs
View File

@ -16,11 +16,11 @@ namespace Wabbajack.BuildServer.Models
public class Metric
{
[BsonId]
public ObjectId Id;
public DateTime Timestamp;
public string Action;
public string Subject;
public string MetricsKey;
public ObjectId Id { get; set; }
public DateTime Timestamp { get; set; }
public string Action { get; set; }
public string Subject { get; set; }
public string MetricsKey { get; set; }
public static async Task<IEnumerable<MetricResult>> Report(DBContext db, string grouping)

11
Wabbajack.BuildServer/Models/Sql/Results/AggregateMetric.cs Normal file
View File

@ -0,0 +1,11 @@
using System;
namespace Wabbajack.BuildServer.Model.Models.Results
{
public class AggregateMetric
{
public DateTime Date { get; set; }
public string Subject { get; set; }
public int Count { get; set; }
}
}

36
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -7,6 +7,7 @@ using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Microsoft.Extensions.Configuration;
using Wabbajack.BuildServer.Model.Models.Results;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
using Wabbajack.VirtualFileSystem;
@ -127,5 +128,40 @@ namespace Wabbajack.BuildServer.Model.Models
}
return Build(0).First();
}
public async Task IngestAllMetrics(IEnumerable<Metric> allMetrics)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", allMetrics);
}
public async Task IngestMetric(Metric metric)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", metric);
}
public async Task<IEnumerable<AggregateMetric>> MetricsReport(string action)
{
await using var conn = await Open();
return (await conn.QueryAsync<AggregateMetric>(@"
SELECT d.Date, d.GroupingSubject as Subject, Count(*) as Count FROM
(select DISTINCT CONVERT(date, Timestamp) as Date, GroupingSubject, Action, MetricsKey from dbo.Metrics) m
RIGHT OUTER JOIN
(SELECT CONVERT(date, DATEADD(DAY, number + 1, dbo.MinMetricDate())) as Date, GroupingSubject, Action
FROM master..spt_values
CROSS JOIN (
SELECT DISTINCT GroupingSubject, Action FROM dbo.Metrics
WHERE MetricsKey is not null
AND Subject != 'Default'
AND TRY_CONVERT(uniqueidentifier, Subject) is null) as keys
WHERE type = 'P'
AND DATEADD(DAY, number+1, dbo.MinMetricDate()) < dbo.MaxMetricDate()) as d
ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action
WHERE d.Action = @action
group by d.Date, d.GroupingSubject, d.Action
ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action}))
.ToList();
}
}
}

2
Wabbajack.Common/FileExtractor.cs
View File

@ -209,7 +209,7 @@ namespace Wabbajack.Common
{
var info = new ProcessStartInfo
{
FileName = "innounp.exe",
FileName = @"Extractors\innounp.exe",
Arguments = $"-t \"{v}\" ",
RedirectStandardError = true,
RedirectStandardInput = true,