From 08a3bc6f239dea4f94e50562f27b90e623aa05a1 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Jan 2020 16:41:53 -0700 Subject: [PATCH 1/5] Fix a ton of bugs with the Author file uploader, and a crash on the build server --- .../Models/Sql/SqlService.cs | 25 ++++--- Wabbajack.Lib/FileUploader/AuthorAPI.cs | 75 ++++++++++--------- Wabbajack/Util/FilePickerVM.cs | 4 +- .../View Models/Settings/AuthorFilesVM.cs | 61 +++++++++------ Wabbajack/Views/Settings/AuthorFilesView.xaml | 12 ++- 5 files changed, 106 insertions(+), 71 deletions(-) diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index 99875469..c4e24dd9 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -16,15 +16,20 @@ namespace Wabbajack.BuildServer.Model.Models public class SqlService { private IConfiguration _configuration; - private IDbConnection _conn; + private AppSettings _settings; - public SqlService(AppSettings configuration) + public SqlService(AppSettings settings) { - _conn = new SqlConnection(configuration.SqlConnection); - _conn.Open(); + _settings = settings; + } - public IDbConnection Connection { get => _conn; } + private async Task Open() + { + var conn = new SqlConnection(_settings.SqlConnection); + await conn.OpenAsync(); + return conn; + } public async Task MergeVirtualFile(VirtualFile vfile) { @@ -36,7 +41,8 @@ namespace Wabbajack.BuildServer.Model.Models files = files.DistinctBy(f => f.Hash).ToList(); contents = contents.DistinctBy(c => (c.Parent, c.Path)).ToList(); - await Connection.ExecuteAsync("dbo.MergeIndexedFiles", new {Files = files.ToDataTable(), Contents = contents.ToDataTable()}, + await using var conn = await Open(); + await conn.ExecuteAsync("dbo.MergeIndexedFiles", new {Files = files.ToDataTable(), Contents = contents.ToDataTable()}, commandType: CommandType.StoredProcedure); } @@ -72,7 +78,8 @@ namespace Wabbajack.BuildServer.Model.Models public async Task HaveIndexdFile(string hash) { - var row = await Connection.QueryAsync(@"SELECT * FROM IndexedFile WHERE Hash = @Hash", + await using var conn = await Open(); + var row = await conn.QueryAsync(@"SELECT * FROM IndexedFile WHERE Hash = @Hash", new {Hash = BitConverter.ToInt64(hash.FromBase64())}); return row.Any(); } @@ -97,8 +104,8 @@ namespace Wabbajack.BuildServer.Model.Models /// public async Task AllArchiveContents(long hash) { - - var files = await Connection.QueryAsync(@" + await using var conn = await Open(); + var files = await conn.QueryAsync(@" SELECT 0 as Parent, i.Hash, i.Size, null as Path FROM IndexedFile WHERE Hash = @Hash UNION ALL SELECT a.Parent, i.Hash, i.Size, a.Path FROM AllArchiveContent a diff --git a/Wabbajack.Lib/FileUploader/AuthorAPI.cs b/Wabbajack.Lib/FileUploader/AuthorAPI.cs index 2ee31c55..7d28b01d 100644 --- a/Wabbajack.Lib/FileUploader/AuthorAPI.cs +++ b/Wabbajack.Lib/FileUploader/AuthorAPI.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Net.Http; using System.Reactive.Linq; +using System.Threading; using System.Threading.Tasks; using Wabbajack.Common; using File = Alphaleonis.Win32.Filesystem.File; @@ -12,10 +13,10 @@ namespace Wabbajack.Lib.FileUploader { public class AuthorAPI { - public static IObservable HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key"); + public static IObservable HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key.txt"); public static IObservable AuthorAPIKey => HaveAuthorAPIKey.Where(h => h) - .Select(_ => File.ReadAllText(Path.Combine(Consts.LocalAppDataPath, "author-api-key"))); + .Select(_ => File.ReadAllText(Path.Combine(Consts.LocalAppDataPath, "author-api-key.txt"))); public static string GetAPIKey() @@ -26,12 +27,14 @@ namespace Wabbajack.Lib.FileUploader public static readonly Uri UploadURL = new Uri("https://build.wabbajack.org/upload_file"); - public static long BLOCK_SIZE = (long)1024 * 1024 * 8; - public static Task UploadFile(WorkQueue queue, string filename) + public static long BLOCK_SIZE = (long)1024 * 1024 * 2; + public static Task UploadFile(WorkQueue queue, string filename, Action progressFn) { var tcs = new TaskCompletionSource(); - queue.QueueTask(async () => + Task.Run(async () => { + + var client = new HttpClient(); var fsize = new FileInfo(filename).Length; @@ -44,41 +47,43 @@ namespace Wabbajack.Lib.FileUploader } var key = await response.Content.ReadAsStringAsync(); - + long sent = 0; using (var iqueue = new WorkQueue(8)) { - - await Enumerable.Range(0, (int)(fsize / BLOCK_SIZE)) - .PMap(iqueue, async block_idx => + iqueue.Report("Starting Upload", 1); + await Enumerable.Range(0, (int)(fsize / BLOCK_SIZE)) + .PMap(iqueue, async block_idx => + { + var block_offset = block_idx * BLOCK_SIZE; + var block_size = block_offset + BLOCK_SIZE > fsize + ? fsize - block_offset + : BLOCK_SIZE; + Interlocked.Add(ref sent, block_size); + progressFn((double)sent / fsize); + + using (var fs = File.OpenRead(filename)) { - var block_offset = block_idx * BLOCK_SIZE; - var block_size = block_offset + BLOCK_SIZE > fsize - ? fsize - block_offset - : BLOCK_SIZE; + fs.Position = block_offset; + var data = new byte[block_size]; + await fs.ReadAsync(data, 0, data.Length); - using (var fs = File.OpenRead(filename)) + response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}", + new ByteArrayContent(data)); + + if (!response.IsSuccessStatusCode) { - fs.Position = block_offset; - var data = new byte[block_size]; - await fs.ReadAsync(data, 0, data.Length); - - response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}", - new ByteArrayContent(data)); - - if (!response.IsSuccessStatusCode) - { - tcs.SetResult("FAILED"); - return; - } - - var val = long.Parse(await response.Content.ReadAsStringAsync()); - if (val != block_offset + data.Length) - { - tcs.SetResult("Sync Error"); - return; - } + tcs.SetResult("FAILED"); + return; } - }); + + var val = long.Parse(await response.Content.ReadAsStringAsync()); + if (val != block_offset + data.Length) + { + tcs.SetResult("Sync Error"); + return; + } + } + }); } response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent("")); @@ -87,6 +92,8 @@ namespace Wabbajack.Lib.FileUploader else tcs.SetResult("FAILED"); + progressFn(0.0); + }); return tcs.Task; } diff --git a/Wabbajack/Util/FilePickerVM.cs b/Wabbajack/Util/FilePickerVM.cs index 4f20675a..94b4af85 100644 --- a/Wabbajack/Util/FilePickerVM.cs +++ b/Wabbajack/Util/FilePickerVM.cs @@ -245,7 +245,7 @@ namespace Wabbajack .ToGuiProperty(this, nameof(ErrorTooltip)); } - public ICommand ConstructTypicalPickerCommand() + public ICommand ConstructTypicalPickerCommand(IObservable canExecute = null) { return ReactiveCommand.Create( execute: () => @@ -280,7 +280,7 @@ namespace Wabbajack } if (dlg.ShowDialog() != CommonFileDialogResult.Ok) return; TargetPath = dlg.FileName; - }); + }, canExecute: canExecute); } } } diff --git a/Wabbajack/View Models/Settings/AuthorFilesVM.cs b/Wabbajack/View Models/Settings/AuthorFilesVM.cs index e0f58c24..6e3f2526 100644 --- a/Wabbajack/View Models/Settings/AuthorFilesVM.cs +++ b/Wabbajack/View Models/Settings/AuthorFilesVM.cs @@ -6,6 +6,8 @@ using System.Reactive.Subjects; using System.Threading.Tasks; using System.Windows; +using System.Windows.Documents; +using System.Windows.Input; using Alphaleonis.Win32.Filesystem; using Microsoft.WindowsAPICodePack.Shell.PropertySystem; using ReactiveUI; @@ -20,39 +22,52 @@ namespace Wabbajack { public class AuthorFilesVM : BackNavigatingVM { - public Visibility IsVisible { get; } + private readonly ObservableAsPropertyHelper _isVisible; + public Visibility IsVisible => _isVisible.Value; - [Reactive] - public string SelectedFile { get; set; } - - public IReactiveCommand SelectFile { get; } + + private readonly ObservableAsPropertyHelper _selectedFile; + + public ICommand SelectFile { get; } + public ICommand HyperlinkCommand { get; } public IReactiveCommand Upload { get; } - - [Reactive] - public double UploadProgress { get; set; } + + [Reactive] public double UploadProgress { get; set; } + [Reactive] public string FinalUrl { get; set; } private WorkQueue Queue = new WorkQueue(1); + public FilePickerVM Picker { get;} + + private Subject _isUploading = new Subject(); + private IObservable IsUploading { get; } + public AuthorFilesVM(SettingsVM vm) : base(vm.MWVM) { - var sub = new Subject(); - Queue.Status.Select(s => (double)s.ProgressPercent).Subscribe(v => - { - UploadProgress = v; - }); - IsVisible = AuthorAPI.HasAPIKey ? Visibility.Visible : Visibility.Collapsed; - - SelectFile = ReactiveCommand.Create(() => - { - var fod = UIUtils.OpenFileDialog("*|*"); - if (fod != null) - SelectedFile = fod; - }); + IsUploading = _isUploading; + Picker = new FilePickerVM(this); + _isVisible = AuthorAPI.HaveAuthorAPIKey.Select(h => h ? Visibility.Visible : Visibility.Collapsed) + .ToProperty(this, x => x.IsVisible); + + SelectFile = Picker.ConstructTypicalPickerCommand(IsUploading.StartWith(false).Select(u => !u)); + + HyperlinkCommand = ReactiveCommand.Create(() => Clipboard.SetText(FinalUrl)); + Upload = ReactiveCommand.Create(async () => { - SelectedFile = await AuthorAPI.UploadFile(Queue, SelectedFile); - }); + _isUploading.OnNext(true); + try + { + FinalUrl = await AuthorAPI.UploadFile(Queue, Picker.TargetPath, progress => UploadProgress = progress ); + } + finally + { + _isUploading.OnNext(false); + } + }, IsUploading.StartWith(false).Select(u => !u) + .CombineLatest(Picker.WhenAnyValue(t => t.TargetPath).Select(f => f != null), + (a, b) => a && b)); } } } diff --git a/Wabbajack/Views/Settings/AuthorFilesView.xaml b/Wabbajack/Views/Settings/AuthorFilesView.xaml index 0228eee6..608d81ee 100644 --- a/Wabbajack/Views/Settings/AuthorFilesView.xaml +++ b/Wabbajack/Views/Settings/AuthorFilesView.xaml @@ -24,6 +24,7 @@ + @@ -36,10 +37,15 @@ FontSize="20" FontWeight="Bold" Text="File Uploader" /> - + - - + + + + + + + From 4d65bacbf3713436b0d8bb6f785f9f8ff28b8fc2 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Jan 2020 21:29:20 -0700 Subject: [PATCH 2/5] Better error handling for author uploading --- Wabbajack.Lib/FileUploader/AuthorAPI.cs | 36 +++++++++++-------- .../View Models/Settings/AuthorFilesVM.cs | 7 +++- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/Wabbajack.Lib/FileUploader/AuthorAPI.cs b/Wabbajack.Lib/FileUploader/AuthorAPI.cs index 7d28b01d..be56db50 100644 --- a/Wabbajack.Lib/FileUploader/AuthorAPI.cs +++ b/Wabbajack.Lib/FileUploader/AuthorAPI.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Linq; +using System.Net; using System.Net.Http; using System.Reactive.Linq; using System.Threading; @@ -28,38 +29,40 @@ namespace Wabbajack.Lib.FileUploader public static readonly Uri UploadURL = new Uri("https://build.wabbajack.org/upload_file"); public static long BLOCK_SIZE = (long)1024 * 1024 * 2; + public static int MAX_CONNECTIONS = 8; public static Task UploadFile(WorkQueue queue, string filename, Action progressFn) { var tcs = new TaskCompletionSource(); Task.Run(async () => { - - - - var client = new HttpClient(); + var handler = new HttpClientHandler {MaxConnectionsPerServer = MAX_CONNECTIONS}; + var client = new HttpClient(handler); var fsize = new FileInfo(filename).Length; client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey()); var response = await client.PutAsync(UploadURL+$"/{Path.GetFileName(filename)}/start", new StringContent("")); if (!response.IsSuccessStatusCode) { - tcs.SetResult("FAILED"); + tcs.SetException(new Exception($"Start Error: {response.StatusCode} {response.ReasonPhrase}")); return; } var key = await response.Content.ReadAsStringAsync(); long sent = 0; - using (var iqueue = new WorkQueue(8)) + using (var iqueue = new WorkQueue(MAX_CONNECTIONS)) { iqueue.Report("Starting Upload", 1); await Enumerable.Range(0, (int)(fsize / BLOCK_SIZE)) .PMap(iqueue, async block_idx => { + if (tcs.Task.IsFaulted) return; var block_offset = block_idx * BLOCK_SIZE; var block_size = block_offset + BLOCK_SIZE > fsize ? fsize - block_offset : BLOCK_SIZE; Interlocked.Add(ref sent, block_size); progressFn((double)sent / fsize); + + int retries = 0; using (var fs = File.OpenRead(filename)) { @@ -67,30 +70,35 @@ namespace Wabbajack.Lib.FileUploader var data = new byte[block_size]; await fs.ReadAsync(data, 0, data.Length); + response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}", new ByteArrayContent(data)); if (!response.IsSuccessStatusCode) { - tcs.SetResult("FAILED"); + tcs.SetException(new Exception($"Put Error: {response.StatusCode} {response.ReasonPhrase}")); return; } var val = long.Parse(await response.Content.ReadAsStringAsync()); if (val != block_offset + data.Length) { - tcs.SetResult("Sync Error"); - return; + tcs.SetResult($"Sync Error {val} vs {block_offset + data.Length}"); + tcs.SetException(new Exception($"Sync Error {val} vs {block_offset + data.Length}")); } } }); } - response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent("")); - if (response.IsSuccessStatusCode) - tcs.SetResult(await response.Content.ReadAsStringAsync()); - else - tcs.SetResult("FAILED"); + if (!tcs.Task.IsFaulted) + { + progressFn(1.0); + response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent("")); + if (response.IsSuccessStatusCode) + tcs.SetResult(await response.Content.ReadAsStringAsync()); + else + tcs.SetException(new Exception($"Finalization Error: {response.StatusCode} {response.ReasonPhrase}")); + } progressFn(0.0); diff --git a/Wabbajack/View Models/Settings/AuthorFilesVM.cs b/Wabbajack/View Models/Settings/AuthorFilesVM.cs index 6e3f2526..e2b5bddf 100644 --- a/Wabbajack/View Models/Settings/AuthorFilesVM.cs +++ b/Wabbajack/View Models/Settings/AuthorFilesVM.cs @@ -59,7 +59,12 @@ namespace Wabbajack _isUploading.OnNext(true); try { - FinalUrl = await AuthorAPI.UploadFile(Queue, Picker.TargetPath, progress => UploadProgress = progress ); + FinalUrl = await AuthorAPI.UploadFile(Queue, Picker.TargetPath, + progress => UploadProgress = progress); + } + catch (Exception ex) + { + FinalUrl = ex.ToString(); } finally { From b7ef33997825d784c92fbe96e3ad11fdf09d6a28 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 30 Jan 2020 06:07:16 -0700 Subject: [PATCH 3/5] Fix file extractor tests, convert metrics to SQL --- Wabbajack.BuildServer/Controllers/GraphQL.cs | 8 +++-- .../Controllers/MetricsController.cs | 20 +++++++++-- Wabbajack.BuildServer/GraphQL/Query.cs | 15 ++++++-- Wabbajack.BuildServer/Models/Metric.cs | 10 +++--- .../Models/Sql/Results/AggregateMetric.cs | 11 ++++++ .../Models/Sql/SqlService.cs | 36 +++++++++++++++++++ Wabbajack.Common/FileExtractor.cs | 2 +- 7 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 Wabbajack.BuildServer/Models/Sql/Results/AggregateMetric.cs diff --git a/Wabbajack.BuildServer/Controllers/GraphQL.cs b/Wabbajack.BuildServer/Controllers/GraphQL.cs index c1d2bf7c..685dd1d7 100644 --- a/Wabbajack.BuildServer/Controllers/GraphQL.cs +++ b/Wabbajack.BuildServer/Controllers/GraphQL.cs @@ -4,6 +4,7 @@ using GraphQL.Types; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Wabbajack.BuildServer.GraphQL; +using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; namespace Wabbajack.BuildServer.Controllers @@ -12,15 +13,18 @@ namespace Wabbajack.BuildServer.Controllers [ApiController] public class GraphQL : AControllerBase { - public GraphQL(ILogger logger, DBContext db) : base(logger, db) + private SqlService _sql; + + public GraphQL(ILogger logger, DBContext db, SqlService sql) : base(logger, db) { + _sql = sql; } [HttpPost] public async Task Post([FromBody] GraphQLQuery query) { var inputs = query.Variables.ToInputs(); - var schema = new Schema {Query = new Query(Db), Mutation = new Mutation(Db)}; + var schema = new Schema {Query = new Query(Db, _sql), Mutation = new Mutation(Db)}; var result = await new DocumentExecuter().ExecuteAsync(_ => { diff --git a/Wabbajack.BuildServer/Controllers/MetricsController.cs b/Wabbajack.BuildServer/Controllers/MetricsController.cs index ae9c2836..d336531d 100644 --- a/Wabbajack.BuildServer/Controllers/MetricsController.cs +++ b/Wabbajack.BuildServer/Controllers/MetricsController.cs @@ -1,11 +1,14 @@ using System; using System.Collections.Generic; +using System.Data.SqlTypes; using System.Linq; using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using MongoDB.Driver; using MongoDB.Driver.Linq; +using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; using Wabbajack.Common; using Wabbajack.Lib.ModListRegistry; @@ -16,8 +19,11 @@ namespace Wabbajack.BuildServer.Controllers [Route("/metrics")] public class MetricsController : AControllerBase { - public MetricsController(ILogger logger, DBContext db) : base(logger, db) + private SqlService _sql; + + public MetricsController(ILogger logger, DBContext db, SqlService sql) : base(logger, db) { + _sql = sql; } [HttpGet] @@ -29,10 +35,20 @@ namespace Wabbajack.BuildServer.Controllers return new Result { Timestamp = date}; } + [Authorize] + [HttpGet] + [Route("transfer")] + public async Task Transfer() + { + var all_metrics = await Db.Metrics.AsQueryable().ToListAsync(); + await _sql.IngestAllMetrics(all_metrics); + return "done"; + } + private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null) { Logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}"); - await Db.Metrics.InsertOneAsync(new Metric + await _sql.IngestMetric(new Metric { Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey }); diff --git a/Wabbajack.BuildServer/GraphQL/Query.cs b/Wabbajack.BuildServer/GraphQL/Query.cs index b61a9815..5ff94f4c 100644 --- a/Wabbajack.BuildServer/GraphQL/Query.cs +++ b/Wabbajack.BuildServer/GraphQL/Query.cs @@ -1,10 +1,13 @@ using System; +using System.Linq; using System.Collections.Generic; +using System.Data.SqlTypes; using GraphQL; using GraphQL.Types; using GraphQLParser.AST; using MongoDB.Driver; using MongoDB.Driver.Linq; +using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; using Wabbajack.Common; @@ -12,7 +15,7 @@ namespace Wabbajack.BuildServer.GraphQL { public class Query : ObjectGraphType { - public Query(DBContext db) + public Query(DBContext db, SqlService sql) { Field>("unfinishedJobs", resolve: context => { @@ -68,7 +71,15 @@ namespace Wabbajack.BuildServer.GraphQL resolve: async context => { var group = context.GetArgument("metric_type"); - return await Metric.Report(db, group); + var data = (await sql.MetricsReport(group)) + .GroupBy(m => m.Subject) + .Select(g => new MetricResult + { + SeriesName = g.Key, + Labels = g.Select(m => m.Date.ToString()).ToList(), + Values = g.Select(m => m.Count).ToList() + }); + return data; }); } } diff --git a/Wabbajack.BuildServer/Models/Metric.cs b/Wabbajack.BuildServer/Models/Metric.cs index 796af29e..b95c6261 100644 --- a/Wabbajack.BuildServer/Models/Metric.cs +++ b/Wabbajack.BuildServer/Models/Metric.cs @@ -16,11 +16,11 @@ namespace Wabbajack.BuildServer.Models public class Metric { [BsonId] - public ObjectId Id; - public DateTime Timestamp; - public string Action; - public string Subject; - public string MetricsKey; + public ObjectId Id { get; set; } + public DateTime Timestamp { get; set; } + public string Action { get; set; } + public string Subject { get; set; } + public string MetricsKey { get; set; } public static async Task> Report(DBContext db, string grouping) diff --git a/Wabbajack.BuildServer/Models/Sql/Results/AggregateMetric.cs b/Wabbajack.BuildServer/Models/Sql/Results/AggregateMetric.cs new file mode 100644 index 00000000..b0cc3708 --- /dev/null +++ b/Wabbajack.BuildServer/Models/Sql/Results/AggregateMetric.cs @@ -0,0 +1,11 @@ +using System; + +namespace Wabbajack.BuildServer.Model.Models.Results +{ + public class AggregateMetric + { + public DateTime Date { get; set; } + public string Subject { get; set; } + public int Count { get; set; } + } +} diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index c4e24dd9..4c1e226b 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Threading.Tasks; using Dapper; using Microsoft.Extensions.Configuration; +using Wabbajack.BuildServer.Model.Models.Results; using Wabbajack.BuildServer.Models; using Wabbajack.Common; using Wabbajack.VirtualFileSystem; @@ -127,5 +128,40 @@ namespace Wabbajack.BuildServer.Model.Models } return Build(0).First(); } + + public async Task IngestAllMetrics(IEnumerable allMetrics) + { + await using var conn = await Open(); + await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", allMetrics); + } + public async Task IngestMetric(Metric metric) + { + await using var conn = await Open(); + await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", metric); + } + + public async Task> MetricsReport(string action) + { + await using var conn = await Open(); + return (await conn.QueryAsync(@" + SELECT d.Date, d.GroupingSubject as Subject, Count(*) as Count FROM + (select DISTINCT CONVERT(date, Timestamp) as Date, GroupingSubject, Action, MetricsKey from dbo.Metrics) m + RIGHT OUTER JOIN + (SELECT CONVERT(date, DATEADD(DAY, number + 1, dbo.MinMetricDate())) as Date, GroupingSubject, Action + FROM master..spt_values + CROSS JOIN ( + SELECT DISTINCT GroupingSubject, Action FROM dbo.Metrics + WHERE MetricsKey is not null + AND Subject != 'Default' + AND TRY_CONVERT(uniqueidentifier, Subject) is null) as keys + WHERE type = 'P' + AND DATEADD(DAY, number+1, dbo.MinMetricDate()) < dbo.MaxMetricDate()) as d + ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action + WHERE d.Action = @action + group by d.Date, d.GroupingSubject, d.Action + ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action})) + .ToList(); + } + } } diff --git a/Wabbajack.Common/FileExtractor.cs b/Wabbajack.Common/FileExtractor.cs index 5a9d68a0..5f444f5c 100644 --- a/Wabbajack.Common/FileExtractor.cs +++ b/Wabbajack.Common/FileExtractor.cs @@ -209,7 +209,7 @@ namespace Wabbajack.Common { var info = new ProcessStartInfo { - FileName = "innounp.exe", + FileName = @"Extractors\innounp.exe", Arguments = $"-t \"{v}\" ", RedirectStandardError = true, RedirectStandardInput = true, From d8e8efc404970f5761f6f89f2f1a845b8722eda1 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 30 Jan 2020 14:22:08 -0700 Subject: [PATCH 4/5] Commit SQL changes --- Wabbajack.BuildServer/sql/wabbajack_db.sql | Bin 10184 -> 28858 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/Wabbajack.BuildServer/sql/wabbajack_db.sql b/Wabbajack.BuildServer/sql/wabbajack_db.sql index bc90d6d67833c88c9a67fdf9b2a2e9fda09ebce6..64c514d524012915579fcbebf572ded57e2b5e5c 100644 GIT binary patch literal 28858 zcmeI5ZFf`05y$s!b56g5-t=HvVqy}~CJj9ZTNoEV5X*!lJ~=vo1vn*kik(3E;oJUy zI}E#fb@gO)B_;9EktN-`cXxJno_1z-{J;M`4BgNPt#BBA2#>>AI1l~sDts6I7(NJ_ zVK07O4}Z>HEnR&QzE``aYW=yMZG{azd#?AB*hfG1_d5RO%J+Id4gU(gaHOYxR_{E+ ztE=m;!g2UfPkS%@*ymcf9zG5q>F*8w<=xWX&%;{yOn+~Lzv}mcu%o{@Y}X^a>tSDW zYKMb3vyJ$z9q#DU=Q;{DX72E4pq_k~Mq#SFi$w0$TXY{%Je z#TD2nX0R!Y?1e|Vb5DKH_ov~eULWhz`n_7E$~dz$}gc&_=)2cJIP zs>U6TSLS^qd?K8FGzP110PQy4;ofOtT8uu@+8FlNwR+ID;kXx|_HDhmx)m{QD^%s6 zp7^~Znrw%MYJCvb#bpJ(!FT&wv+8q5nc)FCf_w7a1gX1Pv5*AYSU}TPJ*_ z_qN&@ZQc}aa(T5S*jiyl)M_p6MmP)$Ew%u4M5%=Jrrh$7EzjGwlirlXrYWHraZb z-ka^5*EQ#T_0KxyIPU4nZoGp0-H9vS941OAs?wn)9%AjW+qN5!5@y?(4$P(0H<~rt zWkdK$*PFuv@)64t%3Ifn*hknlQ1E~rZ%T4vr8^C5 zMV;Q$$lzg9FUD?CqG6wGX#}v0P0`aQxW6AqZbB=$oEgveJwer#EwU^8G>88rd*FqA zy`cMK{UHq=L|Q-+e4E{A4kp&0J?Vk=2&QH32@lPUhipO0Jkt7ih3~dzj2?g|;0I)6 zj+5pf!Ft%&8(Cl$bQR~=Q&`1N>+I>%_RL5Dv%XFGyJV>1ra3%-d-m37rko>tnmMw8 zJGK{S4hEAB_gJ%bBR_yHY-qM=dJ_>8u{4x6ttS|J1RPDW;6%wAAt&7@`!}mq& zXrkHAa6{_|Rbe|~T{UeV`_33CVpP4OU7dSuOvl*%q%EkN!uigAu&bBtn@sPu-tO^c zYk^_w?Z@>%&Q$3Q7~yq1bLJz#|HZ!5tQKsOCf3R1|2G;3Ij|vmKqBS)Z-b}QoEr_m zf?a|Iz>c4FgzFs}gZ4=BVvh1JKfd=hvaAM2WVlUwSKoiVbmqxYS- zw!*Z_b-lj<*#X@ZfcDJ8bq7-nhX!i3^kUx%|ZSV))4n=*(Z+&le z1seiiqWhn&%@w}&xYqc7mk~O!cPk z+cw9M_;Zm@Z}dKrEp!~cqszsM<6&G37aq!%BUd#5D2W6O-%vd1(~R%%eZP`i zhO&vb)Y0{Pq=Q4aHUv7NJd1>GN1SkT2yAP@=^|lUj~bNqGpR+CWs#DewLA|0k@T2M z!Z#^7XCpI-c%8(kzDJMBIMkiU^L^35C(^8^viz`Op2zt&X$yN)a4@D+=OOpjlX*5v z@A`+~Lut2LjlG}&8>1~;qk&IF)t`qy==FsxsdVj{;?%psH7JhRKABgU|;Y;=0Qg7r6 zfoG^2DGs*Jh3%H+K{ktFZ}ePV3MN|AtpPHCeJM0EY!0_{hv*O91!Dd{MJ^H&jVu`K z8^;>z)86~SpSiYKz{@HU+3+7;a+59Nkp)fOk_ zW7yM8@L?LuRtm3`d;eMOb*&+~Kee0fw?rQ~rP;_$j%%~Hs_aBRv$GOCOqQ#Qw}vw% zrg<$Z27eRlT+JjWli6};EW7W4=mEN}K1=Scx3mnP2ckQ2Vxgm3dZ%&n+(|pt#p#?q ztp>Sd-g-Cs!1A^!6Kz=QGVhd}HS@EPq0*5Tk=!85Ql{TJ_~2pp6_;FLXB$V|C#wOx zS`LB4UBydjMZ}gqtkhCSn6pHW=oGX@`I z%W;ko4a=DexSsP4|I{x$_REZmVL0&X)8OKH#D&SXL_w}&O^(xMHKX@s#!bG5bbQ$7 ziefwEJX+DhOIc#2XSaNM@xO?JdJNPw>N2mxc*$2Ug{dnrC>7IHmv-fqT-w>4Qra11 zCrQ@|e-W=I8JJ}2GUIfXG7v05|GRqqb|l|;?O*BulY5DlO1W3nn;oht*?b(iGBb6DKzdt|LFg(n5=e*b)P0w^(d@iO#ve=x< zL)op1&aK3Yi2rE*Ih51pO3v1sGuo#%j2jp2O`EMWNVaP!7n=8BU3#zP>%E$lwl1S@ zzdtQqo&Rk3@lf{muron<^#!X$gr8?EdXt(t&2%WAM>Ug%l_tN|^6{u7^n+|Rt~Lcj zTRO=r7TM^7ktM2R;j?WDhORPwQdPTQPz*m8H5-kEl`xoa9C^Ezi$!KK+1ISU?+%F4 z{Ykxir0ki`M0u#LnX@;>9!4{#GT zoDKd*TPr1fKpACpV_q@8*+|3ODsj2koJDeJiOkm$X)=zMtah~3mZEi|qvO<{$TKL6 z@|uaHmGLb!7N=LL5qR^wc?9<&bMT1dr02{n^65NR<=|?L$#6cLCm&gjGONWyWinzk z)WD&ph)0;IvS;x(;zPu%iQZb~hSwUorPe8O4d$pQiQlu@OCG8IwMDhvOE(zHCFWk% zCM-|BjX&v8$8qUeeukR5J-=ubJk!QcbL6be|;Qw z$u+LwE~c-bt9p;eGIK&b#P81!_5EIQh1FoCBIL0A#N0+)hHOUXih0?6rQEGb>vr^r z*oitk6wAt6)6L@j`K@06RJ}&SeSo7lGuM3VYiz&C?y}AgM#lR%FCRX#lvCMs-tpx= zFW-lPd#AV3zc=lFX6eeaN{tcrXAh!Z_i6l2RS3MWWhD(b;*~tUeeI3i8=LR!&Ms@c zOk5iVj-3DL&$3Dj*>2NTdMSITKeaiFtKg^fltJm79^iZml}GW-Oi!P=By{|jr`qxN zkFWMZOv5>c7zJm`OWvugJaSvUD61sLIdad!t>-4MHP?!B#XkejZ|kH*sS|DsCwaZd zyysE5&cpi$M`#m1M^%<_dxG5?aS8k?mMy)j*-_tcUO_)jI^ z+N#kIEuxMLHDHKyF%v(vKe%qkl0jCD=b2i2Z5AR&$GZMP&z5<3Rt=!6E3;Z;agUnM z-;0(BD|oS~f=hiKs~2n;xYj<0+N$_dV|m=ld;Bb~0-X26SuOWmmK0@GY0#E%2)vR% zXF7w^4XL{D>bnyQreX4Dxd$EGQ}inD$1>=YQ;QHZRZC%{^cEaqYLAaj>i@J8R*&OV-9}ua$87xaspN!Hwsgy3UN- z6(^zl;N*lE>oLDG2-DzQgRtx|!CPKWnOAjV`CKNWsZl&X9c2y5m5b0v1Q3ahG|%AKg~R zsM1PbCa!Q{!bStRi7`I*#(+^5UFb&KBe{E0oqJ`YCe+3GGG}J;&&-+gYKOnOaO7w0 z^6H+Yxy%-r#IA!2QMgPAi^xLE$3*7Wj*UfGj66Z2$V4P(`s8(!v7vq%_Ju24kInE2 zyl-=pb+tW8^akUN2G+v*uzNC6u^I8DWA4Ai+OXC+NCIC+BgE~lG*Pk)6Aj&Q;_ldK z;*d4O52Q;Txi#vcIHpPGlCu|gonf-YC_s3W}%-p9jCH#R~Oe>JulwrX(!iAan5 zoH?xR7XA5TTuXOV>)fhqvL0f{qg*48i9yF)0H`%X9w{8 z#$w&lOK7!M< Date: Thu, 30 Jan 2020 16:39:14 -0700 Subject: [PATCH 5/5] Make number of jobs configurable, fix bug in IndexedFile query --- Wabbajack.BuildServer/AppSettings.cs | 2 ++ Wabbajack.BuildServer/JobManager.cs | 2 +- Wabbajack.BuildServer/Models/Sql/SqlService.cs | 18 +++++++++++------- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Wabbajack.BuildServer/AppSettings.cs b/Wabbajack.BuildServer/AppSettings.cs index 12042db6..af0996eb 100644 --- a/Wabbajack.BuildServer/AppSettings.cs +++ b/Wabbajack.BuildServer/AppSettings.cs @@ -22,5 +22,7 @@ namespace Wabbajack.BuildServer public string BunnyCDN_Password { get; set; } public string SqlConnection { get; set; } + + public int MaxJobs { get; set; } = 2; } } diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index f2541961..f4351078 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -32,7 +32,7 @@ namespace Wabbajack.BuildServer public void StartJobRunners() { if (!Settings.JobRunner) return; - for (var idx = 0; idx < 2; idx++) + for (var idx = 0; idx < Settings.MaxJobs; idx++) { Task.Run(async () => { diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index 4c1e226b..a88acbc6 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -107,7 +107,7 @@ namespace Wabbajack.BuildServer.Model.Models { await using var conn = await Open(); var files = await conn.QueryAsync(@" - SELECT 0 as Parent, i.Hash, i.Size, null as Path FROM IndexedFile WHERE Hash = @Hash + SELECT 0 as Parent, i.Hash, i.Size, null as Path FROM IndexedFile i WHERE Hash = @Hash UNION ALL SELECT a.Parent, i.Hash, i.Size, a.Path FROM AllArchiveContent a LEFT JOIN IndexedFile i ON i.Hash = a.Child @@ -118,13 +118,17 @@ namespace Wabbajack.BuildServer.Model.Models List Build(long parent) { - return grouped[parent].Select(f => new IndexedVirtualFile + if (grouped.TryGetValue(parent, out var children)) { - Name = f.Path, - Hash = BitConverter.GetBytes(f.Hash).ToBase64(), - Size = f.Size, - Children = Build(f.Hash) - }).ToList(); + return children.Select(f => new IndexedVirtualFile + { + Name = f.Path, + Hash = BitConverter.GetBytes(f.Hash).ToBase64(), + Size = f.Size, + Children = Build(f.Hash) + }).ToList(); + } + return new List(); } return Build(0).First(); }