Version 2.0.6.2

CHANGELOG.md

@@ -1,5 +1,9 @@
 ### Changelog
 
+#### Version - 2.0.9.2 - 6/8/2020
+* Several bug fixes
+* Added Darkest Dungeon as a game to alpha test non Bethesda MO2 game integration
+
 #### Version - 2.0.9.1 - 6/8/2020
 * Add crash handling and crash logging to the application startup
 * Use string distance comparisons to find Nexus mod upgrades

Wabbajack.CLI/Wabbajack.CLI.csproj

@@ -6,8 +6,8 @@
         <AssemblyName>wabbajack-cli</AssemblyName>
         <Company>Wabbajack</Company>
         <Platforms>x64</Platforms>
-        <AssemblyVersion>2.0.9.1</AssemblyVersion>
-        <FileVersion>2.0.9.1</FileVersion>
+        <AssemblyVersion>2.0.9.2</AssemblyVersion>
+        <FileVersion>2.0.9.2</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>An automated ModList installer</Description>
         <PublishReadyToRun>true</PublishReadyToRun>        

Wabbajack.Common.Test/Wabbajack.Common.Test.csproj

@@ -14,6 +14,7 @@
     </ItemGroup>
     
     <ItemGroup>
+      <ProjectReference Include="..\Wabbajack.Common.CSP\Wabbajack.Common.CSP.csproj" />
       <ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
     </ItemGroup>
 

Wabbajack.Common/Metrics.cs

@@ -1,9 +1,11 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net;
 using System.Net.Http;
 using System.Text;
 using System.Threading.Tasks;
+using Wabbajack.Common.Exceptions;
 
 namespace Wabbajack.Common
 {
@@ -43,16 +45,9 @@ namespace Wabbajack.Common
         /// <param name="value"></param>
         public static async Task Send(string action, string value)
         {
-            var client = new HttpClient();
-            try
-            {
-                client.DefaultRequestHeaders.Add(Consts.MetricsKeyHeader, await GetMetricsKey());
-                await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}");
-            }
-            catch (Exception)
-            {
-                // ignored
-            }
+            var client = new Http.Client();
+            client.Headers.Add((Consts.MetricsKeyHeader, await GetMetricsKey()));
+            await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}");
         }
     }
 }

Wabbajack.Common/Wabbajack.Common.csproj

@@ -5,6 +5,15 @@
         <Platforms>x64</Platforms>
         <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
         <Nullable>enable</Nullable>
+        <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
+        <PackageVersion>2.0.9.1</PackageVersion>
+        <Title>Wabbajack Common Lib</Title>
+        <Authors>Wabbajack Team</Authors>
+        <Copyright>2020</Copyright>
+        <PackageProjectUrl>https://github.com/wabbajack-tools/wabbajack</PackageProjectUrl>
+        <PackageLicenseUrl>https://raw.githubusercontent.com/wabbajack-tools/wabbajack/master/LICENSE.txt</PackageLicenseUrl>
+        <PackageIconUrl>https://www.wabbajack.org/favicon.ico</PackageIconUrl>
+        <RepositoryUrl>https://github.com/wabbajack-tools/wabbajack</RepositoryUrl>
     </PropertyGroup>
     <ItemGroup>
         <None Remove="7z.dll.gz" />
@@ -42,9 +51,6 @@
         <PackageReference Include="System.Security.Principal.Windows" Version="5.0.0-preview.4.20251.6" />
         <PackageReference Include="YamlDotNet" Version="8.1.2" />
     </ItemGroup>
-    <ItemGroup>
-        <ProjectReference Include="..\Wabbajack.Common.CSP\Wabbajack.Common.CSP.csproj" />
-    </ItemGroup>
     <ItemGroup>
       <Compile Update="Serialization\PrimitiveHandlers.cs">
         <DependentUpon>PrimitiveHandlers.tt</DependentUpon>

Wabbajack.Launcher/Wabbajack.Launcher.csproj

@@ -4,8 +4,8 @@
         <OutputType>WinExe</OutputType>
         <TargetFramework>netcoreapp3.1</TargetFramework>
         <UseWPF>true</UseWPF>
-        <AssemblyVersion>2.0.9.1</AssemblyVersion>
-        <FileVersion>2.0.9.1</FileVersion>
+        <AssemblyVersion>2.0.9.2</AssemblyVersion>
+        <FileVersion>2.0.9.2</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Application Launcher</Description>
         <PublishReadyToRun>true</PublishReadyToRun>

Wabbajack.Lib/ACompiler.cs

@@ -147,6 +147,8 @@ namespace Wabbajack.Lib
             await using (var of = await ModListOutputFolder.Combine("modlist").Create()) 
                 ModList.ToJson(of);
 
+            await ClientAPI.SendModListDefinition(ModList);
+
             await ModListOutputFile.DeleteAsync();
 
             await using (var fs = await ModListOutputFile.Create())

Wabbajack.Lib/ClientAPI.cs

@@ -124,5 +124,11 @@ namespace Wabbajack.Lib
                 .GetJsonAsync<Dictionary<RelativePath, Hash>>($"{Consts.WabbajackBuildServerUri}game_files/{game}/{version}");
                 */
         }
+
+        public static async Task SendModListDefinition(ModList modList)
+        {
+            var client = await GetClient();
+            await client.PostAsync($"{Consts.WabbajackBuildServerUri}list_definitions/ingest", new StringContent(modList.ToJson(), Encoding.UTF8, "application/json"));
+        }
     }
 }

Wabbajack.Lib/MO2Compiler.cs

@@ -79,7 +79,7 @@ namespace Wabbajack.Lib
 
         protected override async Task<bool> _Begin(CancellationToken cancel)
         {
-            var _ = Metrics.Send("begin_compiling", MO2Profile ?? "unknown");
+            await Metrics.Send("begin_compiling", MO2Profile ?? "unknown");
             if (cancel.IsCancellationRequested) return false;
             Queue.SetActiveThreadsObservable(ConstructDynamicNumThreads(await RecommendQueueSize()));
             UpdateTracker.Reset();

Wabbajack.Lib/MO2Installer.cs

@@ -51,7 +51,7 @@ namespace Wabbajack.Lib
         protected override async Task<bool> _Begin(CancellationToken cancel)
         {
             if (cancel.IsCancellationRequested) return false;
-            var metric = Metrics.Send(Metrics.BeginInstall, ModList.Name);
+            await Metrics.Send(Metrics.BeginInstall, ModList.Name);
             Utils.Log("Configuring Processor");
 
             Queue.SetActiveThreadsObservable(ConstructDynamicNumThreads(await RecommendQueueSize()));
@@ -171,7 +171,7 @@ namespace Wabbajack.Lib
             SetScreenSizeInPrefs();
 
             UpdateTracker.NextStep("Installation complete! You may exit the program.");
-            var metric2 = Metrics.Send(Metrics.FinishInstall, ModList.Name);
+            await Metrics.Send(Metrics.FinishInstall, ModList.Name);
 
             return true;
         }

Wabbajack.Server.Test/sql/wabbajack_db.sql

@@ -317,6 +317,16 @@ CREATE TABLE [dbo].[NexusModInfos](
 ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
 GO
 
+CREATE TABLE [dbo].[TarKey](
+    [MetricsKey] [nvarchar](64) NOT NULL,
+    CONSTRAINT [PK_TarKey] PRIMARY KEY CLUSTERED
+        (
+         [MetricsKey] ASC
+            )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+) ON [PRIMARY]
+
+GO
+
 /****** Object:  Table [dbo].[ModLists]    Script Date: 4/2/2020 3:59:19 PM ******/
 CREATE TABLE [dbo].[ModLists](
  [MachineURL] [nvarchar](50) NOT NULL,
@@ -405,6 +415,22 @@ CREATE TABLE [dbo].[Metrics](
 ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
 GO
 
+SET ANSI_NULLS ON
+GO
+SET QUOTED_IDENTIFIER ON
+GO
+CREATE TABLE [dbo].[AccessLog](
+                                [Id] [bigint] IDENTITY(1,1) NOT NULL,
+                                [Timestamp] [datetime] NOT NULL,
+                                [Action] [nvarchar](max) NOT NULL,
+                                [Ip] [nvarchar](36) NOT NULL,
+                                CONSTRAINT [PK_AccessLog] PRIMARY KEY CLUSTERED
+                                    (
+                                     [Id] ASC
+                                        )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
+GO
+
 /****** Object:  Table [dbo].[AuthoredFiles]    Script Date: 5/9/2020 2:22:00 PM ******/
 CREATE TABLE [dbo].[AuthoredFiles](
                                       [ServerAssignedUniqueId] [uniqueidentifier] NOT NULL,

Wabbajack.Server/ApiKeyAuthorizationHandler.cs

@@ -9,7 +9,10 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Newtonsoft.Json;
+using Wabbajack.Common;
+using Wabbajack.Common.Serialization.Json;
 using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
 
 
 namespace Wabbajack.BuildServer
@@ -40,6 +43,18 @@ namespace Wabbajack.BuildServer
 
         protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
         {
+            var metricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault();
+            await LogRequest(metricsKey);
+            if (metricsKey != default)
+            {
+                if (await _sql.IsTarKey(metricsKey))
+                {
+                    await _sql.IngestMetric(new Metric {Action = "TarKey", Subject = "Auth", MetricsKey = metricsKey, Timestamp = DateTime.UtcNow});
+                    await Task.Delay(TimeSpan.FromSeconds(60));
+                    throw new Exception("Error, lipsum timeout of the cross distant cloud.");
+                }
+            }
+
             if (!Request.Headers.TryGetValue(ApiKeyHeaderName, out var apiKeyHeaderValues))
             {
                 return AuthenticateResult.NoResult();
@@ -73,6 +88,24 @@ namespace Wabbajack.BuildServer
             return AuthenticateResult.Fail("Invalid API Key provided.");
         }
 
+        [JsonName("RequestLog")]
+        public class RequestLog
+        {
+            public string Path { get; set; }
+            public string Query { get; set; }
+            public Dictionary<string, string[]> Headers { get; set; }
+        }
+        private async Task LogRequest(string metricsKey)
+        {
+            var action = new RequestLog {
+                Path = Request.Path, 
+                Query = Request.QueryString.Value, 
+                Headers = Request.Headers.GroupBy(s => s.Key)
+                    .ToDictionary(s => s.Key, s => s.SelectMany(v => v.Value).ToArray())
+            };
+            await _sql.IngestAccess(Request.HttpContext.Connection.RemoteIpAddress.ToString(), action.ToJson());
+        }
+
         protected override async Task HandleChallengeAsync(AuthenticationProperties properties)
         {
             Response.StatusCode = 401;

Wabbajack.Server/Controllers/ListDefinitions.cs

@@ -0,0 +1,50 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.Services;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [ApiController]
+    [Route("/list_definitions")]
+    public class ListDefinitions : ControllerBase
+    {
+        private ILogger<ListDefinitions> _logger;
+        private SqlService _sql;
+        private DiscordWebHook _discord;
+
+        public ListDefinitions(ILogger<ListDefinitions> logger, SqlService sql, DiscordWebHook discord)
+        {
+            _logger = logger;
+            _sql = sql;
+            _discord = discord;
+        }
+
+
+        [Route("ingest")]
+        [HttpPost]
+        public async Task<IActionResult> PostIngest()
+        {
+            var user = Request.Headers[Consts.MetricsKeyHeader].First();
+            _logger.Log(LogLevel.Information, $"Ingesting Modlist Definition for {user}");
+            var modlistBytes = await Request.Body.ReadAllAsync();
+            var modlist = new MemoryStream(modlistBytes).FromJson<ModList>();
+
+            var file = AbsolutePath.EntryPoint.Combine("mod_list_definitions")
+                .Combine($"{user}_{DateTime.UtcNow.ToFileTimeUtc()}.json");
+            file.Parent.CreateDirectory();
+            await using var stream = await file.OpenWrite();
+            modlist.ToJson(stream);
+            _logger.Log(LogLevel.Information, $"Done Ingesting Modlist Definition for {user}");
+            return Accepted(0);
+        }
+        
+    }
+}

Wabbajack.Server/Controllers/ListsStatus.cs

@@ -81,6 +81,18 @@ namespace Wabbajack.BuildServer.Controllers
                 <li>{{$.Archive.Name}}</li>
                 {{/each}}
                 </ul>
+                <h3>Updated ({{updated.Count}}):</h3>
+                <ul>
+                {{each $.updated }}
+                <li>{{$.Archive.Name}}</li>
+                {{/each}}
+                </ul>
+                <h3>Updating ({{updating.Count}}):</h3>
+                <ul>
+                {{each $.updating }}
+                <li>{{$.Archive.Name}}</li>
+                {{/each}}
+                </ul>
                 <h3>Passed ({{passed.Count}}):</h3>
                 <ul>
                 {{each $.passed }}
@@ -101,7 +113,9 @@ namespace Wabbajack.BuildServer.Controllers
                 lst,
                 ago = (DateTime.UtcNow - lst.Checked).TotalMinutes,
                 failed = lst.Archives.Where(a => a.IsFailing).ToList(),
-                passed = lst.Archives.Where(a => !a.IsFailing).ToList()
+                passed = lst.Archives.Where(a => !a.IsFailing).ToList(),
+                updated = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Updated).ToList(),
+                updating = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Updating).ToList(),
             });
             return new ContentResult
             {

Wabbajack.Server/DTOs/DetailedStatus.cs

@@ -22,5 +22,7 @@ namespace Wabbajack.Server.DTOs
     {
         public bool IsFailing { get; set; }
         public Archive Archive { get; set; }
+
+        public ArchiveStatus ArchiveStatus { get; set; }
     }
 }

Wabbajack.Server/DataLayer/ApiKeys.cs

@@ -44,5 +44,12 @@ namespace Wabbajack.Server.DataLayer
         }
 
 
+        public async Task<bool> IsTarKey(string metricsKey)
+        {
+            await using var conn = await Open();
+            var result = await conn.QueryFirstOrDefaultAsync<string>(
+                "SELECT MetricsKey FROM TarKey WHERE MetricsKey = @MetricsKey", new {MetricsKey = metricsKey});
+            return result == metricsKey;
+        }
     }
 }

Wabbajack.Server/DataLayer/Metrics.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Dapper;
@@ -14,6 +15,17 @@ namespace Wabbajack.Server.DataLayer
             await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", metric);
         }
         
+        public async Task IngestAccess(string ip, string log)
+        {
+            await using var conn = await Open();
+            await conn.ExecuteAsync(@"INSERT INTO dbo.AccessLog (Timestamp, Action, Ip) VALUES (@Timestamp, @Action, @Ip)", new
+            {
+                Timestamp = DateTime.UtcNow,
+                Ip = ip,
+                Action = log
+            });
+        }
+        
         public async Task<IEnumerable<AggregateMetric>> MetricsReport(string action)
         {
             await using var conn = await Open();

Wabbajack.Server/Services/ListValidator.cs

@@ -83,7 +83,9 @@ namespace Wabbajack.Server.Services
                     MachineName = metadata.Links.MachineURL,
                     Archives = archives.Select(a => new DetailedStatusItem
                     {
-                        Archive = a.Item1, IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
+                        Archive = a.Item1, 
+                        IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating,
+                        ArchiveStatus = a.Item2
                     }).ToList()
                 };
 

Wabbajack.Server/Wabbajack.Server.csproj

@@ -3,8 +3,8 @@
     <PropertyGroup>
         <OutputType>Exe</OutputType>
         <TargetFramework>netcoreapp3.1</TargetFramework>
-        <AssemblyVersion>2.0.9.1</AssemblyVersion>
-        <FileVersion>2.0.9.1</FileVersion>
+        <AssemblyVersion>2.0.9.2</AssemblyVersion>
+        <FileVersion>2.0.9.2</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Server</Description>
         <RuntimeIdentifier>win-x64</RuntimeIdentifier>

Wabbajack.VirtualFileSystem/Context.cs

@@ -9,7 +9,6 @@ using System.Text;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
-using Wabbajack.Common.CSP;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
 using File = System.IO.File;
 using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
@@ -63,8 +62,6 @@ namespace Wabbajack.VirtualFileSystem
 
             var filesToIndex = root.EnumerateFiles().Distinct().ToList();
             
-            var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing {root}", filesToIndex.Count));
-
             var allFiles = await filesToIndex
                             .PMap(Queue, async f =>
                             {

Wabbajack/View Models/Gallery/ModListMetadataVM.cs

@@ -195,7 +195,7 @@ namespace Wabbajack
                 });
 
 
-                Task.Run(() => Metrics.Send(Metrics.Downloading, Metadata.Title))
+                Task.Run(async () => await Metrics.Send(Metrics.Downloading, Metadata.Title))
                     .FireAndForget(ex => Utils.Error(ex, "Error sending download metric"));
 
                 return await tcs.Task;

Wabbajack/View Models/MainWindowVM.cs

@@ -126,7 +126,7 @@ namespace Wabbajack
                 var fvi = FileVersionInfo.GetVersionInfo(assembly.Location);
                 VersionDisplay = $"v{fvi.FileVersion}";
                 Utils.Log($"Wabbajack Version: {fvi.FileVersion}");
-                var _ = Metrics.Send("started_wabbajack", fvi.FileVersion);
+                var tsk = Metrics.Send("started_wabbajack", fvi.FileVersion);
             }
             catch (Exception ex)
             {

Wabbajack/Wabbajack.csproj

@@ -6,8 +6,8 @@
     <UseWPF>true</UseWPF>
     <Platforms>x64</Platforms>
     <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
-      <AssemblyVersion>2.0.9.1</AssemblyVersion>
-      <FileVersion>2.0.9.1</FileVersion>
+      <AssemblyVersion>2.0.9.2</AssemblyVersion>
+      <FileVersion>2.0.9.2</FileVersion>
     <Copyright>Copyright © 2019-2020</Copyright>
     <Description>An automated ModList installer</Description>
     <PublishReadyToRun>true</PublishReadyToRun>