Bunch of small server-side fixes, LZ4 compression speed, and better list validation

Compression.BSA/BSABuilder.cs

@@ -280,7 +280,7 @@ namespace Compression.BSA
                 case VersionType.SSE:
                 {
                     var r = new MemoryStream();
-                    await using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L12_MAX}, true))
+                    await using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L08_HC}, true))
                     {
                         await _srcData.CopyToWithStatusAsync(_srcData.Length, w, $"Compressing {_path}");
                     }

Wabbajack.App.Test/Wabbajack.App.Test.csproj

@@ -7,7 +7,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.0" />       
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />       
         <PackageReference Include="xunit" Version="2.4.1" />
         <PackageReference Include="xunit.runner.console" Version="2.4.1" />
         <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />

Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj

@@ -7,7 +7,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.0" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
         <PackageReference Include="xunit" Version="2.4.1" />
         <PackageReference Include="Xunit.Priority" Version="1.1.6" />
         <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />

Wabbajack.BuildServer/BackendServices/ValidateNonNexusArchives.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Linq;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Identity;
+using Wabbajack.BuildServer.Controllers;
 using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.Common;
 using Wabbajack.Lib.Downloaders;
@@ -24,7 +26,18 @@ namespace Wabbajack.BuildServer.BackendServices
             {
                 try
                 {
-                    var isValid = await archive.State.Verify(archive);
+                    bool isValid;
+                    switch (archive.State)
+                    {
+                        case GoogleDriveDownloader.State _:
+                        case ManualDownloader.State _:
+                        case HTTPDownloader.State s when new Uri(s.Url).Host.StartsWith("wabbajackpush"):
+                            isValid = true;
+                            break;
+                        default:
+                            isValid = await archive.State.Verify(archive);
+                            break;
+                    }
                     return (Archive: archive, IsValid: isValid);
                 }
                 catch (Exception ex)

Wabbajack.BuildServer/Controllers/Heartbeat.cs

@@ -44,7 +44,8 @@ namespace Wabbajack.BuildServer.Controllers
             return Ok(new HeartbeatResult
             {
                 Uptime = DateTime.Now - _startTime,
-                LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync
+                LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync,
+                LastListValidation = DateTime.UtcNow - ListValidation.SummariesLastChecked
             });
         }
 
@@ -53,6 +54,8 @@ namespace Wabbajack.BuildServer.Controllers
         {
             public TimeSpan Uptime { get; set; }
             public TimeSpan LastNexusUpdate { get; set; }
+            
+            public TimeSpan LastListValidation { get; set; }
         }
 
         [HttpGet("only-authenticated")]

Wabbajack.BuildServer/Controllers/ListValidation.cs

@@ -17,6 +17,7 @@ using Wabbajack.Common;
 using Wabbajack.Lib;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.ModListRegistry;
+using Wabbajack.Lib.NexusApi;
 
 namespace Wabbajack.BuildServer.Controllers
 {
@@ -24,7 +25,7 @@ namespace Wabbajack.BuildServer.Controllers
     [Route("/lists")]
     public class ListValidation : AControllerBase<ListValidation>
     {
-        enum ArchiveStatus
+        public enum ArchiveStatus
         {
             Valid,
             InValid,
@@ -37,6 +38,8 @@ namespace Wabbajack.BuildServer.Controllers
             _updater = new ModlistUpdater(null, sql, settings);
             _settings = settings;
             Cache = cache;
+            _nexusClient = NexusApiClient.Get();
+
         }
 
         public static IMemoryCache Cache { get; set; }
@@ -44,76 +47,97 @@ namespace Wabbajack.BuildServer.Controllers
 
         public static void ResetCache()
         {
-            Cache?.Remove(ModListSummariesKey);
+            SummariesLastChecked = DateTime.UnixEpoch;
+            ModListSummaries = null;
         }
 
+        private static IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> ModListSummaries = null;
+        public static DateTime SummariesLastChecked = DateTime.UnixEpoch;
+        private static AsyncLock UpdateLock = new AsyncLock();
         public async Task<IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)>> GetSummaries()
         {
-            
+            static bool TimesUp()
-            if (Cache.TryGetValue(ModListSummariesKey, out object result))
             {
-                return (IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)>)result;
+                return DateTime.UtcNow - SummariesLastChecked > TimeSpan.FromMinutes(5);
             }
             
-            
+            if (ModListSummaries != null && !TimesUp())
-            var data = await SQL.GetValidationData();
-            
-            using var queue = new WorkQueue();
-
-            var results = await data.ModLists.PMap(queue, async list =>
             {
-                var (metadata, modList) = list;
+                return ModListSummaries;
-                var archives = await modList.Archives.PMap(queue, async archive =>
+            }
+
+            var task = Task.Run(async () =>
+            {
+                using var _ = await UpdateLock.WaitAsync();
+                if (ModListSummaries != null && !TimesUp())
                 {
-                    var (_, result) = ValidateArchive(data, archive);
+                    return ModListSummaries;
-                    if (result == ArchiveStatus.InValid)
+                }
+                SummariesLastChecked = DateTime.UtcNow;
+
+                
+                var data = await SQL.GetValidationData();
+
+                using var queue = new WorkQueue();
+
+                var results = await data.ModLists.PMap(queue, async list =>
+                {
+                    var (metadata, modList) = list;
+                    var archives = await modList.Archives.PMap(queue, async archive =>
                     {
-                        var fixResult = await TryToFix(data, archive);
+                        var (_, result) = await ValidateArchive(data, archive);
+                        if (result != ArchiveStatus.InValid) return (archive, result);
 
-                        return fixResult;
+                        return await TryToFix(data, archive);
 
-                    }
+                    });
 
-                    return (archive, result);
+                    var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);
+                    var passCount = archives.Count(f =>
+                        f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated);
+                    var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating);
+
+                    var summary = new ModListSummary
+                    {
+                        Checked = DateTime.UtcNow,
+                        Failed = failedCount,
+                        Passed = passCount,
+                        Updating = updatingCount,
+                        MachineURL = metadata.Links.MachineURL,
+                        Name = metadata.Title,
+                    };
+
+                    var detailed = new DetailedStatus
+                    {
+                        Name = metadata.Title,
+                        Checked = DateTime.UtcNow,
+                        DownloadMetaData = metadata.DownloadMetadata,
+                        HasFailures = failedCount > 0,
+                        MachineName = metadata.Links.MachineURL,
+                        Archives = archives.Select(a => new DetailedStatusItem
+                        {
+                            Archive = a.Item1,
+                            IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
+                        }).ToList()
+                    };
+
+                    return (summary, detailed);
                 });
 
-                var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);
-                var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated);
-                var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating);
 
-                var summary =  new ModListSummary
+                var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1));
-                {
+                Cache.Set(ModListSummariesKey, results, cacheOptions);
-                    Checked = DateTime.UtcNow,
-                    Failed = failedCount,
-                    Passed = passCount,
-                    Updating = updatingCount,
-                    MachineURL = metadata.Links.MachineURL,
-                    Name = metadata.Title,
-                };
                 
-                var detailed = new DetailedStatus
+                ModListSummaries = results;
-                {
+                return results;
-                    Name = metadata.Title,
-                    Checked = DateTime.UtcNow,
-                    DownloadMetaData = metadata.DownloadMetadata,
-                    HasFailures = failedCount > 0,
-                    MachineName = metadata.Links.MachineURL,
-                    Archives = archives.Select(a => new DetailedStatusItem
-                    {
-                        Archive = a.Item1, IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
-                    }).ToList()
-                };
-
-                return (summary, detailed);
             });
-
+            var data = ModListSummaries;
-            
+            if (data == null)
-            var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1));
+                return await task;
-            Cache.Set(ModListSummariesKey, results, cacheOptions);
+            return data;
-            return results;
         }
 
-        private static (Archive archive, ArchiveStatus) ValidateArchive(SqlService.ValidationData data, Archive archive)
+        private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(SqlService.ValidationData data, Archive archive)
         {
             switch (archive.State)
             {
@@ -123,8 +147,10 @@ namespace Wabbajack.BuildServer.Controllers
                 case NexusDownloader.State nexusState when data.NexusFiles.Contains((
                     nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
                     return (archive, ArchiveStatus.Valid);
-                case NexusDownloader.State _:
+                case NexusDownloader.State ns:
-                    return (archive, ArchiveStatus.InValid);
+                    return (archive, await FastNexusModStats(ns));
+                case HTTPDownloader.State s when new Uri(s.Url).Host.StartsWith("wabbajackpush"):
+                    return (archive, ArchiveStatus.Valid);
                 case ManualDownloader.State _:
                     return (archive, ArchiveStatus.Valid);
                 default:
@@ -140,6 +166,47 @@ namespace Wabbajack.BuildServer.Controllers
             }
         }
 
+        private async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
+        {
+            
+            var mod = await SQL.GetNexusModInfoString(ns.Game, ns.ModID);
+            var files = await SQL.GetModFiles(ns.Game, ns.ModID);
+
+            if (mod == null)
+            {
+                Utils.Log($"Found missing Nexus mod info {ns.Game} {ns.ModID}");
+                mod = await (await _nexusClient).GetModInfo(ns.Game, ns.ModID, false);
+                try
+                {
+                    await SQL.AddNexusModInfo(ns.Game, ns.ModID, mod.updated_time, mod);
+                }
+                catch (Exception _)
+                {
+                    // Could be a PK constraint failure
+                }
+            }
+
+            if (files == null)
+            {
+                Utils.Log($"Found missing Nexus mod file infos {ns.Game} {ns.ModID}");
+                files = await (await _nexusClient).GetModFiles(ns.Game, ns.ModID, false);
+
+                try
+                {
+                    await SQL.AddNexusModFiles(ns.Game, ns.ModID, mod.updated_time, files);
+                }
+                catch (Exception _)
+                {
+                    // Could be a PK constraint failure
+                }
+            }
+
+            if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID))
+                return ArchiveStatus.Valid;
+            return ArchiveStatus.InValid;
+
+        }
+
         private static AsyncLock _findPatchLock = new AsyncLock();
         private async Task<(Archive, ArchiveStatus)> TryToFix(SqlService.ValidationData data, Archive archive)
         {
@@ -219,6 +286,7 @@ namespace Wabbajack.BuildServer.Controllers
 
         private AppSettings _settings;
         private ModlistUpdater _updater;
+        private Task<NexusApiClient> _nexusClient;
 
         [HttpGet]
         [Route("status/{Name}.html")]

Wabbajack.BuildServer/Controllers/ModlistUpdater.cs

@@ -88,14 +88,12 @@ namespace Wabbajack.BuildServer.Controllers
         public async Task<IActionResult> GetAlternative(string xxHash)
         {
             var startingHash = Hash.FromHex(xxHash);
-            Utils.Log($"Alternative requested for {startingHash}");
             await Metric("requested_upgrade", startingHash.ToString());
 
             var archive = await SQL.GetStateByHash(startingHash);
 
             if (archive == null)
             {
-                Utils.Log($"No original state for {startingHash}");
                 return NotFound("Original state not found");
             }
 
@@ -120,7 +118,6 @@ namespace Wabbajack.BuildServer.Controllers
             }
 
 
-            Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}");
             if (newArchive.Hash == Hash.Empty)
             {
                 await SQL.EnqueueJob(new Job
@@ -138,6 +135,7 @@ namespace Wabbajack.BuildServer.Controllers
                         }
                     }
                 });
+                Utils.Log($"Enqueued Index and Upgrade for {startingHash} -> {newArchive.State.PrimaryKeyString}");
                 return Accepted("Enqueued for Processing");
             }
 
@@ -155,6 +153,7 @@ namespace Wabbajack.BuildServer.Controllers
                         DestPK = newArchive.State.PrimaryKeyString
                     }
                 });
+                Utils.Log($"Enqueued Upgrade for {startingHash} -> {newArchive.State.PrimaryKeyString}");
             }
             return Ok(newArchive.ToJson());
         }

Wabbajack.BuildServer/Wabbajack.BuildServer.csproj

@@ -18,7 +18,7 @@
     <ItemGroup>
         <PackageReference Include="CsvHelper" Version="15.0.5" />
         <PackageReference Include="Dapper" Version="2.0.35" />
-        <PackageReference Include="FluentFTP" Version="32.3.3" />
+        <PackageReference Include="FluentFTP" Version="32.4.0" />
         <PackageReference Include="graphiql" Version="2.0.0" />
         <PackageReference Include="GraphQL" Version="3.0.0-preview-1352" />
         <PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
@@ -28,7 +28,7 @@
         <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
         <PackageReference Include="Microsoft.OpenApi" Version="1.1.4" />
         <PackageReference Include="Nettle" Version="1.3.0" />
-        <PackageReference Include="Swashbuckle.AspNetCore" Version="5.3.2" />
+        <PackageReference Include="Swashbuckle.AspNetCore" Version="5.4.0" />
         <PackageReference Include="System.Data.SqlClient" Version="4.8.1" />
     </ItemGroup>
 

Wabbajack.Common/Wabbajack.Common.csproj

@@ -32,7 +32,7 @@
         <PackageReference Include="Microsoft.Win32.Registry" Version="4.7.0" />
         <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
         <PackageReference Include="Octodiff" Version="1.2.1" />
-        <PackageReference Include="ReactiveUI" Version="11.3.1" />
+        <PackageReference Include="ReactiveUI" Version="11.3.8" />
         <PackageReference Include="SharpZipLib" Version="1.2.0" />
         <PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
         <PackageReference Include="System.Net.Http" Version="4.3.4" />

Wabbajack.Lib/Downloaders/MEGADownloader.cs

@@ -96,27 +96,30 @@ namespace Wabbajack.Lib.Downloaders
 
             private static MegaApiClient MegaApiClient => DownloadDispatcher.GetInstance<MegaDownloader>().MegaApiClient;
 
-            private static void MegaLogin()
+            private static AsyncLock _loginLock = new AsyncLock();
+            private static async Task MegaLogin()
             {
+                using var _ = await _loginLock.WaitAsync();
+
                 if (MegaApiClient.IsLoggedIn)
                     return;
 
                 if (!Utils.HaveEncryptedJson(DataName))
                 {
                     Utils.Status("Logging into MEGA (as anonymous)");
-                    MegaApiClient.LoginAnonymous();
+                    await MegaApiClient.LoginAnonymousAsync();
                 }
                 else
                 {
                     Utils.Status("Logging into MEGA with saved credentials.");
                     var authInfo = Utils.FromEncryptedJson<MegaApiClient.AuthInfos>(DataName);
-                    MegaApiClient.Login(authInfo);
+                    await MegaApiClient.LoginAsync(authInfo);
                 }
             }
 
             public override async Task<bool> Download(Archive a, AbsolutePath destination)
             {
-                MegaLogin();
+                await MegaLogin();
 
                 var fileLink = new Uri(Url);
                 Utils.Status($"Downloading MEGA file: {a.Name}");
@@ -126,7 +129,7 @@ namespace Wabbajack.Lib.Downloaders
 
             public override async Task<bool> Verify(Archive a)
             {
-                MegaLogin();
+                await MegaLogin();
 
                 var fileLink = new Uri(Url);
                 try

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -35,10 +35,10 @@
         <Version>2.1.0</Version>
       </PackageReference>
       <PackageReference Include="ReactiveUI">
-        <Version>11.3.1</Version>
+        <Version>11.3.8</Version>
       </PackageReference>
       <PackageReference Include="ReactiveUI.Fody">
-        <Version>11.3.1</Version>
+        <Version>11.3.8</Version>
       </PackageReference>
       <PackageReference Include="SharpCompress">
         <Version>0.25.0</Version>
@@ -65,7 +65,7 @@
         <Version>1.0.0</Version>
       </PackageReference>
       <PackageReference Include="YoutubeExplode">
-        <Version>5.0.1</Version>
+        <Version>5.0.2</Version>
       </PackageReference>
     </ItemGroup>
     <ItemGroup>

Wabbajack.Test/Wabbajack.Test.csproj

@@ -27,7 +27,7 @@
   <ItemGroup>
     <PackageReference Include="CefSharp.Common" Version="79.1.360" />
     <PackageReference Include="CefSharp.OffScreen" Version="79.1.360" />
-      <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.0" />
+      <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
       <PackageReference Include="xunit" Version="2.4.1" />
       <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
       <PackageReference Include="coverlet.collector" Version="1.2.1" />

Wabbajack/Wabbajack.csproj

@@ -57,7 +57,7 @@
 
   <ItemGroup>
     <PackageReference Include="CefSharp.Wpf" Version="79.1.360" />
-    <PackageReference Include="DynamicData" Version="6.14.10" />
+    <PackageReference Include="DynamicData" Version="6.14.14" />
     <PackageReference Include="Extended.Wpf.Toolkit" Version="3.8.1" />
     <PackageReference Include="Fody" Version="6.1.1">
       <PrivateAssets>all</PrivateAssets>
@@ -72,9 +72,9 @@
     <PackageReference Include="MahApps.Metro.IconPacks" Version="4.0.0" />
     <PackageReference Include="PInvoke.Gdi32" Version="0.6.6" />
     <PackageReference Include="PInvoke.User32" Version="0.6.6" />
-    <PackageReference Include="ReactiveUI" Version="11.3.1" />
+    <PackageReference Include="ReactiveUI" Version="11.3.8" />
-    <PackageReference Include="ReactiveUI.Fody" Version="11.3.1" />
+    <PackageReference Include="ReactiveUI.Fody" Version="11.3.8" />
-    <PackageReference Include="ReactiveUI.WPF" Version="11.3.1" />
+    <PackageReference Include="ReactiveUI.WPF" Version="11.3.8" />
     <PackageReference Include="SharpDX.DXGI" Version="4.2.0" />
     <PackageReference Include="WindowsAPICodePack-Shell" Version="1.1.1" />
     <PackageReference Include="WPFThemes.DarkBlend" Version="1.0.8" />