Fix enderal Game ids, fixes for game file indexing, fixes for bad metrics reporting (lagged by a day).

Wabbajack.BuildServer/Controllers/ModlistUpdater.cs

@@ -167,7 +167,6 @@ namespace Wabbajack.BuildServer.Controllers
 
                 if (valid)
                 {
-                    Utils.Log($"Http file {archive.Hash} is still valid");
                     return (NotFound("Http file still valid"), null);
                 }
 

Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs

@@ -19,9 +19,9 @@ namespace Wabbajack.BuildServer.Models.Jobs
         {
             using (var queue = new WorkQueue(4))
             {
-                Utils.Log($"Indexing game files");
+                Utils.Log($"Finding game files to Index game files");
                 var states = GameRegistry.Games.Values
-                    .Where(game => game.TryGetGameLocation() != null && game.MainExecutable != null)
+                    .Where(game => game.TryGetGameLocation() != default && game.MainExecutable != null)
                     .SelectMany(game => game.GameLocation().EnumerateFiles()
                         .Select(file => new GameFileSourceDownloader.State(game.InstalledVersion)
                         {
@@ -33,9 +33,9 @@ namespace Wabbajack.BuildServer.Models.Jobs
                 var pks = states.Select(s => s.PrimaryKeyString).ToHashSet();
                 Utils.Log($"Found {pks.Count} archives to cross-reference with the database");
 
-                var found = await sql.FilterByExistingPrimaryKeys(pks);
+                var notFound = await sql.FilterByExistingPrimaryKeys(pks);
                 
-                states = states.Where(s => !found.Contains(s.PrimaryKeyString)).ToList();
+                states = states.Where(s => notFound.Contains(s.PrimaryKeyString)).ToList();
                 Utils.Log($"Found {states.Count} archives to index");
 
                 await states.PMap(queue, async state =>

Wabbajack.BuildServer/Models/Sql/SqlService.cs

@@ -175,7 +175,7 @@ namespace Wabbajack.BuildServer.Model.Models
                           AND Subject != 'Default'
                           AND TRY_CONVERT(uniqueidentifier, Subject) is null) as keys
                         WHERE type = 'P'
-                        AND DATEADD(DAY, number+1, dbo.MinMetricDate()) < dbo.MaxMetricDate()) as d
+                        AND DATEADD(DAY, number+1, dbo.MinMetricDate()) <= dbo.MaxMetricDate()) as d
                         ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action
                         WHERE d.Action = @action
                         AND d.Date >= DATEADD(month, -1, GETUTCDATE())
@@ -666,9 +666,17 @@ namespace Wabbajack.BuildServer.Model.Models
         public async Task<HashSet<string>> FilterByExistingPrimaryKeys(HashSet<string> pks)
         {
             await using var conn = await Open();
-            var found = await conn.QueryAsync<string>("SELECT Hash from dbo.IndexedFile WHERE PrimaryKey in @PrimaryKeys",
+            var results = new List<string>();
-                new {PrimaryKeys = pks.ToList()});
+
-            return pks.Except(found.ToHashSet()).ToHashSet();
+            foreach (var partition in pks.Partition(512))
+            {
+                var found = await conn.QueryAsync<string>(
+                    "SELECT Hash from dbo.DownloadStates WHERE PrimaryKey in @PrimaryKeys",
+                    new {PrimaryKeys = partition.ToList()});
+                results.AddRange(found);
+            }
+
+            return pks.Except(results.ToHashSet()).ToHashSet();
         }
 
         public async Task<long> DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date)

Wabbajack.CLI/Wabbajack.CLI.csproj

@@ -17,7 +17,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="CommandLineParser" Version="2.7.82" />
+      <PackageReference Include="CommandLineParser" Version="2.8.0" />
       <PackageReference Include="F23.StringSimilarity" Version="3.1.0" />
       <PackageReference Include="Markdig" Version="0.20.0" />
     </ItemGroup>

Wabbajack.Common/GameMetaData.cs

@@ -363,7 +363,7 @@ namespace Wabbajack.Common
                     NexusName = "enderal",
                     MO2Name = "Enderal",
                     MO2ArchiveName = "enderal",
-                    SteamIDs = new List<int>{1027920},
+                    SteamIDs = new List<int>{1027920, 933480},
                     RequiredFiles = new List<string>
                     {
                         "TESV.exe"

Wabbajack.Common/Utils.cs

@@ -1183,5 +1183,21 @@ namespace Wabbajack.Common
                 Marshal.ZeroFreeGlobalAllocUnicode(valuePtr);
             }
         }
+
+        public static IEnumerable<IEnumerable<T>> Partition<T>(this IEnumerable<T> coll, int size)
+        {
+            var lst = new List<T>();
+            foreach (var itm in coll)
+            {
+                lst.Add(itm);
+                if (lst.Count != size) continue;
+
+                yield return lst;
+                lst = new List<T>();
+            }
+
+            if (lst.Count > 0 && lst.Count != size)
+                yield return lst;
+        }
     }
 }

Wabbajack.Lib/Downloaders/ManualDownloader.cs

@@ -1,4 +1,5 @@
 using System.IO;
+using System.Linq;
 using System.Reactive.Subjects;
 using System.Threading.Tasks;
 using Newtonsoft.Json;
@@ -82,7 +83,7 @@ namespace Wabbajack.Lib.Downloaders
 
             public override bool IsWhitelisted(ServerWhitelist whitelist)
             {
-                return true;
+                return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p));
             }
 
             public override async Task<bool> Download(Archive a, AbsolutePath destination)

Wabbajack.Lib/Tasks/MakeNewMO2Project.cs

@@ -0,0 +1,13 @@
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.Tasks
+{
+    public class MakeNewMO2Project
+    {
+        public static async Task Execute(AbsolutePath folder, Game game)
+        {
+            
+        }
+    }
+}

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -65,7 +65,7 @@
         <Version>1.0.0</Version>
       </PackageReference>
       <PackageReference Include="YoutubeExplode">
-        <Version>5.0.2</Version>
+        <Version>5.0.3</Version>
       </PackageReference>
     </ItemGroup>
     <ItemGroup>

Wabbajack.Test/DownloaderTests.cs

@@ -544,6 +544,7 @@ namespace Wabbajack.Test
                 Name = "Cori.7z",
                 Hash = Hash.FromBase64("gCRVrvzDNH0="),
             };
+            Utils.Log($"Getting Hash for {(long)archive.Hash}");
             Assert.True(await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, dest));
             Assert.Equal(Hash.FromBase64("gCRVrvzDNH0="), await dest.FileHashCachedAsync());
         }