diff --git a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs index af3dbd01..88b09c7d 100644 --- a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs +++ b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs @@ -167,7 +167,6 @@ namespace Wabbajack.BuildServer.Controllers if (valid) { - Utils.Log($"Http file {archive.Hash} is still valid"); return (NotFound("Http file still valid"), null); } diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs index f3012cd5..e3cdf87f 100644 --- a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs +++ b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs @@ -19,9 +19,9 @@ namespace Wabbajack.BuildServer.Models.Jobs { using (var queue = new WorkQueue(4)) { - Utils.Log($"Indexing game files"); + Utils.Log($"Finding game files to Index game files"); var states = GameRegistry.Games.Values - .Where(game => game.TryGetGameLocation() != null && game.MainExecutable != null) + .Where(game => game.TryGetGameLocation() != default && game.MainExecutable != null) .SelectMany(game => game.GameLocation().EnumerateFiles() .Select(file => new GameFileSourceDownloader.State(game.InstalledVersion) { @@ -33,9 +33,9 @@ namespace Wabbajack.BuildServer.Models.Jobs var pks = states.Select(s => s.PrimaryKeyString).ToHashSet(); Utils.Log($"Found {pks.Count} archives to cross-reference with the database"); - var found = await sql.FilterByExistingPrimaryKeys(pks); + var notFound = await sql.FilterByExistingPrimaryKeys(pks); - states = states.Where(s => !found.Contains(s.PrimaryKeyString)).ToList(); + states = states.Where(s => notFound.Contains(s.PrimaryKeyString)).ToList(); Utils.Log($"Found {states.Count} archives to index"); await states.PMap(queue, async state => diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index f3451128..3b320b67 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -175,7 +175,7 @@ namespace Wabbajack.BuildServer.Model.Models AND Subject != 'Default' AND TRY_CONVERT(uniqueidentifier, Subject) is null) as keys WHERE type = 'P' - AND DATEADD(DAY, number+1, dbo.MinMetricDate()) < dbo.MaxMetricDate()) as d + AND DATEADD(DAY, number+1, dbo.MinMetricDate()) <= dbo.MaxMetricDate()) as d ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action WHERE d.Action = @action AND d.Date >= DATEADD(month, -1, GETUTCDATE()) @@ -666,9 +666,17 @@ namespace Wabbajack.BuildServer.Model.Models public async Task> FilterByExistingPrimaryKeys(HashSet pks) { await using var conn = await Open(); - var found = await conn.QueryAsync("SELECT Hash from dbo.IndexedFile WHERE PrimaryKey in @PrimaryKeys", - new {PrimaryKeys = pks.ToList()}); - return pks.Except(found.ToHashSet()).ToHashSet(); + var results = new List(); + + foreach (var partition in pks.Partition(512)) + { + var found = await conn.QueryAsync( + "SELECT Hash from dbo.DownloadStates WHERE PrimaryKey in @PrimaryKeys", + new {PrimaryKeys = partition.ToList()}); + results.AddRange(found); + } + + return pks.Except(results.ToHashSet()).ToHashSet(); } public async Task DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date) diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 934412e0..a31c76fa 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -17,7 +17,7 @@ - + diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index 02715f42..6ba4e31c 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -363,7 +363,7 @@ namespace Wabbajack.Common NexusName = "enderal", MO2Name = "Enderal", MO2ArchiveName = "enderal", - SteamIDs = new List{1027920}, + SteamIDs = new List{1027920, 933480}, RequiredFiles = new List { "TESV.exe" diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 1124c3a3..56f048a9 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -1183,5 +1183,21 @@ namespace Wabbajack.Common Marshal.ZeroFreeGlobalAllocUnicode(valuePtr); } } + + public static IEnumerable> Partition(this IEnumerable coll, int size) + { + var lst = new List(); + foreach (var itm in coll) + { + lst.Add(itm); + if (lst.Count != size) continue; + + yield return lst; + lst = new List(); + } + + if (lst.Count > 0 && lst.Count != size) + yield return lst; + } } } diff --git a/Wabbajack.Lib/Downloaders/ManualDownloader.cs b/Wabbajack.Lib/Downloaders/ManualDownloader.cs index 805ebc45..f0457240 100644 --- a/Wabbajack.Lib/Downloaders/ManualDownloader.cs +++ b/Wabbajack.Lib/Downloaders/ManualDownloader.cs @@ -1,4 +1,5 @@ using System.IO; +using System.Linq; using System.Reactive.Subjects; using System.Threading.Tasks; using Newtonsoft.Json; @@ -82,7 +83,7 @@ namespace Wabbajack.Lib.Downloaders public override bool IsWhitelisted(ServerWhitelist whitelist) { - return true; + return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p)); } public override async Task Download(Archive a, AbsolutePath destination) diff --git a/Wabbajack.Lib/Tasks/MakeNewMO2Project.cs b/Wabbajack.Lib/Tasks/MakeNewMO2Project.cs new file mode 100644 index 00000000..9a2cc7f4 --- /dev/null +++ b/Wabbajack.Lib/Tasks/MakeNewMO2Project.cs @@ -0,0 +1,13 @@ +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.Lib.Tasks +{ + public class MakeNewMO2Project + { + public static async Task Execute(AbsolutePath folder, Game game) + { + + } + } +} diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index da89051e..66a1ed5d 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -65,7 +65,7 @@ 1.0.0 - 5.0.2 + 5.0.3 diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 3b752249..0c1bfe77 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -544,6 +544,7 @@ namespace Wabbajack.Test Name = "Cori.7z", Hash = Hash.FromBase64("gCRVrvzDNH0="), }; + Utils.Log($"Getting Hash for {(long)archive.Hash}"); Assert.True(await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, dest)); Assert.Equal(Hash.FromBase64("gCRVrvzDNH0="), await dest.FileHashCachedAsync()); }