From 5e8e2c33b5b35037136e81c71a3f465c5755fccc Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 3 Jan 2020 12:21:27 -0700 Subject: [PATCH] Use Windows FileVersion utils instead to get file versions, use that info when saving Game Downloader states. --- Wabbajack.CacheServer/JobQueueEndpoints.cs | 40 +++++++++++++++++++ Wabbajack.CacheServer/Jobs/IndexJob.cs | 2 +- Wabbajack.CacheServer/Server.cs | 13 +++++- .../Downloaders/AbstractDownloadState.cs | 3 ++ Wabbajack.sln | 36 ++++++++--------- 5 files changed, 74 insertions(+), 20 deletions(-) diff --git a/Wabbajack.CacheServer/JobQueueEndpoints.cs b/Wabbajack.CacheServer/JobQueueEndpoints.cs index a6615404..8cf23027 100644 --- a/Wabbajack.CacheServer/JobQueueEndpoints.cs +++ b/Wabbajack.CacheServer/JobQueueEndpoints.cs @@ -1,14 +1,22 @@ using System; +using System.IO; using System.Linq; using System.Linq.Expressions; using System.Security.Policy; using System.Threading.Tasks; +using Windows.Media.Playback; using MongoDB.Driver; using MongoDB.Driver.Linq; using Nancy; using Nettle; using Wabbajack.CacheServer.DTOs.JobQueue; using Wabbajack.CacheServer.Jobs; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.CompilationSteps; +using Wabbajack.Lib.Downloaders; +using Directory = Alphaleonis.Win32.Filesystem.Directory; +using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.CacheServer { @@ -18,6 +26,7 @@ namespace Wabbajack.CacheServer { Get("/", HandleListJobs); Get("/enqueue_curated_for_indexing", HandleEnqueueAllCurated); + Get("/enqueue_game_files_for_indexing", HandleEnqueueAllGameFiles); } private readonly Func HandleListJobsTemplate = NettleEngine.GetCompiler().Compile(@" @@ -70,6 +79,37 @@ namespace Wabbajack.CacheServer return $"Enqueued {states.Count} jobs"; } + private async Task HandleEnqueueAllGameFiles(object arg) + { + using (var queue = new WorkQueue(4)) + { + var states = GameRegistry.Games.Values + .Where(game => game.GameLocation() != null && game.MainExecutable != null) + .SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", SearchOption.AllDirectories) + .Select(file => new GameFileSourceDownloader.State + { + Game = game.Game, + GameVersion = game.InstalledVersion, + GameFile = file.RelativeTo(game.GameLocation()), + })) + .ToList(); + + await states.PMap(queue, state => + { + state.Hash = Path.Combine(state.Game.MetaData().GameLocation(), state.GameFile).FileHash(); + }); + + var jobs = states.Select(state => new IndexJob {Archive = new Archive {Name = Path.GetFileName(state.GameFile), State = state}}) + .Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus}) + .ToList(); + + if (jobs.Count > 0) + await Server.Config.JobQueue.Connect().InsertManyAsync(jobs); + + return $"Enqueued {states.Count} Jobs"; + } + } + public static async Task StartJobQueue() { foreach (var task in Enumerable.Range(0, 4)) diff --git a/Wabbajack.CacheServer/Jobs/IndexJob.cs b/Wabbajack.CacheServer/Jobs/IndexJob.cs index c8777dc3..04bb483e 100644 --- a/Wabbajack.CacheServer/Jobs/IndexJob.cs +++ b/Wabbajack.CacheServer/Jobs/IndexJob.cs @@ -34,7 +34,7 @@ namespace Wabbajack.CacheServer.Jobs string fileName = Archive.Name; string folder = Guid.NewGuid().ToString(); - Utils.Log($"Indexer is downloading ${fileName}"); + Utils.Log($"Indexer is downloading {fileName}"); var downloadDest = Path.Combine(Server.Config.Indexer.DownloadDir, folder, fileName); await Archive.State.Download(downloadDest); diff --git a/Wabbajack.CacheServer/Server.cs b/Wabbajack.CacheServer/Server.cs index edf64462..404ed688 100644 --- a/Wabbajack.CacheServer/Server.cs +++ b/Wabbajack.CacheServer/Server.cs @@ -54,7 +54,8 @@ namespace Wabbajack.CacheServer { ctx.Response.WithHeader("Access-Control-Allow-Origin", "*") .WithHeader("Access-Control-Allow-Methods", "POST, GET") - .WithHeader("Access-Control-Allow-Headers", "Accept, Origin, Content-type"); + .WithHeader("Access-Control-Allow-Headers", "Accept, Origin, Content-type") + .WithHeader("Cache-Control","no-store"); }); } @@ -64,5 +65,15 @@ namespace Wabbajack.CacheServer enabled: true, displayErrorTraces: true); } + + protected override void ConfigureApplicationContainer(TinyIoCContainer container) + { + container.Register(); + container.Register(); + container.Register(); + container.Register(); + container.Register(); + container.Register(); + } } } diff --git a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs index dbaad27e..8b6a9cba 100644 --- a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs +++ b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs @@ -51,6 +51,9 @@ namespace Wabbajack.Lib.Downloaders public async Task Download(string destination) { + if (!Directory.Exists(Path.GetDirectoryName(destination))) + Directory.CreateDirectory(Path.GetDirectoryName(destination)); + await Download(new Archive {Name = Path.GetFileName(destination)}, destination); } diff --git a/Wabbajack.sln b/Wabbajack.sln index 8bca4a6c..3eaf21ee 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -63,10 +63,10 @@ Global {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Debug|x86.Build.0 = Debug|Any CPU {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|Any CPU.ActiveCfg = Release|x64 {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|Any CPU.Build.0 = Release|x64 - {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|x64.ActiveCfg = Release|Any CPU - {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|x64.Build.0 = Release|Any CPU {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|x86.ActiveCfg = Release|Any CPU {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|x86.Build.0 = Release|Any CPU + {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|x64.ActiveCfg = Release|x64 + {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|x64.Build.0 = Release|x64 {33602679-8484-40C7-A10C-774DFF5D8314}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug (no commandargs)|x64 {33602679-8484-40C7-A10C-774DFF5D8314}.Debug (no commandargs)|Any CPU.Build.0 = Debug (no commandargs)|x64 {33602679-8484-40C7-A10C-774DFF5D8314}.Debug (no commandargs)|x64.ActiveCfg = Debug (no commandargs)|x64 @@ -99,10 +99,10 @@ Global {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Debug|x86.Build.0 = Debug|Any CPU {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|Any CPU.ActiveCfg = Release|x64 {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|Any CPU.Build.0 = Release|x64 - {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|x64.ActiveCfg = Release|Any CPU - {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|x64.Build.0 = Release|Any CPU {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|x86.ActiveCfg = Release|Any CPU {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|x86.Build.0 = Release|Any CPU + {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|x64.ActiveCfg = Release|x64 + {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|x64.Build.0 = Release|x64 {A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|x64 {A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Debug (no commandargs)|Any CPU.Build.0 = Debug|x64 {A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Debug (no commandargs)|x64.ActiveCfg = Debug|x64 @@ -117,10 +117,10 @@ Global {A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Debug|x86.Build.0 = Debug|x86 {A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Release|Any CPU.ActiveCfg = Release|x64 {A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Release|Any CPU.Build.0 = Release|x64 - {A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Release|x64.ActiveCfg = Release|Any CPU - {A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Release|x64.Build.0 = Release|Any CPU {A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Release|x86.ActiveCfg = Release|x86 {A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Release|x86.Build.0 = Release|x86 + {A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Release|x64.ActiveCfg = Release|x64 + {A47FFF32-782B-4D9F-8704-C98FB32FA8CC}.Release|x64.Build.0 = Release|x64 {0A820830-A298-497D-85E0-E9A89EFEF5FE}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|x64 {0A820830-A298-497D-85E0-E9A89EFEF5FE}.Debug (no commandargs)|Any CPU.Build.0 = Debug|x64 {0A820830-A298-497D-85E0-E9A89EFEF5FE}.Debug (no commandargs)|x64.ActiveCfg = Debug|x64 @@ -135,10 +135,10 @@ Global {0A820830-A298-497D-85E0-E9A89EFEF5FE}.Debug|x86.Build.0 = Debug|Any CPU {0A820830-A298-497D-85E0-E9A89EFEF5FE}.Release|Any CPU.ActiveCfg = Release|x64 {0A820830-A298-497D-85E0-E9A89EFEF5FE}.Release|Any CPU.Build.0 = Release|x64 - {0A820830-A298-497D-85E0-E9A89EFEF5FE}.Release|x64.ActiveCfg = Release|Any CPU - {0A820830-A298-497D-85E0-E9A89EFEF5FE}.Release|x64.Build.0 = Release|Any CPU {0A820830-A298-497D-85E0-E9A89EFEF5FE}.Release|x86.ActiveCfg = Release|Any CPU {0A820830-A298-497D-85E0-E9A89EFEF5FE}.Release|x86.Build.0 = Release|Any CPU + {0A820830-A298-497D-85E0-E9A89EFEF5FE}.Release|x64.ActiveCfg = Release|x64 + {0A820830-A298-497D-85E0-E9A89EFEF5FE}.Release|x64.Build.0 = Release|x64 {9C004392-571A-4D28-A9F6-0E25115E6727}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|x64 {9C004392-571A-4D28-A9F6-0E25115E6727}.Debug (no commandargs)|Any CPU.Build.0 = Debug|x64 {9C004392-571A-4D28-A9F6-0E25115E6727}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU @@ -153,10 +153,10 @@ Global {9C004392-571A-4D28-A9F6-0E25115E6727}.Debug|x86.Build.0 = Debug|Any CPU {9C004392-571A-4D28-A9F6-0E25115E6727}.Release|Any CPU.ActiveCfg = Release|x64 {9C004392-571A-4D28-A9F6-0E25115E6727}.Release|Any CPU.Build.0 = Release|x64 - {9C004392-571A-4D28-A9F6-0E25115E6727}.Release|x64.ActiveCfg = Release|Any CPU - {9C004392-571A-4D28-A9F6-0E25115E6727}.Release|x64.Build.0 = Release|Any CPU {9C004392-571A-4D28-A9F6-0E25115E6727}.Release|x86.ActiveCfg = Release|Any CPU {9C004392-571A-4D28-A9F6-0E25115E6727}.Release|x86.Build.0 = Release|Any CPU + {9C004392-571A-4D28-A9F6-0E25115E6727}.Release|x64.ActiveCfg = Release|x64 + {9C004392-571A-4D28-A9F6-0E25115E6727}.Release|x64.Build.0 = Release|x64 {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|x64 {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Debug (no commandargs)|Any CPU.Build.0 = Debug|x64 {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU @@ -171,10 +171,10 @@ Global {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Debug|x86.Build.0 = Debug|Any CPU {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|Any CPU.ActiveCfg = Release|x64 {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|Any CPU.Build.0 = Release|x64 - {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x64.ActiveCfg = Release|Any CPU - {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x64.Build.0 = Release|Any CPU {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x86.ActiveCfg = Release|Any CPU {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x86.Build.0 = Release|Any CPU + {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x64.ActiveCfg = Release|x64 + {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x64.Build.0 = Release|x64 {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|x64 {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|Any CPU.Build.0 = Debug|x64 {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU @@ -189,10 +189,10 @@ Global {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|x86.Build.0 = Debug|Any CPU {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|Any CPU.ActiveCfg = Release|x64 {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|Any CPU.Build.0 = Release|x64 - {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x64.ActiveCfg = Release|Any CPU - {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x64.Build.0 = Release|Any CPU {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x86.ActiveCfg = Release|Any CPU {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x86.Build.0 = Release|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x64.ActiveCfg = Release|x64 + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x64.Build.0 = Release|x64 {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|x64 {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|Any CPU.Build.0 = Debug|x64 {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU @@ -207,10 +207,10 @@ Global {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|x86.Build.0 = Debug|Any CPU {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|Any CPU.ActiveCfg = Release|x64 {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|Any CPU.Build.0 = Release|x64 - {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x64.ActiveCfg = Release|Any CPU - {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x64.Build.0 = Release|Any CPU {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x86.ActiveCfg = Release|Any CPU {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x86.Build.0 = Release|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x64.ActiveCfg = Release|x64 + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x64.Build.0 = Release|x64 {BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|x64 {BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug (no commandargs)|Any CPU.Build.0 = Debug|x64 {BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU @@ -243,10 +243,10 @@ Global {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Debug|x86.Build.0 = Debug|Any CPU {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|Any CPU.ActiveCfg = Release|x64 {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|Any CPU.Build.0 = Release|x64 - {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.ActiveCfg = Release|Any CPU - {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.Build.0 = Release|Any CPU {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x86.ActiveCfg = Release|Any CPU {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x86.Build.0 = Release|Any CPU + {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.ActiveCfg = Release|x64 + {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE