diff --git a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs index 820c4588..126bc259 100644 --- a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs @@ -20,7 +20,7 @@ namespace Wabbajack.BuildServer.Controllers } [HttpGet] - [Route("/{xxHashAsBase64}/meta.ini")] + [Route("{xxHashAsBase64}/meta.ini")] public async Task GetFileMeta(string xxHashAsBase64) { var id = xxHashAsBase64.FromHex().ToBase64(); @@ -55,7 +55,35 @@ namespace Wabbajack.BuildServer.Controllers {"as", "ChildFiles"}, {"maxDepth", 8}, {"restrictSearchWithMatch", new BsonDocument()} - }) + }), + new BsonDocument("$project", + new BsonDocument + { + // If we return all fields some BSAs will return more that 16MB which is the + // maximum doc size that can can be returned from MongoDB + { "_id", 1 }, + { "Size", 1 }, + { "Children.Name", 1 }, + { "Children.Hash", 1 }, + { "ChildFiles._id", 1 }, + { "ChildFiles.Size", 1 }, + { "ChildFiles.Children.Name", 1 }, + { "ChildFiles.Children.Hash", 1 }, + { "ChildFiles.ChildFiles._id", 1 }, + { "ChildFiles.ChildFiles.Size", 1 }, + { "ChildFiles.ChildFiles.Children.Name", 1 }, + { "ChildFiles.ChildFiles.Children.Hash", 1 }, + { "ChildFiles.ChildFiles.ChildFiles._id", 1 }, + { "ChildFiles.ChildFiles.ChildFiles.Size", 1 }, + { "ChildFiles.ChildFiles.ChildFiles.Children.Name", 1 }, + { "ChildFiles.ChildFiles.ChildFiles.Children.Hash", 1 }, + { "ChildFiles.ChildFiles.ChildFiles.ChildFiles._id", 1 }, + { "ChildFiles.ChildFiles.ChildFiles.ChildFiles.Size", 1 }, + { "ChildFiles.ChildFiles.ChildFiles.ChildFiles.Children.Name", 1 }, + { "ChildFiles.ChildFiles.ChildFiles.ChildFiles.Children.Hash", 1 }, + { "ChildFiles.ChildFiles.ChildFiles.ChildFiles.ChildFiles._id", 1 }, + { "ChildFiles.ChildFiles.ChildFiles.ChildFiles.ChildFiles.Size", 1 } + }) }; var result = await Db.IndexedFiles.AggregateAsync(query); @@ -66,7 +94,7 @@ namespace Wabbajack.BuildServer.Controllers return null; Dictionary indexed_children = new Dictionary(); - if (t.IsArchive) + if (t.ChildFiles != null && t.ChildFiles.Count > 0) indexed_children = t.ChildFiles.ToDictionary(t => t.Hash); var file = new IndexedVirtualFile @@ -74,7 +102,7 @@ namespace Wabbajack.BuildServer.Controllers Name = Name, Size = t.Size, Hash = t.Hash, - Children = t.IsArchive + Children = t.ChildFiles != null ? t.Children.Select(child => Convert(indexed_children[child.Hash], child.Name)).ToList() : new List() }; diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index 33ebfdde..4dbe21fa 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -74,6 +74,7 @@ namespace Wabbajack.BuildServer await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.High); await ScheduledJob(TimeSpan.FromMinutes(30), Job.JobPriority.High); await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.Low); + await ScheduledJob(TimeSpan.FromHours(24), Job.JobPriority.High); await Task.Delay(10000); } } diff --git a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs index 51f99b41..792b4f71 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs @@ -15,7 +15,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue typeof(IndexJob), typeof(GetNexusUpdatesJob), typeof(UpdateModLists), - typeof(EnqueueAllArchives) + typeof(EnqueueAllArchives), + typeof(EnqueueAllGameFiles) }; public static Dictionary TypeToName { get; set; } public static Dictionary NameToType { get; set; } diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs new file mode 100644 index 00000000..8fde6e50 --- /dev/null +++ b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs @@ -0,0 +1,74 @@ +using System.Linq; +using System.Threading.Tasks; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using System.IO; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Directory = Alphaleonis.Win32.Filesystem.Directory; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.BuildServer.Models.Jobs +{ + public class EnqueueAllGameFiles : AJobPayload + { + public override string Description { get => $"Enqueue all game files for indexing"; } + public override async Task Execute(DBContext db, AppSettings settings) + { + using (var queue = new WorkQueue(4)) + { + Utils.Log($"Indexing game files"); + var states = GameRegistry.Games.Values + .Where(game => game.GameLocation() != null && game.MainExecutable != null) + .SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", SearchOption.AllDirectories) + .Select(file => new GameFileSourceDownloader.State + { + Game = game.Game, + GameVersion = game.InstalledVersion, + GameFile = file.RelativeTo(game.GameLocation()), + })) + .ToList(); + + var pks = states.Select(s => s.PrimaryKeyString).Distinct().ToArray(); + Utils.Log($"Found {pks.Length} archives to cross-reference with the database"); + + var found = (await db.DownloadStates + .AsQueryable().Where(s => pks.Contains(s.Key)) + .Select(s => s.Key) + .ToListAsync()) + .ToDictionary(s => s); + + states = states.Where(s => !found.ContainsKey(s.PrimaryKeyString)).ToList(); + Utils.Log($"Found {states.Count} archives to index"); + + await states.PMap(queue, async state => + { + var path = Path.Combine(state.Game.MetaData().GameLocation(), state.GameFile); + Utils.Log($"Hashing Game file {path}"); + try + { + state.Hash = await path.FileHashAsync(); + } + catch (IOException) + { + Utils.Log($"Unable to hash {path}"); + } + }); + + var with_hash = states.Where(state => state.Hash != null).ToList(); + Utils.Log($"Inserting {with_hash.Count} jobs."); + var jobs = states.Select(state => new IndexJob {Archive = new Archive {Name = Path.GetFileName(state.GameFile), State = state}}) + .Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus}) + .ToList(); + + if (jobs.Count > 0) + await db.Jobs.InsertManyAsync(jobs); + + return JobResult.Success(); + } + + } + } +} diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 66bec548..750573b3 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -197,7 +197,7 @@ namespace Wabbajack.Common throw ex; } } - + public static string FileHashCached(this string file, bool nullOnIOError = false) { var hashPath = file + Consts.HashFileExtension; diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index c3f2a80f..6e269aef 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -5,6 +5,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; @@ -344,31 +345,20 @@ namespace Wabbajack.Lib if (to_find.Count == 0) return; - var games = new[]{CompilingGame}.Concat(GameRegistry.Games.Values.Where(g => g != CompilingGame)); - var game_files = games - .Where(g => g.GameLocation() != null) - .SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", DirectoryEnumerationOptions.Recursive).Select(name => (game, name))) - .GroupBy(f => (Path.GetFileName(f.name), new FileInfo(f.name).Length)) - .ToDictionary(f => f.Key); - - await to_find.PMap(Queue, f => + await to_find.PMap(Queue, async f => { var vf = VFS.Index.ByFullPath[f]; - if (!game_files.TryGetValue((Path.GetFileName(f), vf.Size), out var found)) - return; + var client = new HttpClient(); + var response = + await client.GetAsync( + $"http://build.wabbajack.org/indexed_files/{vf.Hash.FromBase64().ToHex()}/meta.ini"); + + if (!response.IsSuccessStatusCode) return; - var (game, name) = found.FirstOrDefault(ff => ff.name.FileHash() == vf.Hash); - if (name == null) - return; - - File.WriteAllLines(f+".meta", new[] - { - "[General]", - $"gameName={game.MO2ArchiveName}", - $"gameFile={name.RelativeTo(game.GameLocation()).Replace("\\", "/")}" - }); + var ini_data = await response.Content.ReadAsStringAsync(); + Utils.Log($"Inferred .meta for {Path.GetFileName(vf.FullPath)}, writing to disk"); + File.WriteAllText(vf.FullPath + ".meta", ini_data); }); - }