Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

WJ can now infer metas with the help of the build server

Browse Source
This commit is contained in:
Timothy Baldridge 2020-01-11 22:00:41 -07:00
parent 4e4cd49a89
commit 7df6d06e98
6 changed files with 121 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
Wabbajack.BuildServer/Controllers/IndexedFiles.cs
View File

@ -20,7 +20,7 @@ namespace Wabbajack.BuildServer.Controllers
}
[HttpGet]
[Route("/{xxHashAsBase64}/meta.ini")]
[Route("{xxHashAsBase64}/meta.ini")]
public async Task<IActionResult> GetFileMeta(string xxHashAsBase64)
{
var id = xxHashAsBase64.FromHex().ToBase64();
@ -55,7 +55,35 @@ namespace Wabbajack.BuildServer.Controllers
{"as", "ChildFiles"},
{"maxDepth", 8},
{"restrictSearchWithMatch", new BsonDocument()}
})
}),
new BsonDocument("$project",
new BsonDocument
{
// If we return all fields some BSAs will return more that 16MB which is the
// maximum doc size that can can be returned from MongoDB
{ "_id", 1 },
{ "Size", 1 },
{ "Children.Name", 1 },
{ "Children.Hash", 1 },
{ "ChildFiles._id", 1 },
{ "ChildFiles.Size", 1 },
{ "ChildFiles.Children.Name", 1 },
{ "ChildFiles.Children.Hash", 1 },
{ "ChildFiles.ChildFiles._id", 1 },
{ "ChildFiles.ChildFiles.Size", 1 },
{ "ChildFiles.ChildFiles.Children.Name", 1 },
{ "ChildFiles.ChildFiles.Children.Hash", 1 },
{ "ChildFiles.ChildFiles.ChildFiles._id", 1 },
{ "ChildFiles.ChildFiles.ChildFiles.Size", 1 },
{ "ChildFiles.ChildFiles.ChildFiles.Children.Name", 1 },
{ "ChildFiles.ChildFiles.ChildFiles.Children.Hash", 1 },
{ "ChildFiles.ChildFiles.ChildFiles.ChildFiles._id", 1 },
{ "ChildFiles.ChildFiles.ChildFiles.ChildFiles.Size", 1 },
{ "ChildFiles.ChildFiles.ChildFiles.ChildFiles.Children.Name", 1 },
{ "ChildFiles.ChildFiles.ChildFiles.ChildFiles.Children.Hash", 1 },
{ "ChildFiles.ChildFiles.ChildFiles.ChildFiles.ChildFiles._id", 1 },
{ "ChildFiles.ChildFiles.ChildFiles.ChildFiles.ChildFiles.Size", 1 }
})
};
var result = await Db.IndexedFiles.AggregateAsync<TreeResult>(query);
@ -66,7 +94,7 @@ namespace Wabbajack.BuildServer.Controllers
return null;
Dictionary<string, TreeResult> indexed_children = new Dictionary<string, TreeResult>();
if (t.IsArchive)
if (t.ChildFiles != null && t.ChildFiles.Count > 0)
indexed_children = t.ChildFiles.ToDictionary(t => t.Hash);
var file = new IndexedVirtualFile
@ -74,7 +102,7 @@ namespace Wabbajack.BuildServer.Controllers
Name = Name,
Size = t.Size,
Hash = t.Hash,
Children = t.IsArchive
Children = t.ChildFiles != null
? t.Children.Select(child => Convert(indexed_children[child.Hash], child.Name)).ToList()
: new List<IndexedVirtualFile>()
};

1
Wabbajack.BuildServer/JobManager.cs
View File

@ -74,6 +74,7 @@ namespace Wabbajack.BuildServer
await ScheduledJob<GetNexusUpdatesJob>(TimeSpan.FromHours(2), Job.JobPriority.High);
await ScheduledJob<UpdateModLists>(TimeSpan.FromMinutes(30), Job.JobPriority.High);
await ScheduledJob<EnqueueAllArchives>(TimeSpan.FromHours(2), Job.JobPriority.Low);
await ScheduledJob<EnqueueAllGameFiles>(TimeSpan.FromHours(24), Job.JobPriority.High);
await Task.Delay(10000);
}
}

3
Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs
View File

@ -15,7 +15,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue
typeof(IndexJob),
typeof(GetNexusUpdatesJob),
typeof(UpdateModLists),
typeof(EnqueueAllArchives)
typeof(EnqueueAllArchives),
typeof(EnqueueAllGameFiles)
};
public static Dictionary<Type, string> TypeToName { get; set; }
public static Dictionary<string, Type> NameToType { get; set; }

74
Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs Normal file
View File

@ -0,0 +1,74 @@
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using System.IO;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class EnqueueAllGameFiles : AJobPayload
{
public override string Description { get => $"Enqueue all game files for indexing"; }
public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
{
using (var queue = new WorkQueue(4))
{
Utils.Log($"Indexing game files");
var states = GameRegistry.Games.Values
.Where(game => game.GameLocation() != null && game.MainExecutable != null)
.SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", SearchOption.AllDirectories)
.Select(file => new GameFileSourceDownloader.State
{
Game = game.Game,
GameVersion = game.InstalledVersion,
GameFile = file.RelativeTo(game.GameLocation()),
}))
.ToList();
var pks = states.Select(s => s.PrimaryKeyString).Distinct().ToArray();
Utils.Log($"Found {pks.Length} archives to cross-reference with the database");
var found = (await db.DownloadStates
.AsQueryable().Where(s => pks.Contains(s.Key))
.Select(s => s.Key)
.ToListAsync())
.ToDictionary(s => s);
states = states.Where(s => !found.ContainsKey(s.PrimaryKeyString)).ToList();
Utils.Log($"Found {states.Count} archives to index");
await states.PMap(queue, async state =>
{
var path = Path.Combine(state.Game.MetaData().GameLocation(), state.GameFile);
Utils.Log($"Hashing Game file {path}");
try
{
state.Hash = await path.FileHashAsync();
}
catch (IOException)
{
Utils.Log($"Unable to hash {path}");
}
});
var with_hash = states.Where(state => state.Hash != null).ToList();
Utils.Log($"Inserting {with_hash.Count} jobs.");
var jobs = states.Select(state => new IndexJob {Archive = new Archive {Name = Path.GetFileName(state.GameFile), State = state}})
.Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus})
.ToList();
if (jobs.Count > 0)
await db.Jobs.InsertManyAsync(jobs);
return JobResult.Success();
}
}
}
}

2
Wabbajack.Common/Utils.cs
View File

@ -197,7 +197,7 @@ namespace Wabbajack.Common
throw ex;
}
}
public static string FileHashCached(this string file, bool nullOnIOError = false)
{
var hashPath = file + Consts.HashFileExtension;

32
Wabbajack.Lib/MO2Compiler.cs
View File

@ -5,6 +5,7 @@ using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
@ -344,31 +345,20 @@ namespace Wabbajack.Lib
if (to_find.Count == 0) return;
var games = new[]{CompilingGame}.Concat(GameRegistry.Games.Values.Where(g => g != CompilingGame));
var game_files = games
.Where(g => g.GameLocation() != null)
.SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", DirectoryEnumerationOptions.Recursive).Select(name => (game, name)))
.GroupBy(f => (Path.GetFileName(f.name), new FileInfo(f.name).Length))
.ToDictionary(f => f.Key);
await to_find.PMap(Queue, f =>
await to_find.PMap(Queue, async f =>
{
var vf = VFS.Index.ByFullPath[f];
if (!game_files.TryGetValue((Path.GetFileName(f), vf.Size), out var found))
return;
var client = new HttpClient();
var response =
await client.GetAsync(
$"http://build.wabbajack.org/indexed_files/{vf.Hash.FromBase64().ToHex()}/meta.ini");
if (!response.IsSuccessStatusCode) return;
var (game, name) = found.FirstOrDefault(ff => ff.name.FileHash() == vf.Hash);
if (name == null)
return;
File.WriteAllLines(f+".meta", new[]
{
"[General]",
$"gameName={game.MO2ArchiveName}",
$"gameFile={name.RelativeTo(game.GameLocation()).Replace("\\", "/")}"
});
var ini_data = await response.Content.ReadAsStringAsync();
Utils.Log($"Inferred .meta for {Path.GetFileName(vf.FullPath)}, writing to disk");
File.WriteAllText(vf.FullPath + ".meta", ini_data);
});
}