Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Index new files on the nexus each day

Browse Source
This commit is contained in:
Timothy Baldridge 2020-01-12 21:04:46 -07:00
parent 7df6d06e98
commit 527f30f1c0
9 changed files with 135 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Wabbajack.BuildServer/JobManager.cs
View File

@ -75,6 +75,7 @@ namespace Wabbajack.BuildServer
await ScheduledJob<UpdateModLists>(TimeSpan.FromMinutes(30), Job.JobPriority.High);
await ScheduledJob<EnqueueAllArchives>(TimeSpan.FromHours(2), Job.JobPriority.Low);
await ScheduledJob<EnqueueAllGameFiles>(TimeSpan.FromHours(24), Job.JobPriority.High);
await ScheduledJob<EnqueueRecentFiles>(TimeSpan.FromHours(6), Job.JobPriority.Low);
await Task.Delay(10000);
}
}

1
Wabbajack.BuildServer/Models/DBContext.cs
View File

@ -26,6 +26,7 @@ namespace Wabbajack.BuildServer.Models
public IMongoCollection<DownloadState> DownloadStates => Client.GetCollection<DownloadState>(_settings.Collections["DownloadStates"]);
public IMongoCollection<Metric> Metrics => Client.GetCollection<Metric>(_settings.Collections["Metrics"]);
public IMongoCollection<IndexedFile> IndexedFiles => Client.GetCollection<IndexedFile>(_settings.Collections["IndexedFiles"]);
public IMongoCollection<NexusCacheData<List<NexusUpdateEntry>>> NexusUpdates => Client.GetCollection<NexusCacheData<List<NexusUpdateEntry>>>(_settings.Collections["NexusUpdates"]);
public IMongoCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles =>
Client.GetCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>>(

3
Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs
View File

@ -16,7 +16,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue
typeof(GetNexusUpdatesJob),
typeof(UpdateModLists),
typeof(EnqueueAllArchives),
typeof(EnqueueAllGameFiles)
typeof(EnqueueAllGameFiles),
typeof(EnqueueRecentFiles)
};
public static Dictionary<Type, string> TypeToName { get; set; }
public static Dictionary<string, Type> NameToType { get; set; }

1
Wabbajack.BuildServer/Models/JobQueue/Job.cs
View File

@ -6,6 +6,7 @@ using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer.Models.JobQueue
{

84
Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs Normal file
View File

@ -0,0 +1,84 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using MongoDB.Driver;
using MongoDB.Driver.Core.Authentication;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class EnqueueRecentFiles : AJobPayload
{
public override string Description => "Enqueue the past days worth of mods for indexing";
private static HashSet<Game> GamesToScan = new HashSet<Game>
{
Game.Fallout3, Game.Fallout4, Game.Skyrim, Game.SkyrimSpecialEdition, Game.SkyrimVR, Game.FalloutNewVegas, Game.Oblivion
};
public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
{
using (var queue = new WorkQueue())
{
var updates = await db.NexusUpdates.AsQueryable().ToListAsync();
var mods = updates
.Where(list => GamesToScan.Contains(GameRegistry.GetByNexusName(list.Game).Game))
.SelectMany(list =>
list.Data.Where(mod => DateTime.UtcNow - mod.LatestFileUpdate.AsUnixTime() < TimeSpan.FromDays(1))
.Select(mod => (list.Game, mod.ModId)));
var mod_files = (await mods.PMap(queue, async mod =>
{
var client = await NexusApiClient.Get();
try
{
var files = await client.GetModFiles(GameRegistry.GetByNexusName(mod.Game).Game,
(int)mod.ModId);
return (mod.Game, mod.ModId, files.files);
}
catch (Exception)
{
return default;
}
})).Where(t => t.Game != null).ToList();
var archives =
mod_files.SelectMany(mod => mod.files.Select(file => (mod.Game, mod.ModId, File:file)).Where(f => !string.IsNullOrEmpty(f.File.category_name) ))
.Select(tuple =>
{
var state = new NexusDownloader.State
{
GameName = tuple.Game, ModID = tuple.ModId.ToString(), FileID = tuple.File.file_id.ToString()
};
return new Archive {State = state, Name = tuple.File.file_name};
}).ToList();
Utils.Log($"Found {archives.Count} archives from recent Nexus updates to index");
var searching = archives.Select(a => a.State.PrimaryKeyString).Distinct().ToArray();
Utils.Log($"Looking for missing states");
var knownArchives = (await db.DownloadStates.AsQueryable().Where(s => searching.Contains(s.Key))
.Select(d => d.Key).ToListAsync()).ToDictionary(a => a);
Utils.Log($"Found {knownArchives.Count} pre-existing archives");
var missing = archives.Where(a => !knownArchives.ContainsKey(a.State.PrimaryKeyString))
.DistinctBy(d => d.State.PrimaryKeyString)
.ToList();
Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs");
var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}, Priority = Job.JobPriority.Low});
Utils.Log($"Writing jobs to the DB");
await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions {IsOrdered = false});
Utils.Log($"Done adding archives for Nexus Updates");
return JobResult.Success();
}
}
}
}

36
Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs
View File

@ -5,6 +5,7 @@ using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
using MongoDB.Driver;
using Newtonsoft.Json;
namespace Wabbajack.BuildServer.Models.Jobs
@ -21,9 +22,17 @@ namespace Wabbajack.BuildServer.Models.Jobs
.Where(game => game.NexusName != null)
.Select(async game =>
{
return (game,
mods: await api.Get<List<UpdatedMod>>(
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"));
var mods = await api.Get<List<NexusUpdateEntry>>(
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m");
var entry = new NexusCacheData<List<NexusUpdateEntry>>();
entry.Game = game.NexusName;
entry.Path = $"/v1/games/{game.NexusName}/mods/updated.json?period=1m";
entry.Data = mods;
await entry.Upsert(db.NexusUpdates);
return (game, mods);
})
.Select(async rTask =>
{
@ -40,14 +49,16 @@ namespace Wabbajack.BuildServer.Models.Jobs
Utils.Log($"Found {purge.Count} updated mods in the last month");
using (var queue = new WorkQueue())
{
var collected = await purge.Select(d =>
var collected = purge.Select(d =>
{
var a = d.mod.latest_file_update.AsUnixTime();
var a = d.mod.LatestFileUpdate.AsUnixTime();
// Mod activity could hide files
var b = d.mod.latest_mod_activity.AsUnixTime();
var b = d.mod.LastestModActivity.AsUnixTime();
return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.mod_id.ToString()};
}).PMap(queue, async t =>
return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.ModId.ToString()};
});
var purged = await collected.PMap(queue, async t =>
{
var resultA = await db.NexusModInfos.DeleteManyAsync(f =>
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
@ -59,17 +70,12 @@ namespace Wabbajack.BuildServer.Models.Jobs
return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount;
});
Utils.Log($"Purged {collected.Sum()} cache entries");
Utils.Log($"Purged {purged.Sum()} cache entries");
}
return JobResult.Success();
}
class UpdatedMod
{
public long mod_id;
public long latest_file_update;
public long latest_mod_activity;
}
}
}

2
Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs
View File

@ -20,7 +20,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
using (var queue = new WorkQueue())
{
foreach (var list in modlists)
foreach (var list in modlists.Skip(1).Take(1))
{
try
{

8
Wabbajack.BuildServer/Models/NexusCacheData.cs
View File

@ -1,5 +1,7 @@
using System;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
namespace Wabbajack.BuildServer.Models
{
@ -9,6 +11,8 @@ namespace Wabbajack.BuildServer.Models
public string Path { get; set; }
public T Data { get; set; }
public string Game { get; set; }
[BsonIgnoreIfNull]
public string ModId { get; set; }
public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow;
@ -16,5 +20,9 @@ namespace Wabbajack.BuildServer.Models
[BsonIgnoreIfNull]
public string FileId { get; set; }
public async Task Upsert(IMongoCollection<NexusCacheData<T>> coll)
{
await coll.FindOneAndReplaceAsync<NexusCacheData<T>>(s => s.Path == Path, this, new FindOneAndReplaceOptions<NexusCacheData<T>> {IsUpsert = true});
}
}
}

16
Wabbajack.BuildServer/Models/NexusUpdateEntry.cs Normal file
View File

@ -0,0 +1,16 @@
using System;
using MongoDB.Bson.Serialization.Attributes;
using Newtonsoft.Json;
namespace Wabbajack.BuildServer.Models
{
public class NexusUpdateEntry
{
[JsonProperty("mod_id")]
public long ModId { get; set; }
[JsonProperty("latest_file_update")]
public long LatestFileUpdate { get; set; }
[JsonProperty("latest_mod_activity")]
public long LastestModActivity { get; set; }
}
}