diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index 26508d3f..8c7490b8 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -79,6 +79,7 @@ namespace Wabbajack.BuildServer await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.Low); await ScheduledJob(TimeSpan.FromHours(24), Job.JobPriority.High); await ScheduledJob(TimeSpan.FromHours(6), Job.JobPriority.Low); + await ScheduledJob(TimeSpan.FromHours(1), Job.JobPriority.Normal); await Task.Delay(10000); } } diff --git a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs index 7b7169fc..23cc5989 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs @@ -18,7 +18,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue typeof(EnqueueAllArchives), typeof(EnqueueAllGameFiles), typeof(EnqueueRecentFiles), - typeof(UploadToCDN) + typeof(UploadToCDN), + typeof(IndexDynDOLOD) }; public static Dictionary TypeToName { get; set; } public static Dictionary NameToType { get; set; } diff --git a/Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs b/Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs new file mode 100644 index 00000000..49d7d954 --- /dev/null +++ b/Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs @@ -0,0 +1,67 @@ +using System; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using HtmlAgilityPack; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; + +namespace Wabbajack.BuildServer.Models.Jobs +{ + /// + /// DynDOLOD is really hosted on a STEP Forum post as a series of MEGA links. The Nexus URLs come and go + /// but the real releases are on STEP. So let's keep that data fresh. + /// + public class IndexDynDOLOD : AJobPayload + { + public override string Description => "Queue MEGA URLs from the DynDOLOD Post"; + public override async Task Execute(DBContext db, AppSettings settings) + { + var doc = new HtmlDocument(); + var body = await new HttpClient().GetStringAsync(new Uri( + "https://forum.step-project.com/topic/13894-dyndolod-beta-for-skyrim-special-edition-and-skyrim-vr-279/")); + doc.LoadHtml(body); + + var matches = + doc.DocumentNode + .Descendants() + .Where(d=> d.NodeType == HtmlNodeType.Element && d.Attributes.Contains("href")) + .Select(d => d.Attributes["href"].Value) + .Select(m => Uri.TryCreate(m.ToString(), UriKind.Absolute, out var result) ? result : null) + .Where(uri => uri != null && uri.Host == "mega.nz") + .Select(url => new Job() + { + Payload = new IndexJob + { + Archive = new Archive + { + Name = Guid.NewGuid() + ".7z", + State = new MegaDownloader.State + { + Url = url.ToString() + } + } + } + }) + .ToList(); + + + foreach (var job in matches) + { + var key = ((MegaDownloader.State)((IndexJob)job.Payload).Archive.State).PrimaryKeyString; + var found = await db.DownloadStates.AsQueryable().Where(s => s.Key == key).FirstOrDefaultAsync(); + if (found != null) continue; + + Utils.Log($"Queuing {key} for indexing"); + await db.Jobs.InsertOneAsync(job); + } + + return JobResult.Success(); + + } + } +} diff --git a/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs b/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs index 6da356d2..ed232b6f 100644 --- a/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs +++ b/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs @@ -35,6 +35,7 @@ namespace Wabbajack.BuildServer.Models.Jobs await db.Jobs.InsertOneAsync(new Job { + Priority = Job.JobPriority.High, Payload = new IndexJob { Archive = new Archive