diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index 911776b8..dd353f9b 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -7,6 +7,7 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Reflection.Emit; using System.Threading.Tasks; +using System.Web; using Windows.Networking.BackgroundTransfer; using Ceras; using SharpCompress.Common; @@ -105,6 +106,9 @@ namespace Wabbajack.Lib.Downloaders var response = await client.GetAsync(Url, HttpCompletionOption.ResponseHeadersRead); TOP: + if (!response.IsSuccessStatusCode) + throw new HttpException((int)response.StatusCode, response.ReasonPhrase); + Stream stream; try { diff --git a/Wabbajack.Lib/Downloaders/ModDBDownloader.cs b/Wabbajack.Lib/Downloaders/ModDBDownloader.cs index 4a9d3ebd..c06b0c29 100644 --- a/Wabbajack.Lib/Downloaders/ModDBDownloader.cs +++ b/Wabbajack.Lib/Downloaders/ModDBDownloader.cs @@ -1,7 +1,10 @@ using System; +using System.Linq; using System.Net.Http; using System.Text.RegularExpressions; using System.Threading.Tasks; +using System.Web; +using HtmlAgilityPack; using Wabbajack.Common; using Wabbajack.Lib.Validation; @@ -45,24 +48,54 @@ namespace Wabbajack.Lib.Downloaders public override async Task Download(Archive a, string destination) { - var newURL = await GetDownloadUrl(); - await new HTTPDownloader.State {Url = newURL}.Download(a, destination); + var urls = await GetDownloadUrls(); + Utils.Log($"Found {urls.Length} ModDB mirrors for {a.Name}"); + foreach (var (url, idx) in urls.Zip(Enumerable.Range(0, urls.Length), (s, i) => (s, i))) { + try + { + await new HTTPDownloader.State {Url = url}.Download(a, destination); + break; + } + catch (Exception ex) + { + if (idx == urls.Length - 1) + throw; + Utils.Log($"Download from {url} failed, trying next mirror"); + } + } } - private async Task GetDownloadUrl() + private async Task GetDownloadUrls() { - var client = new HttpClient(); - var result = await client.GetStringAsync(Url); - var regex = new Regex("https:\\/\\/www\\.moddb\\.com\\/downloads\\/mirror\\/.*(?=\\\")"); - var match = regex.Match(result); - var newURL = match.Value; - return newURL; + var uri = new Uri(Url); + var modId = uri.AbsolutePath.Split('/').Reverse().First(f => int.TryParse(f, out int _)); + var mirrorUrl = $"https://www.moddb.com/downloads/start/{modId}/all"; + var doc = await new HtmlWeb().LoadFromWebAsync($"https://www.moddb.com/downloads/start/{modId}/all"); + var mirrors = doc.DocumentNode.Descendants().Where(d => d.NodeType == HtmlNodeType.Element && d.HasClass("row")) + .Select(d => new + { + Link = "https://www.moddb.com"+ + d.Descendants().Where(s => s.Id == "downloadon") + .Select(i => i.GetAttributeValue("href", "")) + .FirstOrDefault(), + Load = d.Descendants().Where(s => s.HasClass("subheading")) + .Select(i => i.InnerHtml.Split(',') + .Last() + .Split('%') + .Select(v => double.TryParse(v, out var dr) ? dr : double.MaxValue) + .First()) + .FirstOrDefault() + }) + .OrderBy(d => d.Load) + .ToList(); + + return mirrors.Select(d => d.Link).ToArray(); } public override async Task Verify() { - var newURL = await GetDownloadUrl(); - return await new HTTPDownloader.State { Url = newURL }.Verify(); + await GetDownloadUrls(); + return true; } public override IDownloader GetDownloader() diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 588372bb..fd3f3aa4 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -198,6 +198,9 @@ 0.15.1 + + 1.11.17 + 1.7.1