Rewrite ModDB downloader to support mirrors.

Wabbajack.Lib/Downloaders/HTTPDownloader.cs

@@ -7,6 +7,7 @@ using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Reflection.Emit;
 using System.Threading.Tasks;
+using System.Web;
 using Windows.Networking.BackgroundTransfer;
 using Ceras;
 using SharpCompress.Common;
@@ -105,6 +106,9 @@ namespace Wabbajack.Lib.Downloaders
                     var response = await client.GetAsync(Url, HttpCompletionOption.ResponseHeadersRead);
                     TOP:
 
+                    if (!response.IsSuccessStatusCode)
+                        throw new HttpException((int)response.StatusCode, response.ReasonPhrase);
+                    
                     Stream stream;
                     try
                     {

Wabbajack.Lib/Downloaders/ModDBDownloader.cs

@@ -1,7 +1,10 @@
 using System;
+using System.Linq;
 using System.Net.Http;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using System.Web;
+using HtmlAgilityPack;
 using Wabbajack.Common;
 using Wabbajack.Lib.Validation;
 
@@ -45,24 +48,54 @@ namespace Wabbajack.Lib.Downloaders
 
             public override async Task Download(Archive a, string destination)
             {
-                var newURL = await GetDownloadUrl();
+                var urls = await GetDownloadUrls();
-                await new HTTPDownloader.State {Url = newURL}.Download(a, destination);
+                Utils.Log($"Found {urls.Length} ModDB mirrors for {a.Name}");
+                foreach (var (url, idx) in urls.Zip(Enumerable.Range(0, urls.Length), (s, i) => (s, i))) {
+                    try
+                    {
+                        await new HTTPDownloader.State {Url = url}.Download(a, destination);
+                        break;
+                    }
+                    catch (Exception ex)
+                    {
+                        if (idx == urls.Length - 1)
+                            throw;
+                        Utils.Log($"Download from {url} failed, trying next mirror");
+                    }
+                }
             }
 
-            private async Task<string> GetDownloadUrl()
+            private async Task<string[]> GetDownloadUrls()
             {
-                var client = new HttpClient();
+                var uri = new Uri(Url);
-                var result = await client.GetStringAsync(Url);
+                var modId = uri.AbsolutePath.Split('/').Reverse().First(f => int.TryParse(f, out int _));
-                var regex = new Regex("https:\\/\\/www\\.moddb\\.com\\/downloads\\/mirror\\/.*(?=\\\")");
+                var mirrorUrl = $"https://www.moddb.com/downloads/start/{modId}/all";
-                var match = regex.Match(result);
+                var doc = await new HtmlWeb().LoadFromWebAsync($"https://www.moddb.com/downloads/start/{modId}/all");
-                var newURL = match.Value;
+                var mirrors = doc.DocumentNode.Descendants().Where(d => d.NodeType == HtmlNodeType.Element && d.HasClass("row"))
-                return newURL;
+                    .Select(d => new
+                    {
+                        Link = "https://www.moddb.com"+
+                               d.Descendants().Where(s => s.Id == "downloadon")
+                            .Select(i => i.GetAttributeValue("href", ""))
+                            .FirstOrDefault(),
+                        Load = d.Descendants().Where(s => s.HasClass("subheading"))
+                            .Select(i => i.InnerHtml.Split(',')
+                                .Last()
+                                .Split('%')
+                                .Select(v => double.TryParse(v, out var dr) ? dr : double.MaxValue)
+                                .First())
+                            .FirstOrDefault()
+                    })
+                    .OrderBy(d => d.Load)
+                    .ToList();
+                
+                return mirrors.Select(d => d.Link).ToArray();
             }
 
             public override async Task<bool> Verify()
             {
-                var newURL = await GetDownloadUrl();
+                await GetDownloadUrls();
-                return await new HTTPDownloader.State { Url = newURL }.Verify();
+                return true;
             }
 
             public override IDownloader GetDownloader()

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -198,6 +198,9 @@
     <PackageReference Include="CommonMark.NET">
       <Version>0.15.1</Version>
     </PackageReference>
+    <PackageReference Include="HtmlAgilityPack">
+      <Version>1.11.17</Version>
+    </PackageReference>
     <PackageReference Include="MegaApiClient">
       <Version>1.7.1</Version>
     </PackageReference>