Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #334 from wabbajack-tools/moddb-rewrite

Browse Source 
Rewrite ModDB downloader to support mirrors.
This commit is contained in:
Timothy Baldridge 2020-01-03 14:27:14 -08:00 committed by GitHub
commit eb16865330
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 51 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -7,6 +7,7 @@ using System.Net.Http;
using System.Net.Http.Headers;
using System.Reflection.Emit;
using System.Threading.Tasks;
using System.Web;
using Windows.Networking.BackgroundTransfer;
using Ceras;
using SharpCompress.Common;
@ -105,6 +106,9 @@ namespace Wabbajack.Lib.Downloaders
var response = await client.GetAsync(Url, HttpCompletionOption.ResponseHeadersRead);
TOP:
if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase);
Stream stream;
try
{

55
Wabbajack.Lib/Downloaders/ModDBDownloader.cs
View File

@ -1,7 +1,10 @@
using System;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web;
using HtmlAgilityPack;
using Wabbajack.Common;
using Wabbajack.Lib.Validation;
@ -45,24 +48,54 @@ namespace Wabbajack.Lib.Downloaders
public override async Task Download(Archive a, string destination)
{
var newURL = await GetDownloadUrl();
await new HTTPDownloader.State {Url = newURL}.Download(a, destination);
var urls = await GetDownloadUrls();
Utils.Log($"Found {urls.Length} ModDB mirrors for {a.Name}");
foreach (var (url, idx) in urls.Zip(Enumerable.Range(0, urls.Length), (s, i) => (s, i))) {
try
{
await new HTTPDownloader.State {Url = url}.Download(a, destination);
break;
}
catch (Exception ex)
{
if (idx == urls.Length - 1)
throw;
Utils.Log($"Download from {url} failed, trying next mirror");
}
}
}
private async Task<string> GetDownloadUrl()
private async Task<string[]> GetDownloadUrls()
{
var client = new HttpClient();
var result = await client.GetStringAsync(Url);
var regex = new Regex("https:\\/\\/www\\.moddb\\.com\\/downloads\\/mirror\\/.*(?=\\\")");
var match = regex.Match(result);
var newURL = match.Value;
return newURL;
var uri = new Uri(Url);
var modId = uri.AbsolutePath.Split('/').Reverse().First(f => int.TryParse(f, out int _));
var mirrorUrl = $"https://www.moddb.com/downloads/start/{modId}/all";
var doc = await new HtmlWeb().LoadFromWebAsync($"https://www.moddb.com/downloads/start/{modId}/all");
var mirrors = doc.DocumentNode.Descendants().Where(d => d.NodeType == HtmlNodeType.Element && d.HasClass("row"))
.Select(d => new
{
Link = "https://www.moddb.com"+
d.Descendants().Where(s => s.Id == "downloadon")
.Select(i => i.GetAttributeValue("href", ""))
.FirstOrDefault(),
Load = d.Descendants().Where(s => s.HasClass("subheading"))
.Select(i => i.InnerHtml.Split(',')
.Last()
.Split('%')
.Select(v => double.TryParse(v, out var dr) ? dr : double.MaxValue)
.First())
.FirstOrDefault()
})
.OrderBy(d => d.Load)
.ToList();
return mirrors.Select(d => d.Link).ToArray();
}
public override async Task<bool> Verify()
{
var newURL = await GetDownloadUrl();
return await new HTTPDownloader.State { Url = newURL }.Verify();
await GetDownloadUrls();
return true;
}
public override IDownloader GetDownloader()

3
Wabbajack.Lib/Wabbajack.Lib.csproj
View File

@ -198,6 +198,9 @@
<PackageReference Include="CommonMark.NET">
<Version>0.15.1</Version>
</PackageReference>
<PackageReference Include="HtmlAgilityPack">
<Version>1.11.17</Version>
</PackageReference>
<PackageReference Include="MegaApiClient">
<Version>1.7.1</Version>
</PackageReference>