Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Lots of NexusAPI refactored to async

Browse Source
This commit is contained in:
Justin Swanson 2019-12-06 20:45:13 -06:00
parent 5bd856a7c6
commit a25fc9d1a9
15 changed files with 74 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Compression.BSA.Test/BSATests.cs
View File

@ -47,21 +47,21 @@ namespace Compression.BSA.Test
(Game.Fallout4, 22223) // 10mm SMG
};
foreach (var info in modIDs)
await Task.WhenAll(modIDs.Select(async (info) =>
{
var filename = await DownloadMod(info);
var folder = Path.Combine(_bsaFolder, info.Item1.ToString(), info.Item2.ToString());
if (!Directory.Exists(folder))
Directory.CreateDirectory(folder);
await FileExtractor.ExtractAll(Queue, filename, folder);
}
}));
}
private static async Task<string> DownloadMod((Game, int) info)
{
using (var client = new NexusApiClient())
{
var results = client.GetModFiles(info.Item1, info.Item2);
var results = await client.GetModFiles(info.Item1, info.Item2);
var file = results.files.FirstOrDefault(f => f.is_primary) ??
results.files.OrderByDescending(f => f.uploaded_timestamp).First();
var src = Path.Combine(_stagingFolder, file.file_name);

3
Wabbajack.Lib/AInstaller.cs
View File

@ -231,8 +231,7 @@ namespace Wabbajack.Lib
var dispatchers = missing.Select(m => m.State.GetDownloader()).Distinct();
foreach (var dispatcher in dispatchers)
dispatcher.Prepare();
await Task.WhenAll(dispatchers.Select(d => d.Prepare()));
await DownloadMissingArchives(missing);
}

3
Wabbajack.Lib/Downloaders/DropboxDownloader.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Threading.Tasks;
using System.Web;
namespace Wabbajack.Lib.Downloaders
@ -31,7 +32,7 @@ namespace Wabbajack.Lib.Downloaders
};
}
public void Prepare()
public async Task Prepare()
{
}
}

2
Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs
View File

@ -29,7 +29,7 @@ namespace Wabbajack.Lib.Downloaders
return null;
}
public void Prepare()
public async Task Prepare()
{
}

2
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -46,7 +46,7 @@ namespace Wabbajack.Lib.Downloaders
return null;
}
public void Prepare()
public async Task Prepare()
{
}

6
Wabbajack.Lib/Downloaders/IDownloader.cs
View File

@ -1,4 +1,6 @@
namespace Wabbajack.Lib.Downloaders
using System.Threading.Tasks;
namespace Wabbajack.Lib.Downloaders
{
public interface IDownloader
{
@ -7,7 +9,7 @@
/// <summary>
/// Called before any downloads are inacted by the installer;
/// </summary>
void Prepare();
Task Prepare();
}
}

2
Wabbajack.Lib/Downloaders/MEGADownloader.cs
View File

@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders
return null;
}
public void Prepare()
public async Task Prepare()
{
}

2
Wabbajack.Lib/Downloaders/ManualDownloader.cs
View File

@ -65,7 +65,7 @@ namespace Wabbajack.Lib.Downloaders
return url != null ? new State { Url = url} : null;
}
public void Prepare()
public async Task Prepare()
{
}

2
Wabbajack.Lib/Downloaders/MediaFireDownloader.cs
View File

@ -70,7 +70,7 @@ namespace Wabbajack.Lib.Downloaders
}
public void Prepare()
public async Task Prepare()
{
}

2
Wabbajack.Lib/Downloaders/ModDBDownloader.cs
View File

@ -27,7 +27,7 @@ namespace Wabbajack.Lib.Downloaders
return null;
}
public void Prepare()
public async Task Prepare()
{
}

8
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -41,10 +41,10 @@ namespace Wabbajack.Lib.Downloaders
return null;
}
public void Prepare()
public async Task Prepare()
{
var client = new NexusApiClient();
var status = client.GetUserStatus();
var status = await client.GetUserStatus();
if (!client.IsAuthenticated)
{
Utils.ErrorThrow(new UnconvertedError($"Authenticating for the Nexus failed. A nexus account is required to automatically download mods."));
@ -81,7 +81,7 @@ namespace Wabbajack.Lib.Downloaders
string url;
try
{
url = new NexusApiClient().GetNexusDownloadLink(this, false);
url = await new NexusApiClient().GetNexusDownloadLink(this, false);
}
catch (Exception ex)
{
@ -110,7 +110,7 @@ namespace Wabbajack.Lib.Downloaders
if (!int.TryParse(ModID, out var modID))
return false;
var modFiles = new NexusApiClient().GetModFiles(game, modID);
var modFiles = await new NexusApiClient().GetModFiles(game, modID);
if (!ulong.TryParse(FileID, out var fileID))
return false;

2
Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs
View File

@ -29,7 +29,7 @@ namespace Wabbajack.Lib.Downloaders
return new State {Item = _item};
}
public void Prepare()
public async Task Prepare()
{
}

83
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -45,7 +45,7 @@ namespace Wabbajack.Lib.NexusApi
get
{
if (_userStatus == null)
_userStatus = GetUserStatus();
_userStatus = GetUserStatus().Result;
return _userStatus;
}
}
@ -122,10 +122,10 @@ namespace Wabbajack.Lib.NexusApi
}
}
public UserStatus GetUserStatus()
public async Task<UserStatus> GetUserStatus()
{
var url = "https://api.nexusmods.com/v1/users/validate.json";
return Get<UserStatus>(url);
return await Get<UserStatus>(url);
}
#endregion
@ -200,24 +200,19 @@ namespace Wabbajack.Lib.NexusApi
Directory.CreateDirectory(Consts.NexusCacheDirectory);
}
private T Get<T>(string url)
private async Task<T> Get<T>(string url)
{
Task<HttpResponseMessage> responseTask = _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
responseTask.Wait();
HttpResponseMessage response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
var response = responseTask.Result;
UpdateRemaining(response);
var contentTask = response.Content.ReadAsStreamAsync();
contentTask.Wait();
using (var stream = contentTask.Result)
using (var stream = await response.Content.ReadAsStreamAsync())
{
return stream.FromJSON<T>();
}
}
private T GetCached<T>(string url)
private async Task<T> GetCached<T>(string url)
{
var code = Encoding.UTF8.GetBytes(url).ToHex() + ".json";
@ -232,7 +227,7 @@ namespace Wabbajack.Lib.NexusApi
return cache_file.FromJSON<T>();
}
var result = Get<T>(url);
var result = await Get<T>(url);
if (result != null)
result.ToJSON(cache_file);
return result;
@ -240,27 +235,33 @@ namespace Wabbajack.Lib.NexusApi
try
{
return Get<T>(Consts.WabbajackCacheLocation + code);
return await Get<T>(Consts.WabbajackCacheLocation + code);
}
catch (Exception)
{
return Get<T>(url);
return await Get<T>(url);
}
}
public string GetNexusDownloadLink(NexusDownloader.State archive, bool cache = false)
public async Task<string> GetNexusDownloadLink(NexusDownloader.State archive, bool cache = false)
{
if (cache && TryGetCachedLink(archive, out var result))
return result;
if (cache)
{
var result = await TryGetCachedLink(archive);
if (result.Succeeded)
{
return result.Value;
}
}
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(archive.GameName)}/mods/{archive.ModID}/files/{archive.FileID}/download_link.json";
return Get<List<DownloadLink>>(url).First().URI;
return (await Get<List<DownloadLink>>(url)).First().URI;
}
private bool TryGetCachedLink(NexusDownloader.State archive, out string result)
private async Task<GetResponse<string>> TryGetCachedLink(NexusDownloader.State archive)
{
if (!Directory.Exists(Consts.NexusCacheDirectory))
Directory.CreateDirectory(Consts.NexusCacheDirectory);
@ -269,19 +270,18 @@ namespace Wabbajack.Lib.NexusApi
if (!File.Exists(path) || (DateTime.Now - new FileInfo(path).LastWriteTime).TotalHours > 24)
{
File.Delete(path);
result = GetNexusDownloadLink(archive);
var result = await GetNexusDownloadLink(archive);
File.WriteAllText(path, result);
return true;
return GetResponse<string>.Succeed(result);
}
result = File.ReadAllText(path);
return true;
return GetResponse<string>.Succeed(File.ReadAllText(path));
}
public NexusFileInfo GetFileInfo(NexusDownloader.State mod)
public async Task<NexusFileInfo> GetFileInfo(NexusDownloader.State mod)
{
var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(mod.GameName)}/mods/{mod.ModID}/files/{mod.FileID}.json";
return GetCached<NexusFileInfo>(url);
return await GetCached<NexusFileInfo>(url);
}
public class GetModFilesResponse
@ -289,22 +289,22 @@ namespace Wabbajack.Lib.NexusApi
public List<NexusFileInfo> files;
}
public GetModFilesResponse GetModFiles(Game game, int modid)
public async Task<GetModFilesResponse> GetModFiles(Game game, int modid)
{
var url = $"https://api.nexusmods.com/v1/games/{GameRegistry.Games[game].NexusName}/mods/{modid}/files.json";
return GetCached<GetModFilesResponse>(url);
return await GetCached<GetModFilesResponse>(url);
}
public List<MD5Response> GetModInfoFromMD5(Game game, string md5Hash)
public async Task<List<MD5Response>> GetModInfoFromMD5(Game game, string md5Hash)
{
var url = $"https://api.nexusmods.com/v1/games/{GameRegistry.Games[game].NexusName}/mods/md5_search/{md5Hash}.json";
return Get<List<MD5Response>>(url);
return await Get<List<MD5Response>>(url);
}
public ModInfo GetModInfo(Game game, string modId)
public async Task<ModInfo> GetModInfo(Game game, string modId)
{
var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modId}.json";
return GetCached<ModInfo>(url);
return await GetCached<ModInfo>(url);
}
public async Task<EndorsementResponse> EndorseMod(NexusDownloader.State mod)
@ -359,16 +359,21 @@ namespace Wabbajack.Lib.NexusApi
{
if (!UseLocalCache) return;
var purge = GameRegistry.Games.Values
var gameTasks = GameRegistry.Games.Values
.Where(game => game.NexusName != null)
.Select(game => new
.Select(async game =>
{
game = game,
mods = Get<List<UpdatedMod>>(
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m")
return (game,
mods: await Get<List<UpdatedMod>>(
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"));
})
.SelectMany(r => r.mods.Select(mod => new {game = r.game,
mod = mod}))
.Select(async rTask =>
{
var (game, mods) = await rTask;
return mods.Select(mod => new { game = game, mod = mod });
});
var purge = (await Task.WhenAll(gameTasks))
.SelectMany(i => i)
.ToList();
Utils.Log($"Found {purge.Count} updated mods in the last month");

13
Wabbajack.Lib/VortexCompiler.cs
View File

@ -86,7 +86,7 @@ namespace Wabbajack.Lib
if (cancel.IsCancellationRequested) return false;
Info("Starting pre-compilation steps");
CreateMetaFiles();
await CreateMetaFiles();
if (cancel.IsCancellationRequested) return false;
Info($"Indexing {StagingFolder}");
@ -314,14 +314,15 @@ namespace Wabbajack.Lib
}
}
private void CreateMetaFiles()
private async Task CreateMetaFiles()
{
Utils.Log("Getting Nexus api_key, please click authorize if a browser window appears");
var nexusClient = new NexusApiClient();
Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly)
await Task.WhenAll(
Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly)
.Where(File.Exists)
.Do(f =>
.Select(async f =>
{
if (Path.GetExtension(f) != ".meta" && !File.Exists($"{f}.meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)))
{
@ -343,7 +344,7 @@ namespace Wabbajack.Lib
Utils.Log($"Hash is {hash}");
}
var md5Response = nexusClient.GetModInfoFromMD5(Game, hash);
var md5Response = await nexusClient.GetModInfoFromMD5(Game, hash);
if (md5Response.Count >= 1)
{
var modInfo = md5Response[0].mod;
@ -388,7 +389,7 @@ namespace Wabbajack.Lib
ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
}
}
});
}));
Utils.Log($"Checking for Steam Workshop Items...");
if (!_isSteamGame || _steamGame == null || _steamGame.WorkshopItems.Count <= 0)

3
Wabbajack.Test/EndToEndTests.cs
View File

@ -106,7 +106,8 @@ namespace Wabbajack.Test
{
utils.AddMod(mod_name);
var client = new NexusApiClient();
var file = client.GetModFiles(game, modid).files.First(f => f.is_primary);
var resp = await client.GetModFiles(game, modid);
var file = resp.files.First(f => f.is_primary);
var src = Path.Combine(DOWNLOAD_FOLDER, file.file_name);
var ini = string.Join("\n",