Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Integrated list validation into the caching HTTP server.

Browse Source
This commit is contained in:
halgari 2019-12-13 22:46:20 -07:00
parent f60c1e3c8a
commit 0464f1b43c
8 changed files with 244 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

193
Wabbajack.CacheServer/ListValidationService.cs Normal file
View File

@ -0,0 +1,193 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Nancy;
using Nancy.Responses;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.CacheServer
{
public class ListValidationService : NancyModule
{
public class ModListStatus
{
public string Name;
public DateTime Checked = DateTime.Now;
public List<(Archive archive, bool)> Archives { get; set; }
public DownloadMetadata DownloadMetaData { get; set; }
public bool HasFailures { get; set; }
}
public class ModlistSummary
{
public string Name;
public DateTime Checked;
public int Failed;
public int Passed;
public string Link => $"/lists/status/{Name}.json";
public string Report => $"/lists/status/{Name}.html";
}
public static Dictionary<string, ModListStatus> ModLists { get; set; }
public ListValidationService() : base("/lists")
{
Get("/status", HandleGetLists);
Get("/status/{Name}.json", HandleGetListJson);
Get("/status/{Name}.html", HandleGetListHtml);
}
private object HandleGetLists(object arg)
{
var summaries = ModLists.Values.Select(m => new ModlistSummary
{
Name = m.Name,
Checked = m.Checked,
Failed = m.Archives.Count(a => a.Item2),
Passed = m.Archives.Count(a => !a.Item2),
}).ToList();
return summaries.ToJSON();
}
public class ArchiveSummary
{
public string Name;
public AbstractDownloadState State;
}
public class DetailedSummary
{
public string Name;
public DateTime Checked;
public List<ArchiveSummary> Failed;
public List<ArchiveSummary> Passed;
}
private object HandleGetListJson(dynamic arg)
{
var lst = ModLists[(string)arg.Name];
var summary = new DetailedSummary
{
Name = lst.Name,
Checked = lst.Checked,
Failed = lst.Archives.Where(a => a.Item2)
.Select(a => new ArchiveSummary {Name = a.archive.Name, State = a.archive.State}).ToList(),
Passed = lst.Archives.Where(a => !a.Item2)
.Select(a => new ArchiveSummary { Name = a.archive.Name, State = a.archive.State }).ToList(),
};
return summary.ToJSON();
}
private object HandleGetListHtml(dynamic arg)
{
var lst = ModLists[(string)arg.Name];
var sb = new StringBuilder();
sb.Append("<html><body>");
sb.Append($"<h2>{lst.Name} - {lst.Checked}</h2>");
var failed_list = lst.Archives.Where(a => a.Item2).ToList();
sb.Append($"<h3>Failed ({failed_list.Count}):</h3>");
sb.Append("<ul>");
foreach (var archive in failed_list)
{
sb.Append($"<li>{archive.archive.Name}</li>");
}
sb.Append("</ul>");
var pased_list = lst.Archives.Where(a => !a.Item2).ToList();
sb.Append($"<h3>Passed ({pased_list.Count}):</h3>");
sb.Append("<ul>");
foreach (var archive in pased_list.OrderBy(f => f.archive.Name))
{
sb.Append($"<li>{archive.archive.Name}</li>");
}
sb.Append("</ul>");
sb.Append("</body></html>");
var response = (Response)sb.ToString();
response.ContentType = "text/html";
return response;
}
public static void Start()
{
var tsk = ValidateLists();
}
public static async Task ValidateLists()
{
Utils.Log("Starting Modlist Validation");
var modlists = await ModlistMetadata.LoadFromGithub();
var statuses = new Dictionary<string, ModListStatus>();
using (var queue = new WorkQueue())
{
foreach (var list in modlists)
{
var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ".wabbajack");
if (list.NeedsDownload(modlist_path))
{
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlist_path);
}
else
{
Utils.Log($"No changes detected from downloaded modlist");
}
Utils.Log($"Loading {modlist_path}");
var installer = AInstaller.LoadFromFile(modlist_path);
Utils.Log($"{installer.Archives.Count} archives to validate");
DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
var validated = (await installer.Archives
.PMap(queue, async archive =>
{
Utils.Log($"Validating: {archive.Name}");
bool is_failed;
try
{
is_failed = !(await archive.State.Verify());
}
catch (Exception)
{
is_failed = false;
}
return (archive, is_failed);
}))
.ToList();
var status = new ModListStatus
{
Name = list.Title,
Archives = validated.OrderBy(v => v.archive.Name).ToList(),
DownloadMetaData = list.DownloadMetadata,
HasFailures = validated.Any(v => v.is_failed)
};
statuses.Add(status.Name, status);
}
}
Utils.Log($"Done validating {statuses.Count} lists");
ModLists = statuses;
}
}
}

5
Wabbajack.CacheServer/NexusCacheModule.cs
View File

@ -18,9 +18,6 @@ namespace Wabbajack.CacheServer
public NexusCacheModule() : base("/")
{
// ToDo
// Handle what to do with the fact that lots of these are now a tasks
throw new NotImplementedException("Unsure if following functions still work when taking in a Task");
Get("/v1/games/{GameName}/mods/{ModID}/files/{FileID}.json", HandleFileID);
Get("/v1/games/{GameName}/mods/{ModID}/files.json", HandleGetFiles);
Get("/v1/games/{GameName}/mods/{ModID}.json", HandleModInfo);
@ -29,7 +26,7 @@ namespace Wabbajack.CacheServer
Get("/nexus_api_cache/update", UpdateCache);
}
private async Task<object> UpdateCache(object arg)
public async Task<object> UpdateCache(object arg)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
await api.ClearUpdatedModsInCache();

3
Wabbajack.CacheServer/Program.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Nancy.Hosting.Self;
using Wabbajack.Common;
namespace Wabbajack.CacheServer
@ -12,9 +13,9 @@ namespace Wabbajack.CacheServer
static void Main(string[] args)
{
Utils.LogMessages.Subscribe(Console.WriteLine);
using (var server = new Server("http://localhost:8080"))
{
ListValidationService.Start();
server.Start();
Console.ReadLine();
}

1
Wabbajack.CacheServer/Server.cs
View File

@ -18,6 +18,7 @@ namespace Wabbajack.CacheServer
{
Address = address;
_config = new HostConfiguration();
_config.MaximumConnectionCount = 24;
//_config.UrlReservations.CreateAutomatically = true;
_config.RewriteLocalhost = true;
_server = new NancyHost(_config, new Uri(address));

3
Wabbajack.CacheServer/Wabbajack.CacheServer.csproj
View File

@ -15,7 +15,7 @@
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<PlatformTarget>x64</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
@ -73,6 +73,7 @@
<Reference Include="WindowsBase" />
</ItemGroup>
<ItemGroup>
<Compile Include="ListValidationService.cs" />
<Compile Include="NexusCacheModule.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />

18
Wabbajack.Common/WorkQueue.cs
View File

@ -70,7 +70,16 @@ namespace Wabbajack.Common
{
Report("Waiting", 0, false);
if (_cancel.IsCancellationRequested) return;
var f = Queue.Take(_cancel.Token);
Func<Task> f;
try
{
f = Queue.Take(_cancel.Token);
}
catch (Exception)
{
throw new OperationCanceledException();
}
await f();
}
}
@ -100,13 +109,6 @@ namespace Wabbajack.Common
public void Dispose()
{
_cancel.Cancel();
Threads.Do(th =>
{
if (th.ManagedThreadId != Thread.CurrentThread.ManagedThreadId)
{
th.Join();
}
});
Queue?.Dispose();
}
}

38
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed.Errors;
@ -10,6 +11,11 @@ namespace Wabbajack.Lib.Downloaders
{
public class NexusDownloader : IDownloader
{
private bool _prepared;
private SemaphoreSlim _lock = new SemaphoreSlim(1);
private UserStatus _status;
private NexusApiClient _client;
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
{
var general = archiveINI?.General;
@ -44,16 +50,34 @@ namespace Wabbajack.Lib.Downloaders
public async Task Prepare()
{
var client = await NexusApiClient.Get();
var status = await client.GetUserStatus();
if (!client.IsAuthenticated)
if (!_prepared)
{
Utils.ErrorThrow(new UnconvertedError($"Authenticating for the Nexus failed. A nexus account is required to automatically download mods."));
return;
await _lock.WaitAsync();
try
{
// Could have become prepared while we waited for the lock
if (!_prepared)
{
_client = await NexusApiClient.Get();
_status = await _client.GetUserStatus();
if (!_client.IsAuthenticated)
{
Utils.ErrorThrow(new UnconvertedError(
$"Authenticating for the Nexus failed. A nexus account is required to automatically download mods."));
return;
}
}
}
finally
{
_lock.Release();
}
}
if (status.is_premium) return;
Utils.ErrorThrow(new UnconvertedError($"Automated installs with Wabbajack requires a premium nexus account. {await client.Username()} is not a premium account."));
_prepared = true;
if (_status.is_premium) return;
Utils.ErrorThrow(new UnconvertedError($"Automated installs with Wabbajack requires a premium nexus account. {await _client.Username()} is not a premium account."));
}
public class State : AbstractDownloadState

5
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -329,7 +329,10 @@ namespace Wabbajack.Lib.NexusApi
public async Task<GetModFilesResponse> GetModFiles(Game game, int modid)
{
var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modid}/files.json";
return await GetCached<GetModFilesResponse>(url);
var result = await GetCached<GetModFilesResponse>(url);
if (result.files == null)
throw new InvalidOperationException("Got Null data from the Nexus while finding mod files");
return result;
}
public async Task<List<MD5Response>> GetModInfoFromMD5(Game game, string md5Hash)