Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #714 from wabbajack-tools/issue-708

Browse Source 
Rewrite list validation and healing
This commit is contained in:
Timothy Baldridge 2020-04-15 16:33:43 -06:00 committed by GitHub
commit 7146035cca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 846 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Compression.BSA/BSABuilder.cs
View File

@ -280,7 +280,7 @@ namespace Compression.BSA
case VersionType.SSE:
{
var r = new MemoryStream();
using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L10_OPT}, true))
using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L12_MAX}, true))
{
_srcData.CopyTo(w);
}

104
Wabbajack.BuildServer.Test/ModListValidationTests.cs
View File

@ -6,6 +6,7 @@ using System.Security.Policy;
using System.Text;
using System.Threading.Tasks;
using HtmlAgilityPack;
using Wabbajack.BuildServer.BackendServices;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
@ -17,6 +18,7 @@ using Wabbajack.Lib.FileUploader;
using Wabbajack.Lib.ModListRegistry;
using Xunit;
using Xunit.Abstractions;
using IndexedFile = Wabbajack.BuildServer.Models.IndexedFile;
namespace Wabbajack.BuildServer.Test
{
@ -24,6 +26,7 @@ namespace Wabbajack.BuildServer.Test
{
public ModListValidationTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
@ -36,6 +39,21 @@ namespace Wabbajack.BuildServer.Test
Assert.Equal("test_list", data.First().Links.MachineURL);
}
[Fact]
public async Task CanIngestModLists()
{
var modlist = await MakeModList();
Consts.ModlistMetadataURL = modlist.ToString();
var sql = Fixture.GetService<SqlService>();
var service = new ListIngest(sql, Fixture.GetService<AppSettings>());
await service.Execute();
foreach (var list in ModListMetaData)
{
Assert.True(await sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash));
}
}
[Fact]
public async Task CanValidateModLists()
{
@ -57,8 +75,18 @@ namespace Wabbajack.BuildServer.Test
var archive = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
await archive.MoveToAsync(archive.WithExtension(new Extension(".moved")), true);
// We can revalidate but the non-nexus archives won't be checked yet since the list didn't change
await RevalidateLists();
data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
Assert.Equal(0, data.First().ValidationSummary.Failed);
Assert.Equal(1, data.First().ValidationSummary.Passed);
// Run the non-nexus validator
var evalService = new ValidateNonNexusArchives(Fixture.GetService<SqlService>(), Fixture.GetService<AppSettings>());
await evalService.Execute();
data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
Assert.Equal(1, data.First().ValidationSummary.Failed);
@ -70,6 +98,9 @@ namespace Wabbajack.BuildServer.Test
await archive.WithExtension(new Extension(".moved")).MoveToAsync(archive, false);
await RevalidateLists();
// Rerun the validation service to fix the list
await evalService.Execute();
data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
@ -80,22 +111,59 @@ namespace Wabbajack.BuildServer.Test
}
[Fact]
public async Task CanUpgradeHttpDownloads()
{
await ClearJobQueue();
var modlists = await MakeModList();
await IndexFile(ModListData.Archives.First());
Consts.ModlistMetadataURL = modlists.ToString();
Utils.Log("Updating modlists");
await RevalidateLists();
Utils.Log("Checking validated results");
var data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
Assert.Equal(0, data.First().ValidationSummary.Failed);
Assert.Equal(1, data.First().ValidationSummary.Passed);
await CheckListFeeds(0, 1);
var archive = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
archive.Delete();
await archive.WriteAllBytesAsync(Encoding.UTF8.GetBytes("More Cheese for Everyone!"));
var evalService = new ValidateNonNexusArchives(Fixture.GetService<SqlService>(), Fixture.GetService<AppSettings>());
await evalService.Execute();
await RevalidateLists();
Utils.Log("Checking updated results");
data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
Assert.Equal(0, data.First().ValidationSummary.Failed);
Assert.Equal(0, data.First().ValidationSummary.Passed);
Assert.Equal(1, data.First().ValidationSummary.Updating);
await CheckListFeeds(1, 0);
}
private async Task IndexFile(Archive archive)
{
var job = new IndexJob {Archive = archive};
await job.Execute(Fixture.GetService<SqlService>(), Fixture.GetService<AppSettings>());
}
private async Task RevalidateLists()
{
var result = await AuthorAPI.UpdateServerModLists();
Assert.NotNull(result);
var sql = Fixture.GetService<SqlService>();
var settings = Fixture.GetService<AppSettings>();
var job = await sql.GetJob();
Assert.NotNull(job);
Assert.IsType<UpdateModLists>(job.Payload);
job.Result = await job.Payload.Execute(sql, settings);
await sql.FinishJob(job);
Assert.Equal(JobResultType.Success, job.Result.ResultType);
var jobService = new ListIngest(sql, settings);
await jobService.Execute();
}
private async Task CheckListFeeds(int failed, int passed)
@ -122,8 +190,8 @@ namespace Wabbajack.BuildServer.Test
var modListData = new ModList();
modListData.Archives.Add(
ModListData = new ModList();
ModListData.Archives.Add(
new Archive(new HTTPDownloader.State(MakeURL("test_archive.txt")))
{
Hash = await test_archive_path.FileHashAsync(),
@ -138,10 +206,10 @@ namespace Wabbajack.BuildServer.Test
using var za = new ZipArchive(fs, ZipArchiveMode.Create);
var entry = za.CreateEntry("modlist.json");
await using var es = entry.Open();
modListData.ToJson(es);
ModListData.ToJson(es);
}
var modListMetaData = new List<ModlistMetadata>
ModListMetaData = new List<ModlistMetadata>
{
new ModlistMetadata
{
@ -163,9 +231,13 @@ namespace Wabbajack.BuildServer.Test
var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder);
modListMetaData.ToJson(metadataPath);
ModListMetaData.ToJson(metadataPath);
return new Uri(MakeURL("test_mod_list_metadata.json"));
}
public ModList ModListData { get; set; }
public List<ModlistMetadata> ModListMetaData { get; set; }
}
}

59
Wabbajack.BuildServer.Test/sql/wabbajack_db.sql
View File

@ -320,9 +320,9 @@ GO
/****** Object: Table [dbo].[ModLists] Script Date: 4/2/2020 3:59:19 PM ******/
CREATE TABLE [dbo].[ModLists](
[MachineURL] [nvarchar](50) NOT NULL,
[Summary] [nvarchar](max) NOT NULL,
[Hash] [bigint] NOT NULL,
[Metadata] [nvarchar](max) NOT NULL,
[DetailedStatus] [nvarchar](max) NOT NULL,
[Modlist] [nvarchar](max) NOT NULL,
CONSTRAINT [PK_ModLists] PRIMARY KEY CLUSTERED
(
[MachineURL] ASC
@ -330,6 +330,61 @@ CREATE TABLE [dbo].[ModLists](
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
/****** Object: Table [dbo].[ModListArchive] Script Date: 4/11/2020 10:33:20 AM ******/
CREATE TABLE [dbo].[ModListArchives](
[MachineUrl] [nvarchar](50) NOT NULL,
[Hash] [bigint] NOT NULL,
[PrimaryKeyString] [nvarchar](max) NOT NULL,
[Size] [bigint] NOT NULL,
[State] [nvarchar](max) NOT NULL,
CONSTRAINT [PK_ModListArchive] PRIMARY KEY CLUSTERED
(
[MachineUrl] ASC,
[Hash] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY]
GO
/****** Object: Table [dbo].[ModListArchiveStatus] Script Date: 4/11/2020 9:44:25 PM ******/
CREATE TABLE [dbo].[ModListArchiveStatus](
[PrimaryKeyStringHash] [binary](32) NOT NULL,
[Hash] [bigint] NOT NULL,
[PrimaryKeyString] [nvarchar](max) NOT NULL,
[IsValid] [tinyint] NOT NULL,
CONSTRAINT [PK_ModListArchiveStatus] PRIMARY KEY CLUSTERED
(
[PrimaryKeyStringHash] ASC,
[Hash] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
/****** Object: Table [dbo].[ArchivePatches] Script Date: 4/13/2020 9:39:25 PM ******/
CREATE TABLE [dbo].[ArchivePatches](
[SrcPrimaryKeyStringHash] [binary](32) NOT NULL,
[SrcPrimaryKeyString] [nvarchar](max) NOT NULL,
[SrcHash] [bigint] NOT NULL,
[DestPrimaryKeyStringHash] [binary](32) NOT NULL,
[DestPrimaryKeyString] [nvarchar](max) NOT NULL,
[DestHash] [bigint] NOT NULL,
[SrcState] [nvarchar](max) NOT NULL,
[DestState] [nvarchar](max) NOT NULL,
[SrcDownload] [nvarchar](max) NULL,
[DestDownload] [nvarchar](max) NULL,
[CDNPath] [nvarchar](max) NULL,
CONSTRAINT [PK_ArchivePatches] PRIMARY KEY CLUSTERED
(
[SrcPrimaryKeyStringHash] ASC,
[SrcHash] ASC,
[DestPrimaryKeyStringHash] ASC,
[DestHash] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
/****** Object: Table [dbo].[Metrics] Script Date: 3/28/2020 4:58:59 PM ******/
SET ANSI_NULLS ON
GO

45
Wabbajack.BuildServer/BackendServices/ABackendService.cs Normal file
View File

@ -0,0 +1,45 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.Common;
namespace Wabbajack.BuildServer.BackendServices
{
public abstract class ABackendService
{
protected ABackendService(SqlService sql, AppSettings settings, TimeSpan pollRate)
{
Sql = sql;
Settings = settings;
PollRate = pollRate;
}
public TimeSpan PollRate { get; }
public async Task RunLoop(CancellationToken token)
{
while (!token.IsCancellationRequested)
{
try
{
await Execute();
}
catch (Exception ex)
{
Utils.Log($"Error executing {this}");
Utils.Log(ex.ToString());
}
await Task.Delay(PollRate);
}
}
public abstract Task Execute();
protected AppSettings Settings { get; set; }
protected SqlService Sql { get; set; }
}
}

67
Wabbajack.BuildServer/BackendServices/ListIngest.cs Normal file
View File

@ -0,0 +1,67 @@
using System;
using System.Collections.Generic;
using System.IO.Compression;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.BuildServer.BackendServices
{
public class ListIngest : ABackendService
{
public ListIngest(SqlService sql, AppSettings settings) : base(sql, settings, TimeSpan.FromMinutes(1))
{
}
public override async Task Execute()
{
var client = new Common.Http.Client();
var lists = await client.GetJsonAsync<List<ModlistMetadata>>(Consts.ModlistMetadataURL);
bool newData = false;
foreach (var list in lists)
{
if (await Sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
continue;
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlistPath))
{
modlistPath.Delete();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlistPath);
}
else
{
Utils.Log($"No changes detected from downloaded modlist");
}
ModList modlist;
await using (var fs = modlistPath.OpenRead())
using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
await using (var entry = zip.GetEntry("modlist.json")?.Open())
{
if (entry == null)
{
Utils.Log($"Bad Modlist {list.Links.MachineURL}");
continue;
}
modlist = entry.FromJson<ModList>();
}
newData = true;
await Sql.IngestModList(list.DownloadMetadata.Hash, list, modlist);
}
if (newData)
{
var service = new ValidateNonNexusArchives(Sql, Settings);
await service.Execute();
}
}
}
}

35
Wabbajack.BuildServer/BackendServices/ValidateNonNexusArchives.cs Normal file
View File

@ -0,0 +1,35 @@
using System;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.Common;
namespace Wabbajack.BuildServer.BackendServices
{
public class ValidateNonNexusArchives : ABackendService
{
public ValidateNonNexusArchives(SqlService sql, AppSettings settings) : base(sql, settings, TimeSpan.FromHours(2))
{
}
public override async Task Execute()
{
var archives = await Sql.GetNonNexusModlistArchives();
using var queue = new WorkQueue();
var results = await archives.PMap(queue, async archive =>
{
try
{
var isValid = await archive.State.Verify(archive);
return (Archive: archive, IsValid: isValid);
}
catch (Exception)
{
return (Archive: archive, IsValid: false);
}
});
await Sql.UpdateNonNexusModlistArchivesStatus(results);
}
}
}

2
Wabbajack.BuildServer/Controllers/AControllerBase.cs
View File

@ -26,7 +26,7 @@ namespace Wabbajack.BuildServer.Controllers
{
await SQL.IngestMetric(new Metric
{
MetricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault(),
MetricsKey = Request?.Headers[Consts.MetricsKeyHeader].FirstOrDefault() ?? "",
Subject = subject,
Action = verb,
Timestamp = DateTime.UtcNow

139
Wabbajack.BuildServer/Controllers/ListValidation.cs
View File

@ -1,13 +1,20 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.BuildServer.Controllers
@ -16,15 +23,117 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/lists")]
public class ListValidation : AControllerBase<ListValidation>
{
public ListValidation(ILogger<ListValidation> logger, SqlService sql) : base(logger, sql)
enum ArchiveStatus
{
Valid,
InValid,
Updating,
Updated,
}
public ListValidation(ILogger<ListValidation> logger, SqlService sql, AppSettings settings) : base(logger, sql)
{
_updater = new ModlistUpdater(null, sql, settings);
_settings = settings;
}
public async Task<IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)>> GetSummaries()
{
var data = await SQL.GetValidationData();
using var queue = new WorkQueue();
var results = await data.ModLists.PMap(queue, async list =>
{
var (metadata, modList) = list;
var archives = await modList.Archives.PMap(queue, async archive =>
{
var (_, result) = ValidateArchive(data, archive);
if (result == ArchiveStatus.InValid)
{
return await TryToFix(data, archive);
}
return (archive, result);
});
var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);
var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated);
var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating);
var summary = new ModListSummary
{
Checked = DateTime.UtcNow,
Failed = failedCount,
Passed = passCount,
Updating = updatingCount,
MachineURL = metadata.Links.MachineURL,
Name = metadata.Title,
};
var detailed = new DetailedStatus
{
Name = metadata.Title,
Checked = DateTime.UtcNow,
DownloadMetaData = metadata.DownloadMetadata,
HasFailures = failedCount > 0,
MachineName = metadata.Links.MachineURL,
Archives = archives.Select(a => new DetailedStatusItem
{
Archive = a.Item1, IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
}).ToList()
};
return (summary, detailed);
});
return results;
}
private static (Archive archive, ArchiveStatus) ValidateArchive(SqlService.ValidationData data, Archive archive)
{
switch (archive.State)
{
case NexusDownloader.State nexusState when data.NexusFiles.Contains((
nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
return (archive, ArchiveStatus.Valid);
case NexusDownloader.State nexusState:
return (archive, ArchiveStatus.InValid);
case ManualDownloader.State _:
return (archive, ArchiveStatus.Valid);
default:
{
if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash),
out bool isValid))
{
return isValid ? (archive, ArchiveStatus.Valid) : (archive, ArchiveStatus.InValid);
}
return (archive, ArchiveStatus.InValid);
}
}
}
private static AsyncLock _findPatchLock = new AsyncLock();
private async Task<(Archive, ArchiveStatus)> TryToFix(SqlService.ValidationData data, Archive archive)
{
using var _ = await _findPatchLock.Wait();
var result = await _updater.GetAlternative(archive.Hash.ToHex());
return result switch
{
OkResult ok => (archive, ArchiveStatus.Updated),
AcceptedResult accept => (archive, ArchiveStatus.Updating),
_ => (archive, ArchiveStatus.InValid)
};
}
[HttpGet]
[Route("status.json")]
public async Task<IEnumerable<ModListSummary>> HandleGetLists()
{
return await SQL.GetModListSummaries();
return (await GetSummaries()).Select(d => d.Summary);
}
private static readonly Func<object, string> HandleGetRssFeedTemplate = NettleEngine.GetCompiler().Compile(@"
@ -48,7 +157,7 @@ namespace Wabbajack.BuildServer.Controllers
[Route("status/{Name}/broken.rss")]
public async Task<ContentResult> HandleGetRSSFeed(string Name)
{
var lst = await SQL.GetDetailedModlistStatus(Name);
var lst = await DetailedStatus(Name);
var response = HandleGetRssFeedTemplate(new
{
lst,
@ -81,12 +190,15 @@ namespace Wabbajack.BuildServer.Controllers
</body></html>
");
private AppSettings _settings;
private ModlistUpdater _updater;
[HttpGet]
[Route("status/{Name}.html")]
public async Task<ContentResult> HandleGetListHtml(string Name)
{
var lst = await SQL.GetDetailedModlistStatus(Name);
var lst = await DetailedStatus(Name);
var response = HandleGetListTemplate(new
{
lst,
@ -104,19 +216,16 @@ namespace Wabbajack.BuildServer.Controllers
[HttpGet]
[Route("status/{Name}.json")]
public async Task<ContentResult> HandleGetListJson(string Name)
public async Task<IActionResult> HandleGetListJson(string Name)
{
var lst = await SQL.GetDetailedModlistStatus(Name);
lst.Archives.Do(a => a.Archive.Meta = null);
return new ContentResult
{
ContentType = "application/json",
StatusCode = (int) HttpStatusCode.OK,
Content = lst.ToJson()
};
return Ok((await DetailedStatus(Name)).ToJson());
}
private async Task<DetailedStatus> DetailedStatus(string Name)
{
return (await GetSummaries())
.Select(d => d.Detailed)
.FirstOrDefault(d => d.MachineName == Name);
}
}
}

103
Wabbajack.BuildServer/Controllers/ModlistUpdater.cs
View File

@ -91,43 +91,34 @@ namespace Wabbajack.BuildServer.Controllers
Utils.Log($"Alternative requested for {startingHash}");
await Metric("requested_upgrade", startingHash.ToString());
var state = await SQL.GetNexusStateByHash(startingHash);
/*.DownloadStates.AsQueryable()
.Where(s => s.Hash == startingHash)
.Where(s => s.State is NexusDownloader.State)
.OrderByDescending(s => s.LastValidationTime).FirstOrDefaultAsync();*/
var archive = await SQL.GetStateByHash(startingHash);
if (state == null)
if (archive == null)
{
Utils.Log($"No original state for {startingHash}");
return NotFound("Original state not found");
}
var nexusState = state.State as NexusDownloader.State;
var nexusGame = nexusState.Game;
var nexusModFiles = await SQL.GetModFiles(nexusGame, nexusState.ModID);
if (nexusModFiles == null)
Archive newArchive;
IActionResult result;
switch (archive.State)
{
Utils.Log($"No nexus mod files for {startingHash}");
return NotFound("No nexus info");
}
var mod_files = nexusModFiles.files;
if (mod_files.Any(f => f.category_name != null && f.file_id == nexusState.FileID))
{
Utils.Log($"No available upgrade required for {nexusState.PrimaryKey}");
await Metric("not_required_upgrade", startingHash.ToString());
return BadRequest("Upgrade Not Required");
case NexusDownloader.State _:
{
(result, newArchive) = await FindNexusAlternative(archive);
if (newArchive == null)
return result;
break;
}
case HTTPDownloader.State _:
(result, newArchive) = await FindHttpAlternative(archive);
if (newArchive == null)
return result;
break;
default:
return NotFound("No alternative");
}
Utils.Log($"Found original, looking for alternatives to {startingHash}");
var newArchive = await FindAlternatives(nexusState, startingHash);
if (newArchive == null)
{
Utils.Log($"No available upgrade for {nexusState.PrimaryKey}");
return NotFound("No alternative available");
}
Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}");
if (newArchive.Hash == Hash.Empty)
@ -168,7 +159,61 @@ namespace Wabbajack.BuildServer.Controllers
return Ok(newArchive.ToJson());
}
private async Task<Archive> FindAlternatives(NexusDownloader.State state, Hash srcHash)
private async Task<(IActionResult, Archive)> FindHttpAlternative(Archive archive)
{
try
{
var valid = await archive.State.Verify(archive);
if (valid)
{
Utils.Log($"Http file {archive.Hash} is still valid");
return (NotFound("Http file still valid"), null);
}
archive.Hash = default;
archive.Size = 0;
return (Ok("Index"), archive);
}
catch
{
Utils.Log($"Http file {archive.Hash} no longer exists");
return (NotFound("Http file no longer exists"), null);
}
}
private async Task<(IActionResult, Archive)> FindNexusAlternative(Archive archive)
{
var nexusState = (NexusDownloader.State)archive.State;
var nexusGame = nexusState.Game;
var nexusModFiles = await SQL.GetModFiles(nexusGame, nexusState.ModID);
if (nexusModFiles == null)
{
Utils.Log($"No nexus mod files for {archive.Hash}");
return (NotFound("No nexus info"), null);
}
var mod_files = nexusModFiles.files;
if (mod_files.Any(f => f.category_name != null && f.file_id == nexusState.FileID))
{
Utils.Log($"No available upgrade required for {nexusState.PrimaryKey}");
await Metric("not_required_upgrade", archive.Hash.ToString());
return (BadRequest("Upgrade Not Required"), null);
}
Utils.Log($"Found original, looking for alternatives to {archive.Hash}");
var newArchive = await FindNexusAlternative(nexusState, archive.Hash);
if (newArchive != null)
{
return (Ok(newArchive), newArchive);
}
Utils.Log($"No available upgrade for {nexusState.PrimaryKey}");
return (NotFound("No alternative available"), null);
}
private async Task<Archive> FindNexusAlternative(NexusDownloader.State state, Hash srcHash)
{
var origSize = _settings.PathForArchive(srcHash).Size;
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());

2
Wabbajack.BuildServer/Models/JobQueue/Job.cs
View File

@ -1,8 +1,10 @@
using System;
using System.Threading.Tasks;
using Wabbajack.Common.Serialization.Json;
namespace Wabbajack.BuildServer.Models.JobQueue
{
[JsonName("Job")]
public class Job
{
public enum JobPriority : int

11
Wabbajack.BuildServer/Models/Jobs/IndexJob.cs
View File

@ -18,8 +18,12 @@ namespace Wabbajack.BuildServer.Models.Jobs
public class IndexJob : AJobPayload, IBackEndJob
{
public Archive Archive { get; set; }
public bool ForceIndex { get; set; }
public override string Description => $"Index ${Archive.State.PrimaryKeyString} and save the download/file state";
public override bool UsesNexus { get => Archive.State is NexusDownloader.State; }
public Hash DownloadedHash { get; set; }
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{
if (Archive.State is ManualDownloader.State)
@ -31,10 +35,10 @@ namespace Wabbajack.BuildServer.Models.Jobs
var pkStr = string.Join("|",pk.Select(p => p.ToString()));
var found = await sql.DownloadStateByPrimaryKey(pkStr);
if (found != null)
if (found != null && !ForceIndex)
return JobResult.Success();
string fileName = Archive.Name;
string fileName = Archive.Name ?? Guid.NewGuid().ToString();
string folder = Guid.NewGuid().ToString();
Utils.Log($"Indexer is downloading {fileName}");
var downloadDest = settings.DownloadPath.Combine(folder, fileName);
@ -46,6 +50,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
await vfs.AddRoot(settings.DownloadPath.Combine(folder));
var archive = vfs.Index.ByRootPath.First().Value;
DownloadedHash = archive.Hash;
await sql.MergeVirtualFile(archive);
await sql.AddDownloadState(archive.Hash, Archive.State);
@ -63,6 +69,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
protected override IEnumerable<object> PrimaryKey => Archive.State.PrimaryKey;
}

289
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -6,11 +6,13 @@ using System.Data.SqlClient;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Microsoft.AspNetCore.Mvc.ModelBinding.Metadata;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Wabbajack.BuildServer.Model.Models.Results;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
@ -202,14 +204,14 @@ namespace Wabbajack.BuildServer.Model.Models
new {
job.Id,
Success = job.Result.ResultType == JobResultType.Success,
ResultContent = job.Result.ToJson()
ResultContent = job.Result
});
if (job.OnSuccess != null)
await EnqueueJob(job.OnSuccess);
}
/// <summary>
/// Get a Job from the Job queue to run.
/// </summary>
@ -217,16 +219,24 @@ namespace Wabbajack.BuildServer.Model.Models
public async Task<Job> GetJob()
{
await using var conn = await Open();
var result = await conn.QueryAsync<Job>(
var result = await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>(
@"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy
WHERE ID in (SELECT TOP(1) ID FROM Jobs
WHERE Started is NULL
AND PrimaryKeyString NOT IN (SELECT PrimaryKeyString from jobs WHERE Started IS NOT NULL and Ended IS NULL)
ORDER BY Priority DESC, Created);
SELECT TOP(1) * FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC",
SELECT TOP(1) Id, Started, Ended, Created, Payload, Priority FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC",
new {RunBy = Guid.NewGuid().ToString()});
return result.FirstOrDefault();
}
return result.Select(k =>
new Job {
Id = k.Item1,
Started = k.Item2,
Ended = k.Item3,
Created = k.Item4,
Payload = k.Item5,
Priority = (Job.JobPriority)k.Item6
}).FirstOrDefault();
}
public async Task<IEnumerable<Job>> GetRunningJobs()
@ -254,20 +264,37 @@ namespace Wabbajack.BuildServer.Model.Models
static SqlService()
{
SqlMapper.AddTypeHandler(new PayloadMapper());
SqlMapper.AddTypeHandler(new HashMapper());
SqlMapper.AddTypeHandler(new RelativePathMapper());
SqlMapper.AddTypeHandler(new JsonMapper<AbstractDownloadState>());
SqlMapper.AddTypeHandler(new JsonMapper<AJobPayload>());
SqlMapper.AddTypeHandler(new JsonMapper<JobResult>());
SqlMapper.AddTypeHandler(new JsonMapper<Job>());
}
public class PayloadMapper : SqlMapper.TypeHandler<AJobPayload>
public class JsonMapper<T> : SqlMapper.TypeHandler<T>
{
public override void SetValue(IDbDataParameter parameter, AJobPayload value)
public override void SetValue(IDbDataParameter parameter, T value)
{
parameter.Value = value.ToJson();
}
public override AJobPayload Parse(object value)
public override T Parse(object value)
{
return ((string)value).FromJsonString<AJobPayload>();
return ((string)value).FromJsonString<T>();
}
}
public class RelativePathMapper : SqlMapper.TypeHandler<RelativePath>
{
public override void SetValue(IDbDataParameter parameter, RelativePath value)
{
parameter.Value = value.ToJson();
}
public override RelativePath Parse(object value)
{
return (RelativePath)(string)value;
}
}
@ -530,6 +557,20 @@ namespace Wabbajack.BuildServer.Model.Models
};
}
public async Task<Archive> GetStateByHash(Hash startingHash)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(string, long)>(@"SELECT JsonState, indexed.Size FROM dbo.DownloadStates state
LEFT JOIN dbo.IndexedFile indexed ON indexed.Hash = state.Hash
WHERE state.Hash = @hash",
new {Hash = (long)startingHash});
return result == default ? null : new Archive(result.Item1.FromJsonString<AbstractDownloadState>())
{
Hash = startingHash,
Size = result.Item2
};
}
public async Task<Archive> DownloadStateByPrimaryKey(string primaryKey)
{
await using var conn = await Open();
@ -596,20 +637,220 @@ namespace Wabbajack.BuildServer.Model.Models
public async Task UpdateModListStatus(ModListStatus dto)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"MERGE dbo.ModLists AS Target
USING (SELECT @MachineUrl MachineUrl, @Metadata Metadata, @Summary Summary, @DetailedStatus DetailedStatus) AS Source
ON Target.MachineUrl = Source.MachineUrl
WHEN MATCHED THEN UPDATE SET Target.Summary = Source.Summary, Target.Metadata = Source.Metadata, Target.DetailedStatus = Source.DetailedStatus
WHEN NOT MATCHED THEN INSERT (MachineUrl, Summary, Metadata, DetailedStatus) VALUES (@MachineUrl, @Summary, @Metadata, @DetailedStatus);",
new
{
MachineUrl = dto.Metadata.Links.MachineURL,
Metadata = dto.Metadata.ToJson(),
Summary = dto.Summary.ToJson(),
DetailedStatus = dto.DetailedStatus.ToJson()
});
}
public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist)
{
await using var conn = await Open();
await using var tran = await conn.BeginTransactionAsync();
await conn.ExecuteAsync(@"DELETE FROM dbo.ModLists Where MachineUrl = @MachineUrl",
new {MachineUrl = metadata.Links.MachineURL}, tran);
await conn.ExecuteAsync(
@"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList) VALUES (@MachineUrl, @Hash, @Metadata, @ModList)",
new
{
MachineUrl = metadata.Links.MachineURL,
Hash = hash,
MetaData = metadata.ToJson(),
ModList = modlist.ToJson()
}, tran);
var entries = modlist.Archives.Select(a =>
new
{
MachineUrl = metadata.Links.MachineURL,
Hash = a.Hash,
Size = a.Size,
State = a.State.ToJson(),
PrimaryKeyString = a.State.PrimaryKeyString
}).ToArray();
await conn.ExecuteAsync(@"DELETE FROM dbo.ModListArchives WHERE MachineURL = @machineURL",
new {MachineUrl = metadata.Links.MachineURL}, tran);
foreach (var entry in entries)
{
await conn.ExecuteAsync(
"INSERT INTO dbo.ModListArchives (MachineURL, Hash, Size, PrimaryKeyString, State) VALUES (@MachineURL, @Hash, @Size, @PrimaryKeyString, @State)",
entry, tran);
}
await tran.CommitAsync();
}
public async Task<bool> HaveIndexedModlist(string machineUrl, Hash hash)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT MachineURL from dbo.Modlists WHERE MachineURL = @MachineUrl AND Hash = @Hash",
new {MachineUrl = machineUrl, Hash = hash});
return result != null;
}
public async Task<List<Archive>> GetNonNexusModlistArchives()
{
await using var conn = await Open();
var results = await conn.QueryAsync<(Hash Hash, long Size, string State)>(
@"SELECT Hash, Size, State FROM dbo.ModListArchives WHERE PrimaryKeyString NOT LIKE 'NexusDownloader+State|%'");
return results.Select(r => new Archive (r.State.FromJsonString<AbstractDownloadState>())
{
Size = r.Size,
Hash = r.Hash,
}).ToList();}
public async Task UpdateNonNexusModlistArchivesStatus(IEnumerable<(Archive Archive, bool IsValid)> results)
{
await using var conn = await Open();
var trans = await conn.BeginTransactionAsync();
await conn.ExecuteAsync("DELETE FROM dbo.ModlistArchiveStatus;", transaction:trans);
foreach (var itm in results.DistinctBy(itm => (itm.Archive.Hash, itm.Archive.State.PrimaryKeyString)))
{
await conn.ExecuteAsync(
@"INSERT INTO dbo.ModlistArchiveStatus (PrimaryKeyStringHash, PrimaryKeyString, Hash, IsValid)
VALUES (HASHBYTES('SHA2_256', @PrimaryKeyString), @PrimaryKeyString, @Hash, @IsValid)", new
{
PrimaryKeyString = itm.Archive.State.PrimaryKeyString,
Hash = itm.Archive.Hash,
IsValid = itm.IsValid
}, trans);
}
await trans.CommitAsync();
}
public async Task<ValidationData> GetValidationData()
{
var nexusFiles = AllNexusFiles();
var archiveStatus = AllModListArchivesStatus();
var modLists = AllModLists();
var archivePatches = AllArchivePatches();
return new ValidationData
{
NexusFiles = await nexusFiles,
ArchiveStatus = await archiveStatus,
ModLists = await modLists,
ArchivePatches = await archivePatches
};
}
public async Task<Dictionary<(string PrimaryKeyString, Hash Hash), bool>> AllModListArchivesStatus()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<(string, Hash, bool)>(
@"SELECT PrimaryKeyString, Hash, IsValid FROM dbo.ModListArchiveStatus");
return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3);
}
public async Task<HashSet<(long NexusGameId, long ModId, long FileId)>> AllNexusFiles()
{
await using var conn = await Open();
var results = await conn.QueryAsync<(long, long, long)>(@"SELECT Game, ModId, p.file_id
FROM [NexusModFiles] files
CROSS APPLY
OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p
WHERE p.category is not null");
return results.ToHashSet();
}
public async Task<List<(ModlistMetadata, ModList)>> AllModLists()
{
await using var conn = await Open();
var results = await conn.QueryAsync<(string, string)>(@"SELECT Metadata, ModList FROM dbo.ModLists");
return results.Select(m => (m.Item1.FromJsonString<ModlistMetadata>(), m.Item2.FromJsonString<ModList>())).ToList();
}
public class ValidationData
{
public HashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; }
public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
public List<(ModlistMetadata Metadata, ModList ModList)> ModLists { get; set; }
public List<ArchivePatch> ArchivePatches { get; set; }
}
#region ArchivePatches
public class ArchivePatch
{
public Hash SrcHash { get; set; }
public AbstractDownloadState SrcState { get; set; }
public Hash DestHash { get; set; }
public AbstractDownloadState DestState { get; set; }
public RelativePath DestDownload { get; set; }
public RelativePath SrcDownload { get; set; }
public Uri CDNPath { get; set; }
}
public async Task UpsertArchivePatch(ArchivePatch patch)
{
await using var conn = await Open();
await using var trans = conn.BeginTransaction();
await conn.ExecuteAsync(@"DELETE FROM dbo.ArchivePatches
WHERE SrcHash = @SrcHash
AND DestHash = @DestHash
AND SrcPrimaryKeyStringHash = HASHBYTES('SHA2-256', @SrcPrimaryKeyString)
AND DestPrimaryKeyStringHash = HASHBYTES('SHA2-256', @DestPrimaryKeyString)",
new
{
SrcHash = patch.SrcHash,
DestHash = patch.DestHash,
SrcPrimaryKeyString = patch.SrcState.PrimaryKeyString,
DestPrimaryKeyString = patch.DestState.PrimaryKeyString
}, trans);
await conn.ExecuteAsync(@"INSERT INTO dbo.ArchivePatches
(SrcHash, SrcPrimaryKeyString, SrcPrimaryKeyStringHash, SrcState,
DestHash, DestPrimaryKeyString, DestPrimaryKeyStringHash, DestState,
SrcDownload, DestDownload, CDNPath)
VALUES (@SrcHash, @SrcPrimaryKeyString, HASHBYTES('SHA2-256', @SrcPrimaryKeyString), @SrcState,
@DestHash, @DestPrimaryKeyString, HASHBYTES('SHA2-256', @DestPrimaryKeyString), @DestState,
@SrcDownload, @DestDownload, @CDNPAth)",
new
{
SrcHash = patch.SrcHash,
DestHash = patch.DestHash,
SrcPrimaryKeyString = patch.SrcState.PrimaryKeyString,
DestPrimaryKeyString = patch.DestState.PrimaryKeyString,
SrcState = patch.SrcState.ToJson(),
DestState = patch.DestState.ToString(),
DestDownload = patch.DestDownload,
SrcDownload = patch.SrcDownload,
CDNPath = patch.CDNPath
}, trans);
await trans.CommitAsync();
}
public async Task<List<ArchivePatch>> AllArchivePatches()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<(Hash, AbstractDownloadState, Hash, AbstractDownloadState, RelativePath, RelativePath, Uri)>(
@"SELECT SrcHash, SrcState, DestHash, DestState, SrcDownload, DestDownload, CDNPath FROM dbo.ArchivePatches");
return results.Select(a => new ArchivePatch
{
SrcHash = a.Item1,
SrcState = a.Item2,
DestHash = a.Item3,
DestState = a.Item4,
SrcDownload = a.Item5,
DestDownload = a.Item6,
CDNPath = a.Item7
}).ToList();
}
#endregion
}
}

1
Wabbajack.BuildServer/Startup.cs
View File

@ -143,6 +143,7 @@ namespace Wabbajack.BuildServer
FileProvider = new PhysicalFileProvider(
Path.Combine(Directory.GetCurrentDirectory(), "public")),
StaticFileOptions = {ServeUnknownFileTypes = true},
});
app.UseEndpoints(endpoints =>

41
Wabbajack.Common/GameMetaData.cs
View File

@ -207,8 +207,15 @@ namespace Wabbajack.Common
public static bool TryGetByFuzzyName(string someName, [MaybeNullWhen(false)] out GameMetaData gameMetaData)
{
gameMetaData = TryGetByFuzzyName(someName);
return gameMetaData != null;
var result = TryGetByFuzzyName(someName);
if (result == null)
{
gameMetaData = Games.Values.First();
return false;
}
gameMetaData = result;
return true;
}
public static IReadOnlyDictionary<Game, GameMetaData> Games = new Dictionary<Game, GameMetaData>
@ -535,21 +542,21 @@ namespace Wabbajack.Common
}
}
},
{
Game.Enderal, new GameMetaData
{
SupportedModManager = ModManager.MO2,
Game = Game.Enderal,
NexusName = "enderal",
MO2Name = "Enderal",
MO2ArchiveName = "enderal",
SteamIDs = new List<int>{1027920},
RequiredFiles = new List<string>
{
"TESV.exe"
},
MainExecutable = "TESV.exe"
}
{
Game.Enderal, new GameMetaData
{
SupportedModManager = ModManager.MO2,
Game = Game.Enderal,
NexusName = "enderal",
MO2Name = "Enderal",
MO2ArchiveName = "enderal",
SteamIDs = new List<int>{1027920},
RequiredFiles = new List<string>
{
"TESV.exe"
},
MainExecutable = "TESV.exe"
}
}
};

13
Wabbajack.Common/Json.cs
View File

@ -305,7 +305,18 @@ namespace Wabbajack.Common
if (!_typeToName.ContainsKey(serializedType))
{
throw new InvalidDataException($"No Binding name for {serializedType}");
var custom = serializedType.GetCustomAttributes(false)
.OfType<JsonNameAttribute>().FirstOrDefault();
if (custom == null)
{
throw new InvalidDataException($"No Binding name for {serializedType}");
}
_nameToType[custom.Name] = serializedType;
_typeToName[serializedType] = custom.Name;
assemblyName = null;
typeName = custom.Name;
return;
}
var name = _typeToName[serializedType];

6
Wabbajack.Common/Util/DiskSlabAllocator.cs
View File

@ -30,8 +30,12 @@ namespace Wabbajack.Common
{
lock (this)
{
// This can happen at times due to differences in compression sizes
if (_head + size >= _size)
throw new InvalidDataException($"Size out of range. Declared {_size} used {_head + size}");
{
return new MemoryStream();
}
var startAt = _head;
_head += size;
var stream = _mmap.CreateViewStream(startAt, size, MemoryMappedFileAccess.ReadWrite);

2
Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs
View File

@ -66,7 +66,7 @@ namespace Wabbajack.Lib.CompilationSteps
var id = Guid.NewGuid().ToString();
var matches = await sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e, Consts.BSACreationDir.Combine((RelativePath)id, e.Name.FileName))));
var matches = await sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e, Consts.BSACreationDir.Combine((RelativePath)id, (RelativePath)e.Name))));
foreach (var match in matches)

2
Wabbajack.Lib/CompilationSteps/IncludeDummyESPs.cs
View File

@ -17,7 +17,7 @@ namespace Wabbajack.Lib.CompilationSteps
source.AbsolutePath.Extension != Consts.ESM) return null;
var bsa = source.AbsolutePath.ReplaceExtension(Consts.BSA);
var bsaTextures = source.AbsolutePath.AppendToName(" - Textures");
var bsaTextures = source.AbsolutePath.AppendToName(" - Textures").ReplaceExtension(Consts.BSA);
if (source.AbsolutePath.Size > 250 || !bsa.IsFile && !bsaTextures.IsFile) return null;

7
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -14,7 +14,7 @@ namespace Wabbajack.Lib.Downloaders
string? Name { get; set; }
string? Author { get; set; }
string? Version { get; set; }
string? ImageURL { get; set; }
Uri? ImageURL { get; set; }
bool IsNSFW { get; set; }
string? Description { get; set; }
@ -54,7 +54,6 @@ namespace Wabbajack.Lib.Downloaders
[JsonIgnore]
public abstract object[] PrimaryKey { get; }
[JsonIgnore]
public string PrimaryKeyString
{
get
@ -83,9 +82,7 @@ namespace Wabbajack.Lib.Downloaders
public async Task<bool> Download(AbsolutePath destination)
{
destination.Parent.CreateDirectory();
// ToDo
// Is this null override needed? Why is state allowed to be null here?
return await Download(new Archive(state: null!) {Name = (string)destination.FileName}, destination);
return await Download(new Archive(this) {Name = (string)destination.FileName}, destination);
}
/// <summary>

2
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -86,7 +86,7 @@ namespace Wabbajack.Lib.Downloaders
public string? Name { get; set; }
public string? Author { get; set; }
public string? Version { get; set; }
public string? ImageURL { get; set; }
public Uri? ImageURL { get; set; }
public virtual bool IsNSFW { get; set; }
public string? Description { get; set; }

15
Wabbajack.Lib/Downloaders/LoversLabDownloader.cs
View File

@ -63,20 +63,25 @@ namespace Wabbajack.Lib.Downloaders
?
.First().InnerHtml);
ImageURL = HttpUtility.HtmlDecode(node
var url = HttpUtility.HtmlDecode(node
.SelectNodes(
"//div[@class='ipsBox ipsSpacer_top ipsSpacer_double']/section/div[@class='ipsPad ipsAreaBackground']/div[@class='ipsCarousel ipsClearfix']/div[@class='ipsCarousel_inner']/ul[@class='cDownloadsCarousel ipsClearfix']/li[@class='ipsCarousel_item ipsAreaBackground_reset ipsPad_half']/span[@class='ipsThumb ipsThumb_medium ipsThumb_bg ipsCursor_pointer']")
?.First().GetAttributeValue("data-fullurl", "none"));
if (!string.IsNullOrWhiteSpace(ImageURL))
if (!string.IsNullOrWhiteSpace(url))
{
ImageURL = new Uri(url);
return true;
}
ImageURL = HttpUtility.HtmlDecode(node
url = HttpUtility.HtmlDecode(node
.SelectNodes(
"//article[@class='ipsColumn ipsColumn_fluid']/div[@class='ipsPad']/section/div[@class='ipsType_richText ipsContained ipsType_break']/p/a/img[@class='ipsImage ipsImage_thumbnailed']")
?.First().GetAttributeValue("src", ""));
if (string.IsNullOrWhiteSpace(ImageURL))
ImageURL = "";
if (!string.IsNullOrWhiteSpace(url))
{
ImageURL = new Uri(url);
}
return true;
}

2
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -141,7 +141,7 @@ namespace Wabbajack.Lib.Downloaders
public string? Version { get; set; }
public string? ImageURL { get; set; }
public Uri? ImageURL { get; set; }
public bool IsNSFW { get; set; }

3
Wabbajack.Lib/ModListRegistry/ModListMetadata.cs
View File

@ -116,6 +116,9 @@ namespace Wabbajack.Lib.ModListRegistry
public int Failed { get; set; }
[JsonProperty("passed")]
public int Passed { get; set; }
[JsonProperty("updating")]
public int Updating { get; set; }
[JsonProperty("link")]
public string Link => $"/lists/status/{MachineURL}.json";
[JsonProperty("report")]

21
Wabbajack.Lib/NexusApi/Dtos.cs
View File

@ -34,16 +34,27 @@ namespace Wabbajack.Lib.NexusApi
public class ModInfo
{
public uint _internal_version { get; set; }
public string game_name { get; set; } = string.Empty;
public string mod_id { get; set; } = string.Empty;
public string name { get; set; } = string.Empty;
public string summary { get; set; } = string.Empty;
public string description { get; set; } = string.Empty;
public Uri? picture_url { get; set; }
public string mod_id { get; set; } = string.Empty;
public long game_id { get; set; }
public bool allow_rating { get; set; }
public string domain_name { get; set; } = string.Empty;
public long category_id { get; set; }
public string version { get; set; } = string.Empty;
public long endorsement_count { get; set; }
public long created_timestamp { get; set; }
public DateTime created_time { get; set; }
public long updated_timestamp { get; set; }
public DateTime updated_time { get; set; }
public string author { get; set; } = string.Empty;
public string uploaded_by { get; set; } = string.Empty;
public string uploaded_users_profile_url { get; set; } = string.Empty;
public string picture_url { get; set; } = string.Empty;
public Uri? uploaded_users_profile_url { get; set; }
public bool contains_adult_content { get; set; }
public string status { get; set; } = string.Empty;
public bool available { get; set; } = true;
}
public class MD5Response

2
Wabbajack/View Models/ModVM.cs
View File

@ -21,7 +21,7 @@ namespace Wabbajack
{
State = state;
ImageObservable = Observable.Return(State.ImageURL)
ImageObservable = Observable.Return(State.ImageURL.ToString())
.ObserveOn(RxApp.TaskpoolScheduler)
.DownloadBitmapImage((ex) => Utils.Log($"Skipping slide for mod {State.Name}"))
.Replay(1)