Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes for when authors create lists with broken download links

Browse Source
This commit is contained in:
Timothy Baldridge 2020-05-13 06:09:20 -06:00
parent 4357564347
commit 8d8af953ac
5 changed files with 61 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Wabbajack.Server.Test/ModListValidationTests.cs
View File

@ -38,12 +38,16 @@ namespace Wabbajack.BuildServer.Test
Consts.ModlistMetadataURL = modlist.ToString();
var sql = Fixture.GetService<SqlService>();
var downloader = Fixture.GetService<ModListDownloader>();
await downloader.CheckForNewLists();
Assert.Equal(2, await downloader.CheckForNewLists());
foreach (var list in ModListMetaData)
{
Assert.True(await sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash));
}
// Nothing has changed so we shouldn't be downloading anything this time
Assert.Equal(0, await downloader.CheckForNewLists());
}
private async Task<Uri> MakeModList()
@ -90,6 +94,22 @@ namespace Wabbajack.BuildServer.Test
MachineURL = "test_list",
Download = MakeURL("test_modlist.wabbajack")
}
},
new ModlistMetadata
{
Official = true,
Author = "Test Suite",
Description = "A list with a broken hash",
DownloadMetadata = new DownloadMetadata()
{
Hash = Hash.FromLong(42),
Size = 42
},
Links = new ModlistMetadata.LinksObject
{
MachineURL = "broken_list",
Download = MakeURL("test_modlist.wabbajack")
}
}
};

1
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -323,6 +323,7 @@ CREATE TABLE [dbo].[ModLists](
[Hash] [bigint] NOT NULL,
[Metadata] [nvarchar](max) NOT NULL,
[Modlist] [nvarchar](max) NOT NULL,
[BrokenDownload] [tinyint] NOT NULL,
CONSTRAINT [PK_ModLists] PRIMARY KEY CLUSTERED
(
[MachineURL] ASC

7
Wabbajack.Server/DataLayer/ModLists.cs
View File

@ -9,7 +9,7 @@ namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist)
public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist, bool brokenDownload)
{
await using var conn = await Open();
await using var tran = await conn.BeginTransactionAsync();
@ -18,13 +18,14 @@ namespace Wabbajack.Server.DataLayer
new {MachineUrl = metadata.Links.MachineURL}, tran);
await conn.ExecuteAsync(
@"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList) VALUES (@MachineUrl, @Hash, @Metadata, @ModList)",
@"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList, BrokenDownload) VALUES (@MachineUrl, @Hash, @Metadata, @ModList, @BrokenDownload)",
new
{
MachineUrl = metadata.Links.MachineURL,
Hash = hash,
MetaData = metadata.ToJson(),
ModList = modlist.ToJson()
ModList = modlist.ToJson(),
BrokenDownload = brokenDownload
}, tran);
var entries = modlist.Archives.Select(a =>

2
Wabbajack.Server/Services/ArchiveMaintainer.cs
View File

@ -12,7 +12,7 @@ using File = System.IO.File;
namespace Wabbajack.Server.Services
{
/// <summary>
/// Maintains a concurrent cache of all the files we've downloaded, indexed by Hash
/// Maintains a concurrent cache of all the files we've downloaded, indexed by Hash.
/// </summary>
public class ArchiveMaintainer
{

48
Wabbajack.Server/Services/ModListDownloader.cs
View File

@ -1,5 +1,6 @@
using System;
using System.IO.Compression;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Logging;
@ -38,6 +39,7 @@ namespace Wabbajack.Server.Services
{
try
{
_logger.Log(LogLevel.Information, "Checking for updated mod lists");
await CheckForNewLists();
}
catch (Exception ex)
@ -52,31 +54,47 @@ namespace Wabbajack.Server.Services
}
}
public async Task CheckForNewLists()
public async Task<int> CheckForNewLists()
{
int downloaded = 0;
var lists = await ModlistMetadata.LoadFromGithub();
foreach (var list in lists)
{
try
{
if (_maintainer.HaveArchive(list.DownloadMetadata!.Hash))
if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
continue;
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
var tf = new TempFile();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
if (state == null)
if (!_maintainer.HaveArchive(list.DownloadMetadata!.Hash))
{
_logger.Log(LogLevel.Error,
$"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}");
continue;
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
var tf = new TempFile();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
if (state == null)
{
_logger.Log(LogLevel.Error,
$"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}");
continue;
}
downloaded += 1;
await state.Download(new Archive(state) {Name = $"{list.Links.MachineURL}.wabbajack"}, tf.Path);
var hash = await tf.Path.FileHashAsync();
if (hash != list.DownloadMetadata.Hash)
{
_logger.Log(LogLevel.Error,
$"Downloaded modlist {list.Links.MachineURL} {list.DownloadMetadata.Hash} didn't match metadata hash of {hash}");
await _sql.IngestModList(list.DownloadMetadata.Hash, list, new ModList(), true);
continue;
}
await _maintainer.Ingest(tf.Path);
}
await state.Download(new Archive(state) {Name = $"{list.Links.MachineURL}.wabbajack"}, tf.Path);
var modistPath = await _maintainer.Ingest(tf.Path);
_maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath);
ModList modlist;
await using (var fs = modistPath.OpenRead())
await using (var fs = modlistPath.OpenRead())
using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
await using (var entry = zip.GetEntry("modlist")?.Open())
{
@ -97,13 +115,15 @@ namespace Wabbajack.Server.Services
}
}
await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist);
await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist, false);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
}
}
_logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
return downloaded;
}
}