Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Speed up list validation on the srver

Browse Source
This commit is contained in:
Timothy Baldridge 2020-07-15 16:29:43 -06:00
parent 4bff2a3336
commit 0c39698225
6 changed files with 61 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.Server/DTOs/ValidationData.cs
View File

@ -9,7 +9,7 @@ namespace Wabbajack.Server.DTOs
{
public ConcurrentHashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } = new ConcurrentHashSet<(long Game, long ModId, long FileId)>();
public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
public List<(ModlistMetadata Metadata, ModList ModList)> ModLists { get; set; }
public List<ModlistMetadata> ModLists { get; set; }
public ConcurrentHashSet<(Game Game, long ModId)> SlowQueriedFor { get; set; } = new ConcurrentHashSet<(Game Game, long ModId)>();
}

3
Wabbajack.Server/DataLayer/Mappers.cs
View File

@ -2,8 +2,10 @@
using System.Data;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.AuthorApi;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.Server.DataLayer
{
@ -15,6 +17,7 @@ namespace Wabbajack.Server.DataLayer
SqlMapper.AddTypeHandler(new RelativePathMapper());
SqlMapper.AddTypeHandler(new JsonMapper<AbstractDownloadState>());
SqlMapper.AddTypeHandler(new JsonMapper<CDNFileDefinition>());
SqlMapper.AddTypeHandler(new JsonMapper<ModlistMetadata>());
SqlMapper.AddTypeHandler(new VersionMapper());
SqlMapper.AddTypeHandler(new GameMapper());
SqlMapper.AddTypeHandler(new DateTimeHandler());

19
Wabbajack.Server/DataLayer/ModLists.cs
View File

@ -1,9 +1,11 @@
using System.Linq;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Lib;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.Server.DataLayer
{
@ -17,6 +19,11 @@ namespace Wabbajack.Server.DataLayer
await conn.ExecuteAsync(@"DELETE FROM dbo.ModLists Where MachineUrl = @MachineUrl",
new {MachineUrl = metadata.Links.MachineURL}, tran);
var archives = modlist.Archives;
var directives = modlist.Directives;
modlist.Archives = new List<Archive>();
modlist.Directives = new List<Directive>();
await conn.ExecuteAsync(
@"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList, BrokenDownload) VALUES (@MachineUrl, @Hash, @Metadata, @ModList, @BrokenDownload)",
new
@ -28,7 +35,7 @@ namespace Wabbajack.Server.DataLayer
BrokenDownload = brokenDownload
}, tran);
var entries = modlist.Archives.Select(a =>
var entries = archives.Select(a =>
new
{
MachineUrl = metadata.Links.MachineURL,
@ -67,5 +74,13 @@ namespace Wabbajack.Server.DataLayer
new {Hash = hash});
return result;
}
public async Task<List<Archive>> ModListArchives(string machineURL)
{
await using var conn = await Open();
var archives = await conn.QueryAsync<(Hash, long, AbstractDownloadState)>("SELECT Hash, Size, State FROM dbo.ModListArchives WHERE MachineUrl = @MachineUrl",
new {MachineUrl = machineURL});
return archives.Select(t => new Archive(t.Item3) {Size = t.Item2, Hash = t.Item1}).ToList();
}
}
}

6
Wabbajack.Server/DataLayer/ValidationData.cs
View File

@ -48,11 +48,11 @@ namespace Wabbajack.Server.DataLayer
return results.ToHashSet();
}
public async Task<List<(ModlistMetadata, ModList)>> AllModLists()
public async Task<List<ModlistMetadata>> AllModLists()
{
await using var conn = await Open();
var results = await conn.QueryAsync<(string, string)>(@"SELECT Metadata, ModList FROM dbo.ModLists");
return results.Select(m => (m.Item1.FromJsonString<ModlistMetadata>(), m.Item2.FromJsonString<ModList>())).ToList();
var results = await conn.QueryAsync<ModlistMetadata>(@"SELECT Metadata FROM dbo.ModLists");
return results.ToList();
}
}
}

8
Wabbajack.Server/Services/ListValidator.cs
View File

@ -46,13 +46,13 @@ namespace Wabbajack.Server.Services
using var queue = new WorkQueue();
var oldSummaries = Summaries;
var results = await data.ModLists.PMap(queue, async list =>
var results = await data.ModLists.PMap(queue, async metadata =>
{
var oldSummary =
oldSummaries.FirstOrDefault(s => s.Summary.MachineURL == list.Metadata.Links.MachineURL);
oldSummaries.FirstOrDefault(s => s.Summary.MachineURL == metadata.Links.MachineURL);
var (metadata, modList) = list;
var archives = await modList.Archives.PMap(queue, async archive =>
var listArchives = await _sql.ModListArchives(metadata.Links.MachineURL);
var archives = await listArchives.PMap(queue, async archive =>
{
var (_, result) = await ValidateArchive(data, archive);
if (result == ArchiveStatus.InValid)

38
Wabbajack.Server/Services/PatchBuilder.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
@ -118,7 +119,13 @@ namespace Wabbajack.Server.Services
private static string PatchName(Patch patch)
{
return $"{Consts.ArchiveUpdatesCDNFolder}\\{patch.Src.Archive.Hash.ToHex()}_{patch.Dest.Archive.Hash.ToHex()}";
return PatchName(patch.Src.Archive.Hash, patch.Dest.Archive.Hash);
}
private static string PatchName(Hash oldHash, Hash newHash)
{
return $"{Consts.ArchiveUpdatesCDNFolder}\\{oldHash.ToHex()}_{newHash.ToHex()}";
}
private async Task CleanupOldPatches()
@ -154,18 +161,39 @@ namespace Wabbajack.Server.Services
var sqlFiles = await _sql.AllPatchHashes();
_logger.LogInformation($"Found {sqlFiles.Count} in SQL");
var hashPairs = files.Select(f => f.Name).Where(f => f.Contains("_")).Select(p =>
HashSet<(Hash, Hash)> NamesToPairs(IEnumerable<FtpListItem> ftpFiles)
{
var lst = p.Split("_", StringSplitOptions.RemoveEmptyEntries).Select(Hash.FromHex).ToArray();
return (lst[0], lst[1]);
}).ToHashSet();
return ftpFiles.Select(f => f.Name).Where(f => f.Contains("_")).Select(p =>
{
try
{
var lst = p.Split("_", StringSplitOptions.RemoveEmptyEntries).Select(Hash.FromHex).ToArray();
return (lst[0], lst[1]);
}
catch (FormatException ex)
{
return default;
}
}).Where(f => f != default).ToHashSet();
}
var oldHashPairs = NamesToPairs(files.Where(f => DateTime.UtcNow - f.Modified > TimeSpan.FromDays(2)));
foreach (var (oldHash, newHash) in oldHashPairs.Where(o => !sqlFiles.Contains(o)))
{
_logger.LogInformation($"Removing CDN File entry for {oldHash} -> {newHash} it's not SQL");
await client.DeleteFileAsync(PatchName(oldHash, newHash));
}
var hashPairs = NamesToPairs(files);
foreach (var sqlFile in sqlFiles.Where(s => !hashPairs.Contains(s)))
{
_logger.LogInformation($"Removing SQL File entry for {sqlFile.Item1} -> {sqlFile.Item2} it's not on the CDN");
await _sql.DeletePatchesForHashPair(sqlFile);
}
}
private async Task UploadToCDN(AbsolutePath patchFile, string patchName)