Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tests for INI uploading/downloading

Browse Source
This commit is contained in:
Timothy Baldridge 2020-03-31 16:05:36 -06:00
parent a6e3ef5f72
commit e39f483b81
21 changed files with 377 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Compression.BSA.Test/BSATests.cs
View File

@ -13,12 +13,13 @@ using Xunit.Abstractions;
namespace Compression.BSA.Test
{
public class BSATests
public class BSATests : IAsyncLifetime
{
private static AbsolutePath _stagingFolder = ((RelativePath)"NexusDownloads").RelativeToEntryPoint();
private static AbsolutePath _bsaFolder = ((RelativePath)"BSAs").RelativeToEntryPoint();
private static AbsolutePath _testDir = ((RelativePath)"BSA Test Dir").RelativeToEntryPoint();
private static AbsolutePath _tempDir = ((RelativePath)"BSA Temp Dir").RelativeToEntryPoint();
private IDisposable _unsub;
public ITestOutputHelper TestContext { get; }
@ -27,13 +28,26 @@ namespace Compression.BSA.Test
public BSATests(ITestOutputHelper helper)
{
TestContext = helper;
}
public async Task InitializeAsync()
{
Queue = new WorkQueue();
Utils.LogMessages.Subscribe(f => TestContext.WriteLine(f.ShortDescription));
_unsub = Utils.LogMessages.Subscribe(f => TestContext.WriteLine(f.ShortDescription));
_stagingFolder.CreateDirectory();
_bsaFolder.DeleteDirectory();
await _bsaFolder.DeleteDirectory();
_bsaFolder.CreateDirectory();
}
public async Task DisposeAsync()
{
await _bsaFolder.DeleteDirectory();
Queue.Dispose();
_unsub.Dispose();
}
private static async Task<AbsolutePath> DownloadMod(Game game, int mod)
{
using var client = await NexusApiClient.Get();
@ -142,5 +156,6 @@ namespace Compression.BSA.Test
{
return i.ToJSON().FromJSONString<T>();
}
}
}

24
Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs
View File

@ -33,6 +33,8 @@ namespace Wabbajack.BuildServer.Test
$"WabbajackSettings:SQLConnection={PublicConnStr}",
$"WabbajackSettings:BunnyCDN_User=TEST",
$"WabbajackSettings:BunnyCDN_Password=TEST",
"WabbajackSettings:JobScheduler=false",
"WabbajackSettings:JobRunner=false"
}, true);
_host = builder.Build();
_token = new CancellationTokenSource();
@ -40,11 +42,25 @@ namespace Wabbajack.BuildServer.Test
Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080");
}
public T GetService<T>()
{
return (T)_host.Services.GetService(typeof(T));
}
public void Dispose()
{
if (!_token.IsCancellationRequested)
_token.Cancel();
_task.Wait();
try
{
_task.Wait();
}
catch (Exception)
{
//
}
_severTempFolder.DisposeAsync().AsTask().Wait();
}
}
@ -69,8 +85,11 @@ namespace Wabbajack.BuildServer.Test
_authedClient.Headers.Add(("x-api-key", fixture.APIKey));
_queue = new WorkQueue();
Fixture = fixture;
Queue = new WorkQueue();
}
public WorkQueue Queue { get; set; }
public BuildServerFixture Fixture { get; set; }
protected string MakeURL(string path)
@ -80,10 +99,11 @@ namespace Wabbajack.BuildServer.Test
public override void Dispose()
{
Queue.Dispose();
base.Dispose();
_unsubMsgs.Dispose();
_unsubErr.Dispose();
}
}
}

87
Wabbajack.BuildServer.Test/IndexedFilesTests.cs Normal file
View File

@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.FileUploader;
using Xunit;
using Xunit.Abstractions;
using Xunit.Priority;
namespace Wabbajack.BuildServer.Test
{
public class IndexedFilesTests : ABuildServerSystemTest
{
public IndexedFilesTests(ITestOutputHelper output, BuildServerFixture fixture) : base(output, fixture)
{
}
[Fact, Priority(1)]
public async Task CanIngestExportedInis()
{
var to = Fixture.ServerTempFolder.Combine("IniIngest");
await @"sql\DownloadStates".RelativeTo(AbsolutePath.EntryPoint).CopyDirectoryToAsync(to);
var result = await _authedClient.GetStringAsync(MakeURL("indexed_files/ingest/IniIngest"));
Assert.Equal("5", result);
}
[Fact, Priority(2)]
public async Task CanQueryViaHash()
{
var hashes = new HashSet<Hash>
{
Hash.FromHex("097ad17ef4b9f5b7"),
Hash.FromHex("96fb53c3dc6397d2"),
Hash.FromHex("97a6d27b7becba19")
};
foreach (var hash in hashes)
{
Utils.Log($"Testing Archive {hash}");
var ini = await ClientAPI.GetModIni(hash);
Assert.NotNull(ini);
Assert.NotNull(DownloadDispatcher.ResolveArchive(ini.LoadIniString()));
}
}
[Fact]
public async Task CanNotifyOfInis()
{
var files = await @"sql\NotifyStates".RelativeTo(AbsolutePath.EntryPoint)
.EnumerateFiles()
.Where(f => f.Extension == Consts.IniExtension)
.PMap(Queue, async ini => (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(ini.LoadIniFile())));
var archives = files.Select(f =>
new Archive
{
State = f,
Name = Guid.NewGuid().ToString()
});
Assert.True(await AuthorAPI.UploadPackagedInis(archives));
var SQL = Fixture.GetService<SqlService>();
var job = await SQL.GetJob();
Assert.IsType<IndexJob>(job.Payload);
var payload = (IndexJob)job.Payload;
Assert.IsType<NexusDownloader.State>(payload.Archive.State);
var casted = (NexusDownloader.State)payload.Archive.State;
Assert.Equal(Game.Skyrim, casted.Game);
// Insert the record into SQL
await SQL.AddDownloadState(Hash.FromHex("00e8bbbf591f61a3"), casted);
// Enqueue the same file again
Assert.True(await AuthorAPI.UploadPackagedInis(archives));
// File is aleady indexed so nothing gets enqueued
Assert.Null(await SQL.GetJob());
}
}
}

23
Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj
View File

@ -28,6 +28,29 @@
<None Update="sql\uploaded_files_ingest.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="sql\DownloadStates\097ad17ef4b9f5b7_68d29ad947f2bf80d887407b6e8794c37ac08f3728eca95c8774184c56df3800.ini">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="sql\DownloadStates\96fb53c3dc6397d2_9ff1b17c4fafdb70ef51390a1706d8aec66cdc09ca950f8a9daa1570db9b1c94.ini">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="sql\DownloadStates\97a6d27b7becba19_6ba040ef3bc1775bb41f97427fb830a907b9b74ccbe056624c537c8e5f214529.ini">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="sql\DownloadStates\e5223a83ab49e25c_1be0991cec07ee378b0891ce576cb75b3a7adc56232945772961e3a9428f17e5.ini">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="sql\DownloadStates\e5409bdeb0e77bd3_985c554f1bf98c1569fcbb2926f38e61c86e4ce6a416e6cb6cf020913f24d802.ini">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="sql\NotifyStates\00e8bbbf591f61a3_6a5eb07c4b3c03fde38c9223a94a38c9076ef8fc8167f77c875c58db8f2aefd2.ini">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<Folder Include="sql\DownloadStates" />
<Folder Include="sql\NotifyStates" />
</ItemGroup>
</Project>

4
Wabbajack.BuildServer.Test/sql/DownloadStates/097ad17ef4b9f5b7_68d29ad947f2bf80d887407b6e8794c37ac08f3728eca95c8774184c56df3800.ini Normal file
View File

@ -0,0 +1,4 @@
[General]
gameName=Skyrim
modID=58118
fileID=1000126774

4
Wabbajack.BuildServer.Test/sql/DownloadStates/96fb53c3dc6397d2_9ff1b17c4fafdb70ef51390a1706d8aec66cdc09ca950f8a9daa1570db9b1c94.ini Normal file
View File

@ -0,0 +1,4 @@
[General]
gameName=fallout4
modID=34297
fileID=141870

4
Wabbajack.BuildServer.Test/sql/DownloadStates/97a6d27b7becba19_6ba040ef3bc1775bb41f97427fb830a907b9b74ccbe056624c537c8e5f214529.ini Normal file
View File

@ -0,0 +1,4 @@
[General]
gameName=SkyrimSE
modID=23774
fileID=98580

4
Wabbajack.BuildServer.Test/sql/DownloadStates/e5223a83ab49e25c_1be0991cec07ee378b0891ce576cb75b3a7adc56232945772961e3a9428f17e5.ini Normal file
View File

@ -0,0 +1,4 @@
[General]
gameName=skyrimspecialedition
modID=13675
fileID=121575

4
Wabbajack.BuildServer.Test/sql/DownloadStates/e5409bdeb0e77bd3_985c554f1bf98c1569fcbb2926f38e61c86e4ce6a416e6cb6cf020913f24d802.ini Normal file
View File

@ -0,0 +1,4 @@
[General]
gameName=fallout4
modID=33578
fileID=137486

4
Wabbajack.BuildServer.Test/sql/NotifyStates/00e8bbbf591f61a3_6a5eb07c4b3c03fde38c9223a94a38c9076ef8fc8167f77c875c58db8f2aefd2.ini Normal file
View File

@ -0,0 +1,4 @@
[General]
gameName=skyrim
modID=81066
fileID=1000284635

23
Wabbajack.BuildServer.Test/sql/wabbajack_db.sql
View File

@ -321,6 +321,29 @@ CREATE UNIQUE NONCLUSTERED INDEX [ByAPIKey] ON [dbo].[ApiKeys]
INCLUDE([Owner]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
GO
/****** Object: Table [dbo].[DownloadStates] Script Date: 3/31/2020 6:22:47 AM ******/
CREATE TABLE [dbo].[DownloadStates](
[Id] [binary](32) NOT NULL,
[Hash] [bigint] NOT NULL,
[PrimaryKey] [varchar](max) NOT NULL,
[IniState] [varchar](max) NOT NULL,
[JsonState] [varchar](max) NOT NULL,
CONSTRAINT [PK_DownloadStates] PRIMARY KEY CLUSTERED
(
[Id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
CREATE NONCLUSTERED INDEX [ByHash] ON [dbo].[DownloadStates]
(
[Hash] ASC
)
INCLUDE([IniState]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
GO
/****** Object: Index [IX_Child] Script Date: 3/28/2020 4:58:59 PM ******/
CREATE NONCLUSTERED INDEX [IX_Child] ON [dbo].[AllFilesInArchive]
(

50
Wabbajack.BuildServer/Controllers/IndexedFiles.cs
View File

@ -28,9 +28,11 @@ namespace Wabbajack.BuildServer.Controllers
public class IndexedFiles : AControllerBase<IndexedFiles>
{
private SqlService _sql;
private AppSettings _settings;
public IndexedFiles(ILogger<IndexedFiles> logger, DBContext db, SqlService sql) : base(logger, db, sql)
public IndexedFiles(ILogger<IndexedFiles> logger, DBContext db, SqlService sql, AppSettings settings) : base(logger, db, sql)
{
_settings = settings;
_sql = sql;
}
@ -39,16 +41,38 @@ namespace Wabbajack.BuildServer.Controllers
public async Task<IActionResult> GetFileMeta(string xxHashAsBase64)
{
var id = Hash.FromHex(xxHashAsBase64);
var state = await Db.DownloadStates.AsQueryable()
.Where(d => d.Hash == id && d.IsValid)
.OrderByDescending(d => d.LastValidationTime)
.Take(1)
.ToListAsync();
if (state.Count == 0)
var result = await SQL.GetIniForHash(id);
if (result == null)
return NotFound();
Response.ContentType = "text/plain";
return Ok(string.Join("\r\n", state.FirstOrDefault().State.GetMetaIni()));
return Ok(result);
}
[HttpGet]
[Route("ingest/{folder}")]
[Authorize]
public async Task<IActionResult> Ingest(string folder)
{
var fullPath = folder.RelativeTo((AbsolutePath)_settings.TempFolder);
Utils.Log($"Ingesting Inis from {fullPath}");
int loadCount = 0;
foreach (var file in fullPath.EnumerateFiles().Where(f => f.Extension == Consts.IniExtension))
{
var loaded = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile()));
if (loaded == null)
{
Utils.Log($"Unsupported Ini {file}");
continue;
}
var hash = Hash.FromHex(((string)file.FileNameWithoutExtension).Split("_").First());
await SQL.AddDownloadState(hash, loaded);
loadCount += 1;
}
return Ok(loadCount);
}
[HttpPost]
@ -74,15 +98,13 @@ namespace Wabbajack.BuildServer.Controllers
if (data is ManualDownloader.State)
continue;
var key = data.PrimaryKeyString;
var found = await Db.DownloadStates.AsQueryable().Where(f => f.Key == key).Take(1).ToListAsync();
if (found.Count > 0)
if (await SQL.HaveIndexedArchivePrimaryKey(data.PrimaryKeyString))
continue;
await Db.Jobs.InsertOneAsync(new Job
await SQL.EnqueueJob(new Job
{
Priority = Job.JobPriority.Low,
Payload = new IndexJob()
Payload = new IndexJob
{
Archive = new Archive
{

40
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -7,10 +7,13 @@ using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Microsoft.Extensions.Configuration;
using Microsoft.VisualBasic;
using ReactiveUI;
using Wabbajack.BuildServer.Model.Models.Results;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.BuildServer.Model.Models
@ -29,7 +32,6 @@ namespace Wabbajack.BuildServer.Model.Models
private async Task<SqlConnection> Open()
{
var conn = new SqlConnection(_settings.SqlConnection);
Utils.Log("CONN : " + _settings.SqlConnection);
await conn.OpenAsync();
return conn;
}
@ -298,5 +300,41 @@ namespace Wabbajack.BuildServer.Model.Models
return await conn.QueryAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles WHERE UploadedBy = @uploadedBy",
new {UploadedBy = user});
}
public async Task AddDownloadState(Hash hash, AbstractDownloadState state)
{
await using var conn = await Open();
await conn.ExecuteAsync("INSERT INTO dbo.DownloadStates (Id, Hash, PrimaryKey, IniState, JsonState) " +
"VALUES (@Id, @Hash, @PrimaryKey, @IniState, @JsonState)",
new
{
Id = state.PrimaryKeyString.StringSha256Hex().FromHex(),
Hash = hash,
PrimaryKey = state.PrimaryKeyString,
IniState = string.Join("\n", state.GetMetaIni()),
JsonState = state.ToJSON()
});
}
public async Task<string> GetIniForHash(Hash id)
{
await using var conn = await Open();
var results = await conn.QueryAsync<string>("SELECT IniState FROM dbo.DownloadStates WHERE Hash = @Hash",
new {
Hash = id
});
return results.FirstOrDefault();
}
public async Task<bool> HaveIndexedArchivePrimaryKey(string key)
{
await using var conn = await Open();
var results = await conn.QueryAsync<string>(
"SELECT * FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey",
new {PrimaryKey = key});
return results.Any();
}
}
}

1
Wabbajack.Common/Consts.cs
View File

@ -97,6 +97,7 @@ namespace Wabbajack.Common
}
public static RelativePath MetaIni = new RelativePath("meta.ini");
public static Extension IniExtension = new Extension(".ini");
public static Extension HashFileExtension = new Extension(".xxHash");
public static Extension MetaFileExtension = new Extension(".meta");

10
Wabbajack.Common/Paths.cs
View File

@ -389,6 +389,16 @@ namespace Wabbajack.Common
{
return File.Open(_path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite);
}
public async Task CopyDirectoryToAsync(AbsolutePath destination)
{
destination.CreateDirectory();
foreach (var file in EnumerateFiles())
{
var dest = file.RelativeTo(this).RelativeTo(destination);
await file.CopyToAsync(dest);
}
}
}
public struct RelativePath : IPath, IEquatable<RelativePath>, IComparable<RelativePath>

20
Wabbajack.Lib/ClientAPI.cs
View File

@ -1,5 +1,6 @@
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.Exceptions;
namespace Wabbajack.Lib
{
@ -19,5 +20,24 @@ namespace Wabbajack.Lib
.GetAsync($"https://{Consts.WabbajackCacheHostname}/alternative/{hash.ToHex()}");
return !response.IsSuccessStatusCode ? null : (await response.Content.ReadAsStringAsync()).FromJSONString<Archive>();
}
/// <summary>
/// Given an archive hash, search the Wabbajack server for a matching .ini file
/// </summary>
/// <param name="hash"></param>
/// <returns></returns>
public static async Task<string> GetModIni(Hash hash)
{
var client = new Common.Http.Client();
try
{
return await client.GetStringAsync(
$"{Consts.WabbajackBuildServerUri}indexed_files/{hash.ToHex()}/meta.ini");
}
catch (HttpException)
{
return null;
}
}
}
}

9
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -137,7 +137,7 @@ namespace Wabbajack.Lib.FileUploader
return await RunJob("UpdateModLists");
}
public static async Task UploadPackagedInis(WorkQueue queue, IEnumerable<Archive> archives)
public static async Task<bool> UploadPackagedInis(IEnumerable<Archive> archives)
{
archives = archives.ToArray(); // defensive copy
Utils.Log($"Packaging {archives.Count()} inis");
@ -155,13 +155,14 @@ namespace Wabbajack.Lib.FileUploader
}
}
var webClient = new WebClient();
await webClient.UploadDataTaskAsync($"https://{Consts.WabbajackCacheHostname}/indexed_files/notify",
"POST", ms.ToArray());
var client = new Common.Http.Client();
await client.PostAsync($"{Consts.WabbajackBuildServerUri}indexed_files/notify", new ByteArrayContent(ms.ToArray()));
return true;
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
return false;
}
}

2
Wabbajack.Lib/MO2Compiler.cs
View File

@ -281,7 +281,7 @@ namespace Wabbajack.Lib
// Don't await this because we don't care if it fails.
Utils.Log("Finding States to package");
await AuthorAPI.UploadPackagedInis(Queue, SelectedArchives.ToArray());
await AuthorAPI.UploadPackagedInis(SelectedArchives.ToArray());
UpdateTracker.NextStep("Including Archive Metadata");
await IncludeArchiveMetadata();

4
Wabbajack/Converters/ConverterRegistration.cs
View File

@ -5,6 +5,7 @@ using System.Text;
using System.Threading.Tasks;
using ReactiveUI;
using Splat;
using Wabbajack.Converters;
namespace Wabbajack
{
@ -24,6 +25,9 @@ namespace Wabbajack
new PercentToDoubleConverter(),
typeof(IBindingTypeConverter)
);
Locator.CurrentMutable.RegisterConstant(
new PathToStringConverter(),
typeof(IBindingTypeConverter));
}
}
}

64
Wabbajack/Converters/PathToStringConverter.cs Normal file
View File

@ -0,0 +1,64 @@
using System;
using ReactiveUI;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.Converters
{
public class PathToStringConverter : IBindingTypeConverter
{
public int GetAffinityForObjects(Type fromType, Type toType)
{
if (toType == typeof(object)) return 1;
if (toType == typeof(string)) return 1;
if (toType == typeof(AbsolutePath)) return 1;
if (toType == typeof(AbsolutePath?)) return 1;
return 0;
}
public bool TryConvert(object @from, Type toType, object conversionHint, out object result)
{
if (toType == typeof(AbsolutePath))
{
if (@from is string s)
{
try
{
result = (AbsolutePath)s;
return true;
}
catch
{
result = (AbsolutePath)"";
return false;
}
}
if (@from is AbsolutePath abs)
{
result = abs;
return true;
}
}
else if (toType == typeof(string))
{
if (@from is string s)
{
result = default;
return false;
}
if (@from is AbsolutePath abs)
{
result = (string)abs;
return true;
}
}
result = default;
return false;
}
}
}

1
Wabbajack/Views/Compilers/VortexCompilerConfigView.xaml
View File

@ -5,7 +5,6 @@
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:local="clr-namespace:Wabbajack"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
d:DataContext="{d:DesignInstance local:VortexCompilerVM}"
d:DesignHeight="450"
d:DesignWidth="800"
mc:Ignorable="d">