Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #831 from wabbajack-tools/downloading-and-hashing-engine

Browse Source 
Downloading and hashing engine
This commit is contained in:
Timothy Baldridge 2020-05-12 20:58:39 -07:00 committed by GitHub
commit d107b1fc34
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 437 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs
View File

@ -1,6 +1,7 @@
using System;
using System.IO;
using System.IO.Compression;
using System.IO.MemoryMappedFiles;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
@ -58,15 +59,18 @@ namespace Wabbajack.Lib.Downloaders
{
destination.Parent.CreateDirectory();
var definition = await GetDefinition();
await using var fs = destination.Create();
using var fs = destination.Create();
using var mmfile = MemoryMappedFile.CreateFromFile(fs, null, definition.Size, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, false);
var client = new Common.Http.Client();
await definition.Parts.DoProgress($"Downloading {a.Name}", async part =>
using var queue = new WorkQueue();
await definition.Parts.PMap(queue, async part =>
{
fs.Position = part.Offset;
Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(definition.Parts.Length - part.Index, definition.Parts.Length));
await using var ostream = mmfile.CreateViewStream(part.Offset, part.Size);
using var response = await client.GetAsync($"{Url}/parts/{part.Index}");
if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase);
await response.Content.CopyToAsync(fs);
await response.Content.CopyToAsync(ostream);
});
return true;
}

5
Wabbajack.Server.Test/ABuildServerSystemTest.cs
View File

@ -32,6 +32,9 @@ namespace Wabbajack.BuildServer.Test
public BuildServerFixture()
{
ServerArchivesFolder.DeleteDirectory();
ServerArchivesFolder.CreateDirectory();
var builder = Program.CreateHostBuilder(
new[]
{
@ -54,6 +57,7 @@ namespace Wabbajack.BuildServer.Test
"ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllText(
"GoogleIDs:\nAllowedPrefixes:\n - http://localhost");
}
~BuildServerFixture()
@ -149,6 +153,7 @@ namespace Wabbajack.BuildServer.Test
Consts.ModlistSummaryURL = MakeURL("lists/status.json");
Consts.ServerWhitelistURL = MakeURL("ServerWhitelist.yaml");
}
public WorkQueue Queue { get; set; }

49
Wabbajack.Server.Test/ArchiveMaintainerTests.cs Normal file
View File

@ -0,0 +1,49 @@
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class ArchiveMaintainerTests : ABuildServerSystemTest
{
public ArchiveMaintainerTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanIngestFiles()
{
var maintainer = Fixture.GetService<ArchiveMaintainer>();
using var tf = new TempFile();
using var tf2 = new TempFile();
await tf.Path.WriteAllBytesAsync(RandomData(1024));
await tf.Path.CopyToAsync(tf2.Path);
var hash = await tf.Path.FileHashAsync();
await maintainer.Ingest(tf.Path);
Assert.True(maintainer.TryGetPath(hash, out var found));
Assert.Equal(await tf2.Path.ReadAllBytesAsync(), await found.ReadAllBytesAsync());
}
[Fact]
public async Task IngestsExistingFiles()
{
var maintainer = Fixture.GetService<ArchiveMaintainer>();
using var tf = new TempFile();
await tf.Path.WriteAllBytesAsync(RandomData(1024));
var hash = await tf.Path.FileHashAsync();
await tf.Path.CopyToAsync(Fixture.ServerArchivesFolder.Combine(hash.ToHex()));
maintainer.Start();
Assert.True(maintainer.TryGetPath(hash, out var found));
}
}
}

108
Wabbajack.Server.Test/ModListValidationTests.cs Normal file
View File

@ -0,0 +1,108 @@
using System;
using System.Collections.Generic;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class ModListValidationTests : ABuildServerSystemTest
{
public ModListValidationTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanLoadMetadataFromTestServer()
{
var modlist = await MakeModList();
Consts.ModlistMetadataURL = modlist.ToString();
var data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
Assert.Equal("test_list", data.First().Links.MachineURL);
}
[Fact]
public async Task CanIngestModLists()
{
var modlist = await MakeModList();
Consts.ModlistMetadataURL = modlist.ToString();
var sql = Fixture.GetService<SqlService>();
var downloader = Fixture.GetService<ModListDownloader>();
await downloader.CheckForNewLists();
foreach (var list in ModListMetaData)
{
Assert.True(await sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash));
}
}
private async Task<Uri> MakeModList()
{
var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!");
var test_archive_path = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
await test_archive_path.WriteAllBytesAsync(archive_data);
ModListData = new ModList();
ModListData.Archives.Add(
new Archive(new HTTPDownloader.State(MakeURL("test_archive.txt")))
{
Hash = await test_archive_path.FileHashAsync(),
Name = "test_archive",
Size = test_archive_path.Size,
});
var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder);
await using (var fs = modListPath.Create())
{
using var za = new ZipArchive(fs, ZipArchiveMode.Create);
var entry = za.CreateEntry("modlist");
await using var es = entry.Open();
ModListData.ToJson(es);
}
ModListMetaData = new List<ModlistMetadata>
{
new ModlistMetadata
{
Official = false,
Author = "Test Suite",
Description = "A test",
DownloadMetadata = new DownloadMetadata
{
Hash = await modListPath.FileHashAsync(),
Size = modListPath.Size
},
Links = new ModlistMetadata.LinksObject
{
MachineURL = "test_list",
Download = MakeURL("test_modlist.wabbajack")
}
}
};
var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder);
ModListMetaData.ToJson(metadataPath);
return new Uri(MakeURL("test_mod_list_metadata.json"));
}
public ModList ModListData { get; set; }
public List<ModlistMetadata> ModListMetaData { get; set; }
}
}

62
Wabbajack.Server/DataLayer/ModLists.cs Normal file
View File

@ -0,0 +1,62 @@
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Lib;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Common;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist)
{
await using var conn = await Open();
await using var tran = await conn.BeginTransactionAsync();
await conn.ExecuteAsync(@"DELETE FROM dbo.ModLists Where MachineUrl = @MachineUrl",
new {MachineUrl = metadata.Links.MachineURL}, tran);
await conn.ExecuteAsync(
@"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList) VALUES (@MachineUrl, @Hash, @Metadata, @ModList)",
new
{
MachineUrl = metadata.Links.MachineURL,
Hash = hash,
MetaData = metadata.ToJson(),
ModList = modlist.ToJson()
}, tran);
var entries = modlist.Archives.Select(a =>
new
{
MachineUrl = metadata.Links.MachineURL,
Hash = a.Hash,
Size = a.Size,
State = a.State.ToJson(),
PrimaryKeyString = a.State.PrimaryKeyString
}).ToArray();
await conn.ExecuteAsync(@"DELETE FROM dbo.ModListArchives WHERE MachineURL = @machineURL",
new {MachineUrl = metadata.Links.MachineURL}, tran);
foreach (var entry in entries)
{
await conn.ExecuteAsync(
"INSERT INTO dbo.ModListArchives (MachineURL, Hash, Size, PrimaryKeyString, State) VALUES (@MachineURL, @Hash, @Size, @PrimaryKeyString, @State)",
entry, tran);
}
await tran.CommitAsync();
}
public async Task<bool> HaveIndexedModlist(string machineUrl, Hash hash)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT MachineURL from dbo.Modlists WHERE MachineURL = @MachineUrl AND Hash = @Hash",
new {MachineUrl = machineUrl, Hash = hash});
return result != null;
}
}
}

81
Wabbajack.Server/Services/ArchiveMaintainer.cs Normal file
View File

@ -0,0 +1,81 @@
using System;
using System.Collections.Concurrent;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using File = System.IO.File;
namespace Wabbajack.Server.Services
{
/// <summary>
/// Maintains a concurrent cache of all the files we've downloaded, indexed by Hash
/// </summary>
public class ArchiveMaintainer
{
private AppSettings _settings;
private ILogger<ArchiveMaintainer> _logger;
private ConcurrentDictionary<Hash, AbsolutePath> _archives = new ConcurrentDictionary<Hash, AbsolutePath>();
public ArchiveMaintainer(ILogger<ArchiveMaintainer> logger, AppSettings settings)
{
_settings = settings;
_logger = logger;
}
public void Start()
{
foreach (var path in _settings.ArchivePath.EnumerateFiles(false))
{
try
{
var hash = Hash.FromHex((string)path.FileNameWithoutExtension);
_archives[hash] = path;
}
catch (Exception ex)
{
_logger.Log(LogLevel.Error, ex.ToString());
}
}
_logger.Log(LogLevel.Information, $"Found {_archives.Count} archives");
}
public async Task<AbsolutePath> Ingest(AbsolutePath file)
{
var hash = await file.FileHashAsync();
if (HaveArchive(hash))
{
file.Delete();
return _archives[hash];
}
var newPath = _settings.ArchivePath.Combine(hash.ToHex());
await file.MoveToAsync(newPath);
_archives[hash] = newPath;
return _archives[hash];
}
public bool HaveArchive(Hash hash)
{
return _archives.ContainsKey(hash);
}
public bool TryGetPath(Hash hash, out AbsolutePath path)
{
return _archives.TryGetValue(hash, out path);
}
}
public static class ArchiveMaintainerExtensions
{
public static void UseArchiveMaintainer(this IApplicationBuilder b)
{
var poll = (ArchiveMaintainer)b.ApplicationServices.GetService(typeof(ArchiveMaintainer));
poll.Start();
}
}
}

119
Wabbajack.Server/Services/ModListDownloader.cs Normal file
View File

@ -0,0 +1,119 @@
using System;
using System.IO.Compression;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.Services
{
public class ModListDownloader
{
private ILogger<ModListDownloader> _logger;
private AppSettings _settings;
private ArchiveMaintainer _maintainer;
private SqlService _sql;
public ModListDownloader(ILogger<ModListDownloader> logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql)
{
_logger = logger;
_settings = settings;
_maintainer = maintainer;
_sql = sql;
}
public void Start()
{
if (_settings.RunBackEndJobs)
{
Task.Run(async () =>
{
while (true)
{
try
{
await CheckForNewLists();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error checking list");
}
await Task.Delay(TimeSpan.FromMinutes(5));
}
});
}
}
public async Task CheckForNewLists()
{
var lists = await ModlistMetadata.LoadFromGithub();
foreach (var list in lists)
{
try
{
if (_maintainer.HaveArchive(list.DownloadMetadata!.Hash))
continue;
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
var tf = new TempFile();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
if (state == null)
{
_logger.Log(LogLevel.Error,
$"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}");
continue;
}
await state.Download(new Archive(state) {Name = $"{list.Links.MachineURL}.wabbajack"}, tf.Path);
var modistPath = await _maintainer.Ingest(tf.Path);
ModList modlist;
await using (var fs = modistPath.OpenRead())
using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
await using (var entry = zip.GetEntry("modlist")?.Open())
{
if (entry == null)
{
Utils.Log($"Bad Modlist {list.Links.MachineURL}");
continue;
}
try
{
modlist = entry.FromJson<ModList>();
}
catch (JsonReaderException ex)
{
Utils.Log($"Bad JSON format for {list.Links.MachineURL}");
continue;
}
}
await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
}
}
}
}
public static class ModListDownloaderExtensions
{
public static void UseModListDownloader(this IApplicationBuilder b)
{
var poll = (ModListDownloader)b.ApplicationServices.GetService(typeof(ModListDownloader));
poll.Start();
}
}
}

1
Wabbajack.Server/Services/NexusPoll.cs
View File

@ -170,6 +170,5 @@ namespace Wabbajack.Server.Services
var poll = (NexusPoll)b.ApplicationServices.GetService(typeof(NexusPoll));
poll.Start();
}
}
}

6
Wabbajack.Server/Startup.cs
View File

@ -56,7 +56,9 @@ namespace Wabbajack.Server
services.AddSingleton<AppSettings>();
services.AddSingleton<SqlService>();
services.AddSingleton<GlobalInformation>();
services.AddSingleton<NexusPoll>();
services.AddSingleton<NexusPoll>();
services.AddSingleton<ArchiveMaintainer>();
services.AddSingleton<ModListDownloader>();
services.AddMvc();
services.AddControllers()
.AddNewtonsoftJson(o =>
@ -99,6 +101,8 @@ namespace Wabbajack.Server
app.UseAuthentication();
app.UseAuthorization();
app.UseNexusPoll();
app.UseArchiveMaintainer();
app.UseModListDownloader();
app.Use(next =>
{