Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

CDN FTP uploads, fixed meta download issues with 3dNPCs, Uploads to the build server are now multi-threaded.

Browse Source
This commit is contained in:
Timothy Baldridge 2020-01-21 20:43:53 -07:00
parent 7e436818b7
commit 8f92862b74
7 changed files with 132 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Wabbajack.BuildServer/AppSettings.cs
View File

@ -17,7 +17,7 @@ namespace Wabbajack.BuildServer
public bool RunFrontEndJobs { get; set; }
public bool RunBackEndJobs { get; set; }
public string BunnyCDNZone { get; set; }
public string BunnyCDNApiKey { get; set; }
public string BunnyCDN_User { get; set; }
public string BunnyCDN_Password { get; set; }
}
}

20
Wabbajack.BuildServer/Controllers/UploadedFiles.cs
View File

@ -52,7 +52,7 @@ namespace Wabbajack.BuildServer.Controllers
if (!Key.All(a => HexChars.Contains(a)))
return BadRequest("NOT A VALID FILENAME");
Utils.Log($"Writing at position {Offset} in ingest file {Key}");
await using (var file = System.IO.File.Open(Path.Combine("public", "files", Key), FileMode.Open, FileAccess.Write))
await using (var file = System.IO.File.Open(Path.Combine("public", "files", Key), FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
{
file.Position = Offset;
await Request.Body.CopyToAsync(file);
@ -82,25 +82,15 @@ namespace Wabbajack.BuildServer.Controllers
Hash = hash,
Name = original_name,
Uploader = user,
Size = new FileInfo(final_path).Length
Size = new FileInfo(final_path).Length,
CDNName = "wabbajackpush"
};
await Db.UploadedFiles.InsertOneAsync(record);
await Db.Jobs.InsertOneAsync(new Job
{
Payload = new IndexJob
{
Archive = new Archive
{
Name = record.MungedName,
Size = record.Size,
Hash = record.Hash,
State = new HTTPDownloader.State
{
Url = record.Uri
}
}
}
Priority = Job.JobPriority.High, Payload = new UploadToCDN {FileId = record.Id}
});
return Ok(record.Uri);
}

58
Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs
View File

@ -1,10 +1,17 @@
using System.Threading.Tasks;
using System;
using System.Net;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using BunnyCDN.Net.Storage;
using CG.Web.MegaApiClient;
using FluentFTP;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using File = System.IO.File;
namespace Wabbajack.BuildServer.Models.Jobs
{
@ -17,10 +24,53 @@ namespace Wabbajack.BuildServer.Models.Jobs
public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
{
var file = await db.UploadedFiles.AsQueryable().Where(f => f.Id == FileId).FirstOrDefaultAsync();
var cdn = new BunnyCDNStorage(settings.BunnyCDNZone, settings.BunnyCDNApiKey);
Utils.Log($"CDN Push {file.MungedName} to {settings.BunnyCDNZone}");
await cdn.UploadAsync(Path.Combine("public", "files", file.MungedName), $"{settings.BunnyCDNZone}/{file.MungedName}");
using (var client = new FtpClient("storage.bunnycdn.com"))
{
client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);
await client.ConnectAsync();
using (var stream = File.OpenRead(Path.Combine("public", "files", file.MungedName)))
{
await client.UploadAsync(stream, file.MungedName, progress: new Progress(file.MungedName));
}
/*
await db.Jobs.InsertOneAsync(new Job
{
Payload = new IndexJob
{
Archive = new Archive
{
Name = file.MungedName,
Size = file.Size,
Hash = file.Hash,
State = new HTTPDownloader.State
{
Url = file.Uri
}
}
}
});*/
}
return JobResult.Success();
}
private class Progress : IProgress<FluentFTP.FtpProgress>
{
private string _name;
private DateTime LastUpdate = DateTime.UnixEpoch;
public Progress(string name)
{
_name = name;
}
public void Report(FtpProgress value)
{
if (DateTime.Now - LastUpdate <= TimeSpan.FromSeconds(5)) return;
Utils.Log($"Uploading {_name} - {value.Progress}% {(int)((value.TransferSpeed + 1) / 1024 / 1024)} MB/sec ETA: {value.ETA}");
LastUpdate = DateTime.Now;
}
}
}
}

1
Wabbajack.BuildServer/Wabbajack.BuildServer.csproj
View File

@ -11,6 +11,7 @@
<ItemGroup>
<PackageReference Include="BunnyCDN.Net.Storage" Version="1.0.2" />
<PackageReference Include="FluentFTP" Version="29.0.3" />
<PackageReference Include="graphiql" Version="1.2.0" />
<PackageReference Include="GraphQL" Version="3.0.0-preview-1352" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />

4
Wabbajack.BuildServer/appsettings.json
View File

@ -35,8 +35,8 @@
"MinimalMode": true,
"RunFrontEndJobs": true,
"RunBackEndJobs": true,
"BunnyCDNZone": "",
"BunnyCDNApiKey": ""
"BunnyCDN_User": "wabbajackcdn",
"BunnyCDN_Password": "XXXX"
},
"AllowedHosts": "*"
}

107
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -1,9 +1,12 @@
using System;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reactive.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Lib.FileUploader
{
@ -29,57 +32,61 @@ namespace Wabbajack.Lib.FileUploader
var tcs = new TaskCompletionSource<string>();
queue.QueueTask(async () =>
{
using (var stream =
new StatusFileStream(File.OpenRead(filename), $"Uploading {Path.GetFileName(filename)}", queue))
var client = new HttpClient();
var fsize = new FileInfo(filename).Length;
client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
var response = await client.PutAsync(UploadURL+$"/{Path.GetFileName(filename)}/start", new StringContent(""));
if (!response.IsSuccessStatusCode)
{
var client = new HttpClient();
client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
var response = await client.PutAsync(UploadURL+$"/{Path.GetFileName(filename)}/start", new StringContent(""));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var key = await response.Content.ReadAsStringAsync();
var data = new byte[BLOCK_SIZE];
while (stream.Position < stream.Length)
{
var old_offset = stream.Position;
var new_size = Math.Min(stream.Length - stream.Position, BLOCK_SIZE);
if (new_size != data.Length)
data = new byte[new_size];
await stream.ReadAsync(data, 0, data.Length);
response = await client.PutAsync(UploadURL + $"/{key}/data/{old_offset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != old_offset + data.Length)
{
tcs.SetResult("Sync Error");
return;
}
}
response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent(""));
if (response.IsSuccessStatusCode)
tcs.SetResult(await response.Content.ReadAsStringAsync());
else
tcs.SetResult("FAILED");
tcs.SetResult("FAILED");
return;
}
var key = await response.Content.ReadAsStringAsync();
using (var iqueue = new WorkQueue(8))
{
await Enumerable.Range(0, (int)(fsize / BLOCK_SIZE))
.PMap(iqueue, async block_idx =>
{
var block_offset = block_idx * BLOCK_SIZE;
var block_size = block_offset + BLOCK_SIZE > fsize
? fsize - block_offset
: BLOCK_SIZE;
using (var fs = File.OpenRead(filename))
{
fs.Position = block_offset;
var data = new byte[block_size];
await fs.ReadAsync(data, 0, data.Length);
response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != block_offset + data.Length)
{
tcs.SetResult("Sync Error");
return;
}
}
});
}
response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent(""));
if (response.IsSuccessStatusCode)
tcs.SetResult(await response.Content.ReadAsStringAsync());
else
tcs.SetResult("FAILED");
});
return tcs.Task;
}

13
Wabbajack.Lib/MO2Compiler.cs
View File

@ -11,6 +11,7 @@ using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
@ -338,9 +339,17 @@ namespace Wabbajack.Lib
private async Task InferMetas()
{
var to_find = Directory.EnumerateFiles(MO2DownloadsFolder)
async Task<bool> HasInvalidMeta(string filename)
{
string metaname = filename + ".meta";
if (!File.Exists(metaname)) return true;
return (AbstractDownloadState) await DownloadDispatcher.ResolveArchive(metaname.LoadIniFile()) == null;
}
var to_find = (await Directory.EnumerateFiles(MO2DownloadsFolder)
.Where(f => !f.EndsWith(".meta") && !f.EndsWith(Consts.HashFileExtension))
.Where(f => !File.Exists(f + ".meta"))
.PMap(Queue, async f => await HasInvalidMeta(f) ? f : null))
.Where(f => f != null)
.ToList();
if (to_find.Count == 0) return;