Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #414 from wabbajack-tools/bunny-cdn-push

Browse Source 
Server changes for CDN optimization
This commit is contained in:
Timothy Baldridge 2020-01-22 05:40:13 -07:00 committed by GitHub
commit 4a76340cc5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 178 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -1,5 +1,6 @@
### Changelog
=======
* Fix for queue size recommendation of 0GB RAM on low-end machines
* Fix for website readme compilation
* Fix for compiler downloads folder specification (was always standard path)

3
Wabbajack.BuildServer/AppSettings.cs
View File

@ -16,5 +16,8 @@ namespace Wabbajack.BuildServer
public bool RunFrontEndJobs { get; set; }
public bool RunBackEndJobs { get; set; }
public string BunnyCDN_User { get; set; }
public string BunnyCDN_Password { get; set; }
}
}

23
Wabbajack.BuildServer/Controllers/UploadedFiles.cs
View File

@ -15,16 +15,22 @@ using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.BuildServer.Controllers
{
public class UploadedFiles : AControllerBase<UploadedFiles>
{
public UploadedFiles(ILogger<UploadedFiles> logger, DBContext db) : base(logger, db)
{
private AppSettings _settings;
public UploadedFiles(ILogger<UploadedFiles> logger, DBContext db, AppSettings settings) : base(logger, db)
{
_settings = settings;
}
[HttpPut]
@ -46,7 +52,7 @@ namespace Wabbajack.BuildServer.Controllers
if (!Key.All(a => HexChars.Contains(a)))
return BadRequest("NOT A VALID FILENAME");
Utils.Log($"Writing at position {Offset} in ingest file {Key}");
await using (var file = System.IO.File.Open(Path.Combine("public", "files", Key), FileMode.Open, FileAccess.Write))
await using (var file = System.IO.File.Open(Path.Combine("public", "files", Key), FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
{
file.Position = Offset;
await Request.Body.CopyToAsync(file);
@ -69,15 +75,23 @@ namespace Wabbajack.BuildServer.Controllers
System.IO.File.Move(Path.Combine("public", "files", Key), final_path);
var hash = await final_path.FileHashAsync();
var record = new UploadedFile
{
Id = parts[1],
Hash = hash,
Name = original_name,
Uploader = user,
Size = new FileInfo(final_path).Length
Size = new FileInfo(final_path).Length,
CDNName = "wabbajackpush"
};
await Db.UploadedFiles.InsertOneAsync(record);
await Db.Jobs.InsertOneAsync(new Job
{
Priority = Job.JobPriority.High, Payload = new UploadToCDN {FileId = record.Id}
});
return Ok(record.Uri);
}
@ -92,6 +106,7 @@ namespace Wabbajack.BuildServer.Controllers
</body></html>
");
[HttpGet]
[Route("uploaded_files")]
public async Task<ContentResult> UploadedFilesGet()

3
Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs
View File

@ -17,7 +17,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue
typeof(UpdateModLists),
typeof(EnqueueAllArchives),
typeof(EnqueueAllGameFiles),
typeof(EnqueueRecentFiles)
typeof(EnqueueRecentFiles),
typeof(UploadToCDN)
};
public static Dictionary<Type, string> TypeToName { get; set; }
public static Dictionary<string, Type> NameToType { get; set; }

76
Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs Normal file
View File

@ -0,0 +1,76 @@
using System;
using System.Net;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using BunnyCDN.Net.Storage;
using CG.Web.MegaApiClient;
using FluentFTP;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using File = System.IO.File;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class UploadToCDN : AJobPayload
{
public override string Description => $"Push an uploaded file ({FileId}) to the CDN";
public string FileId { get; set; }
public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
{
var file = await db.UploadedFiles.AsQueryable().Where(f => f.Id == FileId).FirstOrDefaultAsync();
using (var client = new FtpClient("storage.bunnycdn.com"))
{
client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);
await client.ConnectAsync();
using (var stream = File.OpenRead(Path.Combine("public", "files", file.MungedName)))
{
await client.UploadAsync(stream, file.MungedName, progress: new Progress(file.MungedName));
}
/*
await db.Jobs.InsertOneAsync(new Job
{
Payload = new IndexJob
{
Archive = new Archive
{
Name = file.MungedName,
Size = file.Size,
Hash = file.Hash,
State = new HTTPDownloader.State
{
Url = file.Uri
}
}
}
});*/
}
return JobResult.Success();
}
private class Progress : IProgress<FluentFTP.FtpProgress>
{
private string _name;
private DateTime LastUpdate = DateTime.UnixEpoch;
public Progress(string name)
{
_name = name;
}
public void Report(FtpProgress value)
{
if (DateTime.Now - LastUpdate <= TimeSpan.FromSeconds(5)) return;
Utils.Log($"Uploading {_name} - {value.Progress}% {(int)((value.TransferSpeed + 1) / 1024 / 1024)} MB/sec ETA: {value.ETA}");
LastUpdate = DateTime.Now;
}
}
}
}

5
Wabbajack.BuildServer/Models/UploadedFile.cs
View File

@ -17,9 +17,12 @@ namespace Wabbajack.BuildServer.Models
public string Uploader { get; set; }
public DateTime UploadDate { get; set; } = DateTime.UtcNow;
public string CDNName { get; set; }
[BsonIgnore]
public string MungedName => $"{Path.GetFileNameWithoutExtension(Name)}-{Id}{Path.GetExtension(Name)}";
[BsonIgnore] public object Uri => $"https://wabbajack.b-cdn.net/{MungedName}";
[BsonIgnore]
public string Uri => CDNName == null ? $"https://wabbajack.b-cdn.net/{MungedName}" : $"https://{CDNName}.b-cdn.net/{MungedName}";
}
}

2
Wabbajack.BuildServer/Wabbajack.BuildServer.csproj
View File

@ -10,6 +10,8 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="BunnyCDN.Net.Storage" Version="1.0.2" />
<PackageReference Include="FluentFTP" Version="29.0.3" />
<PackageReference Include="graphiql" Version="1.2.0" />
<PackageReference Include="GraphQL" Version="3.0.0-preview-1352" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />

4
Wabbajack.BuildServer/appsettings.json
View File

@ -34,7 +34,9 @@
"ArchiveDir": "c:\\archives",
"MinimalMode": true,
"RunFrontEndJobs": true,
"RunBackEndJobs": true
"RunBackEndJobs": true,
"BunnyCDN_User": "wabbajackcdn",
"BunnyCDN_Password": "XXXX"
},
"AllowedHosts": "*"
}

107
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -1,9 +1,12 @@
using System;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reactive.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Lib.FileUploader
{
@ -29,57 +32,61 @@ namespace Wabbajack.Lib.FileUploader
var tcs = new TaskCompletionSource<string>();
queue.QueueTask(async () =>
{
using (var stream =
new StatusFileStream(File.OpenRead(filename), $"Uploading {Path.GetFileName(filename)}", queue))
var client = new HttpClient();
var fsize = new FileInfo(filename).Length;
client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
var response = await client.PutAsync(UploadURL+$"/{Path.GetFileName(filename)}/start", new StringContent(""));
if (!response.IsSuccessStatusCode)
{
var client = new HttpClient();
client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
var response = await client.PutAsync(UploadURL+$"/{Path.GetFileName(filename)}/start", new StringContent(""));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var key = await response.Content.ReadAsStringAsync();
var data = new byte[BLOCK_SIZE];
while (stream.Position < stream.Length)
{
var old_offset = stream.Position;
var new_size = Math.Min(stream.Length - stream.Position, BLOCK_SIZE);
if (new_size != data.Length)
data = new byte[new_size];
await stream.ReadAsync(data, 0, data.Length);
response = await client.PutAsync(UploadURL + $"/{key}/data/{old_offset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != old_offset + data.Length)
{
tcs.SetResult("Sync Error");
return;
}
}
response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent(""));
if (response.IsSuccessStatusCode)
tcs.SetResult(await response.Content.ReadAsStringAsync());
else
tcs.SetResult("FAILED");
tcs.SetResult("FAILED");
return;
}
var key = await response.Content.ReadAsStringAsync();
using (var iqueue = new WorkQueue(8))
{
await Enumerable.Range(0, (int)(fsize / BLOCK_SIZE))
.PMap(iqueue, async block_idx =>
{
var block_offset = block_idx * BLOCK_SIZE;
var block_size = block_offset + BLOCK_SIZE > fsize
? fsize - block_offset
: BLOCK_SIZE;
using (var fs = File.OpenRead(filename))
{
fs.Position = block_offset;
var data = new byte[block_size];
await fs.ReadAsync(data, 0, data.Length);
response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != block_offset + data.Length)
{
tcs.SetResult("Sync Error");
return;
}
}
});
}
response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent(""));
if (response.IsSuccessStatusCode)
tcs.SetResult(await response.Content.ReadAsStringAsync());
else
tcs.SetResult("FAILED");
});
return tcs.Task;
}

13
Wabbajack.Lib/MO2Compiler.cs
View File

@ -11,6 +11,7 @@ using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
@ -323,9 +324,17 @@ namespace Wabbajack.Lib
private async Task InferMetas()
{
var to_find = Directory.EnumerateFiles(MO2DownloadsFolder)
async Task<bool> HasInvalidMeta(string filename)
{
string metaname = filename + Consts.MetaFileExtension;
if (!File.Exists(metaname)) return true;
return (AbstractDownloadState) await DownloadDispatcher.ResolveArchive(metaname.LoadIniFile()) == null;
}
var to_find = (await Directory.EnumerateFiles(MO2DownloadsFolder)
.Where(f => !f.EndsWith(Consts.MetaFileExtension) && !f.EndsWith(Consts.HashFileExtension))
.Where(f => !File.Exists(f + Consts.MetaFileExtension))
.PMap(Queue, async f => await HasInvalidMeta(f) ? f : null))
.Where(f => f != null)
.ToList();
if (to_find.Count == 0) return;