Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rework WJ caching, move cache server routines to MongoDB

Browse Source
This commit is contained in:
Timothy Baldridge 2020-01-01 09:19:06 -07:00
parent a7bf8f42ed
commit 717ad8c70a
31 changed files with 587 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
Wabbajack.CacheServer/DTOs/DownloadState.cs Normal file
View File

@ -0,0 +1,23 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.CacheServer.DTOs
{
public class DownloadState
{
[BsonId]
public string Key { get; set; }
public string Hash { get; set; }
public AbstractDownloadState State { get; set; }
public bool IsValid { get; set; }
public DateTime LastValidationTime { get; set; } = DateTime.Now;
public DateTime FirstValidationTime { get; set; } = DateTime.Now;
}
}

26
Wabbajack.CacheServer/DTOs/IndexedFile.cs Normal file
View File

@ -0,0 +1,26 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.RightsManagement;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.CacheServer.DTOs
{
public class IndexedFile
{
[BsonId]
public string Hash { get; set; }
public string SHA256 { get; set; }
public string SHA1 { get; set; }
public string MD5 { get; set; }
public string CRC { get; set; }
public string Name { get; set; }
public string Extension { get; set; }
public long Size { get; set; }
public bool IsArchive { get; set; }
public List<string> Children { get; set; } = new List<string>();
}
}

3
Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs
View File

@ -4,6 +4,7 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.CacheServer.Jobs;
namespace Wabbajack.CacheServer.DTOs.JobQueue
{
@ -19,7 +20,7 @@ namespace Wabbajack.CacheServer.DTOs.JobQueue
public virtual bool UsesNexus { get; } = false;
public abstract JobResult Execute();
public abstract Task<JobResult> Execute();
static AJobPayload()
{

20
Wabbajack.CacheServer/DTOs/JobQueue/IndexJob.cs
View File

@ -1,20 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.CacheServer.DTOs.JobQueue
{
public class IndexJob : AJobPayload
{
public AbstractDownloadState State { get; set; }
public override string Description { get; } = "Validate and index an archive";
public override bool UsesNexus { get => State is NexusDownloader.State; }
public override JobResult Execute()
{
throw new NotImplementedException();
}
}
}

16
Wabbajack.CacheServer/DTOs/JobQueue/Job.cs
View File

@ -5,6 +5,7 @@ using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
namespace Wabbajack.CacheServer.DTOs.JobQueue
{
@ -23,6 +24,8 @@ namespace Wabbajack.CacheServer.DTOs.JobQueue
public DateTime? Ended { get; set; }
public DateTime Created { get; set; } = DateTime.Now;
public JobPriority Priority { get; set; } = JobPriority.Normal;
public JobResult Result { get; set; }
public bool RequiresNexus { get; set; } = true;
public AJobPayload Payload { get; set; }
@ -36,19 +39,18 @@ namespace Wabbajack.CacheServer.DTOs.JobQueue
{
var filter = new BsonDocument
{
{"query", new BsonDocument {{"Started", null}}},
{"sort", new BsonDocument{{"Priority", -1}, {"Created", 1}}},
{"Started", BsonNull.Value}
};
var update = new BsonDocument
{
{"update", new BsonDocument {{"$set", new BsonDocument {{"Started", DateTime.Now}}}}}
{"$set", new BsonDocument {{"Started", DateTime.Now}}}
};
var job = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update);
var sort = new {Priority=-1, Created=1}.ToBsonDocument();
var job = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update, new FindOneAndUpdateOptions<Job>{Sort = sort});
return job;
}
public static async Task<Job> Finish(Job job)
public static async Task<Job> Finish(Job job, JobResult jobResult)
{
var filter = new BsonDocument
{
@ -56,7 +58,7 @@ namespace Wabbajack.CacheServer.DTOs.JobQueue
};
var update = new BsonDocument
{
{"update", new BsonDocument {{"$set", new BsonDocument {{"Ended", DateTime.Now}}}}}
{"$set", new BsonDocument {{"Ended", DateTime.Now}, {"Result", jobResult.ToBsonDocument()}}}
};
var result = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update);
return result;

24
Wabbajack.CacheServer/DTOs/NexusCacheData.cs Normal file
View File

@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
namespace Wabbajack.CacheServer.DTOs
{
public class NexusCacheData<T>
{
[BsonId]
public string Path { get; set; }
public T Data { get; set; }
public string Game { get; set; }
public string ModId { get; set; }
public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow;
[BsonIgnoreIfNull]
public string FileId { get; set; }
}
}

7
Wabbajack.CacheServer/Extensions.cs
View File

@ -1,12 +1,19 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
namespace Wabbajack.CacheServer
{
public static class Extensions
{
public static async Task<T> FindOneAsync<T>(this IMongoCollection<T> coll, Expression<Func<T, bool>> expr)
{
return (await coll.AsQueryable().Where(expr).Take(1).ToListAsync()).FirstOrDefault();
}
}
}

30
Wabbajack.CacheServer/JobQueueEndpoints.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Linq.Expressions;
using System.Security.Policy;
using System.Threading.Tasks;
using MongoDB.Driver;
@ -7,6 +8,7 @@ using MongoDB.Driver.Linq;
using Nancy;
using Nettle;
using Wabbajack.CacheServer.DTOs.JobQueue;
using Wabbajack.CacheServer.Jobs;
namespace Wabbajack.CacheServer
{
@ -55,10 +57,10 @@ namespace Wabbajack.CacheServer
var states = await Server.Config.ListValidation.Connect()
.AsQueryable()
.SelectMany(lst => lst.DetailedStatus.Archives)
.Select(a => a.Archive.State)
.Select(a => a.Archive)
.ToListAsync();
var jobs = states.Select(state => new IndexJob {State = state})
var jobs = states.Select(state => new IndexJob {Archive = state})
.Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus})
.ToList();
@ -67,5 +69,29 @@ namespace Wabbajack.CacheServer
return $"Enqueued {states.Count} jobs";
}
public static async Task StartJobQueue()
{
while (true)
{
try
{
var job = await Job.GetNext();
if (job == null)
{
await Task.Delay(5000);
continue;
}
var result = await job.Payload.Execute();
await Job.Finish(job, result);
}
catch (Exception ex)
{
}
}
}
}
}

98
Wabbajack.CacheServer/Jobs/IndexJob.cs Normal file
View File

@ -0,0 +1,98 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.CacheServer.DTOs;
using Wabbajack.CacheServer.DTOs.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.CacheServer.Jobs
{
public class IndexJob : AJobPayload
{
public Archive Archive { get; set; }
public override string Description { get; } = "Validate and index an archive";
public override bool UsesNexus { get => Archive.State is NexusDownloader.State; }
public override async Task<JobResult> Execute()
{
var pk = new List<object>();
pk.Add(AbstractDownloadState.TypeToName[Archive.State.GetType()]);
pk.AddRange(Archive.State.PrimaryKey);
var pk_str = string.Join("|",pk.Select(p => p.ToString()));
var found = await Server.Config.DownloadStates.Connect().AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync();
if (found.Count > 0)
return JobResult.Success();
string fileName = Archive.Name;
string folder = Guid.NewGuid().ToString();
Utils.Log($"Indexer is downloading ${fileName}");
var downloadDest = Path.Combine(Server.Config.Indexer.DownloadDir, folder, fileName);
await Archive.State.Download(downloadDest);
using (var queue = new WorkQueue())
{
var vfs = new Context(queue, true);
await vfs.AddRoot(Path.Combine(Server.Config.Indexer.DownloadDir, folder));
var archive = vfs.Index.ByRootPath.First();
var converted = ConvertArchive(new List<IndexedFile>(), archive.Value);
try
{
await Server.Config.IndexedFiles.Connect().InsertManyAsync(converted, new InsertManyOptions {IsOrdered = false});
}
catch (MongoBulkWriteException)
{
}
await Server.Config.DownloadStates.Connect().InsertOneAsync(new DownloadState
{
Key = pk_str,
Hash = archive.Value.Hash,
State = Archive.State,
IsValid = true
});
var to_path = Path.Combine(Server.Config.Indexer.ArchiveDir,
$"{Path.GetFileName(fileName)}_{archive.Value.Hash.FromBase64().ToHex()}_{Path.GetExtension(fileName)}");
if (File.Exists(to_path))
File.Delete(downloadDest);
else
File.Move(downloadDest, to_path);
}
return JobResult.Success();
}
private List<IndexedFile> ConvertArchive(List<IndexedFile> files, VirtualFile file, bool isTop = true)
{
var name = isTop ? Path.GetFileName(file.Name) : file.Name;
var ifile = new IndexedFile
{
Name = name,
Extension = Path.GetExtension(name),
Hash = file.Hash,
SHA256 = file.ExtendedHashes.SHA256,
SHA1 = file.ExtendedHashes.SHA1,
MD5 = file.ExtendedHashes.MD5,
CRC = file.ExtendedHashes.CRC,
Size = file.Size,
Children = file.Children != null ? file.Children.Select(f =>
{
ConvertArchive(files, f, false);
return f.Hash;
}).ToList() : new List<string>()
};
ifile.IsArchive = ifile.Children.Count > 0;
files.Add(ifile);
return files;
}
}
}

246
Wabbajack.CacheServer/NexusCacheModule.cs
View File

@ -1,12 +1,13 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Driver;
using Nancy;
using Nancy.Helpers;
using Wabbajack.CacheServer.DTOs;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
@ -24,6 +25,7 @@ namespace Wabbajack.CacheServer
Get("/nexus_api_cache/{request}.json", HandleCacheCall);
Get("/nexus_api_cache", ListCache);
Get("/nexus_api_cache/update", UpdateCache);
Get("/nexus_api_cache/ingest/{Folder}", HandleIngestCache);
}
public async Task<object> UpdateCache(object arg)
@ -47,30 +49,123 @@ namespace Wabbajack.CacheServer
}));
}
private async Task<object> HandleModInfo(dynamic arg)
private async Task<Response> HandleModInfo(dynamic arg)
{
Utils.Log($"{DateTime.Now} - Mod Info - {arg.GameName}/{arg.ModID}/");
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
return api.GetModInfo(GameRegistry.GetByNexusName((string)arg.GameName).Game, (string)arg.ModID).ToJSON();
string gameName = arg.GameName;
string modId = arg.ModId;
var result = await Server.Config.NexusModInfos.Connect()
.FindOneAsync(info => info.Game == gameName && info.ModId == modId);
string method = "CACHED";
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"/v1/{gameName}/mods/{modId}.json";
var body = await api.Get<ModInfo>(path);
result = new NexusCacheData<ModInfo>
{
Data = body,
Path = path,
Game = gameName,
ModId = modId
};
try
{
await Server.Config.NexusModInfos.Connect().InsertOneAsync(result);
}
catch (MongoWriteException)
{
}
method = "NOT_CACHED";
}
Response response = result.Data.ToJSON();
response.Headers.Add("WABBAJACK_CACHE_FROM", method);
response.ContentType = "application/json";
return response;
}
private async Task<object> HandleFileID(dynamic arg)
{
Utils.Log($"{DateTime.Now} - File Info - {arg.GameName}/{arg.ModID}/{arg.FileID}");
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
return api.GetFileInfo(new NexusDownloader.State
string gameName = arg.GameName;
string modId = arg.ModId;
string fileId = arg.FileId;
var result = await Server.Config.NexusFileInfos.Connect()
.FindOneAsync(info => info.Game == gameName && info.ModId == modId && info.FileId == fileId);
string method = "CACHED";
if (result == null)
{
GameName = arg.GameName,
ModID = arg.ModID,
FileID = arg.FileID
}).ToJSON();
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"/v1/{gameName}/mods/{modId}/files/{fileId}.json";
var body = await api.Get<NexusFileInfo>(path);
result = new NexusCacheData<NexusFileInfo>
{
Data = body,
Path = path,
Game = gameName,
ModId = modId,
FileId = fileId
};
try
{
await Server.Config.NexusFileInfos.Connect().InsertOneAsync(result);
}
catch (MongoWriteException)
{
}
method = "NOT_CACHED";
}
Response response = result.Data.ToJSON();
response.Headers.Add("WABBAJACK_CACHE_FROM", method);
response.ContentType = "application/json";
return response;
}
private async Task<object> HandleGetFiles(dynamic arg)
{
Utils.Log($"{DateTime.Now} - Mod Files - {arg.GameName} {arg.ModID}");
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
return api.GetModFiles(GameRegistry.GetByNexusName((string)arg.GameName).Game, (int)arg.ModID).ToJSON();
string gameName = arg.GameName;
string modId = arg.ModId;
var result = await Server.Config.NexusModFiles.Connect()
.FindOneAsync(info => info.Game == gameName && info.ModId == modId);
string method = "CACHED";
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"/v1/{gameName}/mods/{modId}/files.json";
var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
result = new NexusCacheData<NexusApiClient.GetModFilesResponse>
{
Data = body,
Path = path,
Game = gameName,
ModId = modId
};
try
{
await Server.Config.NexusModFiles.Connect().InsertOneAsync(result);
}
catch (MongoWriteException)
{
}
method = "NOT_CACHED";
}
Response response = result.Data.ToJSON();
response.Headers.Add("WABBAJACK_CACHE_FROM", method);
response.ContentType = "application/json";
return response;
}
private async Task<string> HandleCacheCall(dynamic arg)
@ -79,27 +174,12 @@ namespace Wabbajack.CacheServer
{
string param = (string)arg.request;
var url = new Uri(Encoding.UTF8.GetString(param.FromHex()));
var path = Path.Combine(NexusApiClient.LocalCacheDir, arg.request + ".json");
if (!File.Exists(path))
{
Utils.Log($"{DateTime.Now} - Not Cached - {url}");
var client = new HttpClient();
var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 8080, Scheme = "http"};
client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]);
await client.GetStringAsync(builder.Uri.ToString());
if (!File.Exists(path))
{
Utils.Log($"Still not cached : {path}");
throw new InvalidDataException("Invalid Data");
}
Utils.Log($"Is Now Cached : {path}");
}
Utils.Log($"{DateTime.Now} - From Cached - {url}");
return File.ReadAllText(path);
Utils.Log($"{DateTime.Now} - Not Cached - {url}");
var client = new HttpClient();
var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 8080, Scheme = "http"};
client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]);
return await client.GetStringAsync(builder.Uri.ToString());
}
catch (Exception ex)
{
@ -107,5 +187,105 @@ namespace Wabbajack.CacheServer
return "ERROR";
}
}
private async Task<string> HandleIngestCache(dynamic arg)
{
int count = 0;
int failed = 0;
using (var queue = new WorkQueue())
{
await Directory.EnumerateFiles(Path.Combine("c:\\tmp", (string)arg.Folder)).PMap(queue,
async file =>
{
Utils.Log($"Ingesting {file}");
if (!file.EndsWith(".json")) return;
var fileInfo = new FileInfo(file);
count++;
var url = new Url(
Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(file).FromHex()));
var split = url.Path.Split(new[] {'/'}, StringSplitOptions.RemoveEmptyEntries);
try
{
switch (split.Length)
{
case 5 when split[3] == "mods":
{
var body = file.FromJSON<ModInfo>();
var payload = new NexusCacheData<ModInfo>();
payload.Data = body;
payload.Game = split[2];
payload.Path = url.Path;
payload.ModId = body.mod_id;
payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc;
try
{
await Server.Config.NexusModInfos.Connect().InsertOneAsync(payload);
}
catch (MongoWriteException ex)
{
}
break;
}
case 6 when split[5] == "files.json":
{
var body = file.FromJSON<NexusApiClient.GetModFilesResponse>();
var payload = new NexusCacheData<NexusApiClient.GetModFilesResponse>();
payload.Path = url.Path;
payload.Data = body;
payload.Game = split[2];
payload.ModId = split[4];
payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc;
try
{
await Server.Config.NexusModFiles.Connect().InsertOneAsync(payload);
}
catch (MongoWriteException ex)
{
}
break;
}
case 7 when split[5] == "files":
{
var body = file.FromJSON<NexusFileInfo>();
var payload = new NexusCacheData<NexusFileInfo>();
payload.Data = body;
payload.Path = url.Path;
payload.Game = split[2];
payload.FileId = Path.GetFileNameWithoutExtension(split[6]);
payload.ModId = split[4];
payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc;
try
{
await Server.Config.NexusFileInfos.Connect().InsertOneAsync(payload);
}
catch (MongoWriteException ex)
{
}
break;
}
}
}
catch (Exception ex)
{
failed++;
}
});
}
return $"Inserted {count} caches, {failed} failed";
}
}
}

1
Wabbajack.CacheServer/Program.cs
View File

@ -16,6 +16,7 @@ namespace Wabbajack.CacheServer
using (var server = new Server("http://localhost:8080"))
{
//ListValidationService.Start();
var tsk = JobQueueEndpoints.StartJobQueue();
server.Start();
Console.ReadLine();
}

10
Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs
View File

@ -5,6 +5,7 @@ using System.Text;
using System.Threading.Tasks;
using Wabbajack.CacheServer.DTOs;
using Wabbajack.CacheServer.DTOs.JobQueue;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.CacheServer.ServerConfig
{
@ -14,5 +15,14 @@ namespace Wabbajack.CacheServer.ServerConfig
public MongoConfig<ModListStatus> ListValidation { get; set; }
public MongoConfig<Job> JobQueue { get; set; }
public MongoConfig<IndexedFile> IndexedFiles { get; set; }
public MongoConfig<DownloadState> DownloadStates { get; set; }
public MongoConfig<NexusCacheData<ModInfo>> NexusModInfos { get; set; }
public MongoConfig<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles { get; set; }
public MongoConfig<NexusCacheData<NexusFileInfo>> NexusFileInfos { get; set; }
public IndexerConfig Indexer { get; set; }
}
}

16
Wabbajack.CacheServer/ServerConfig/IndexerConfig.cs Normal file
View File

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.CacheServer.ServerConfig
{
public class IndexerConfig
{
public string DownloadDir { get; set; }
public string TempDir { get; set; }
public string ArchiveDir { get; set; }
}
}

10
Wabbajack.CacheServer/Wabbajack.CacheServer.csproj
View File

@ -73,8 +73,11 @@
<Reference Include="WindowsBase" />
</ItemGroup>
<ItemGroup>
<Compile Include="DTOs\DownloadState.cs" />
<Compile Include="DTOs\IndexedFile.cs" />
<Compile Include="DTOs\JobQueue\AJobPayload.cs" />
<Compile Include="DTOs\JobQueue\IndexJob.cs" />
<Compile Include="DTOs\NexusCacheData.cs" />
<Compile Include="Jobs\IndexJob.cs" />
<Compile Include="DTOs\JobQueue\Job.cs" />
<Compile Include="DTOs\JobQueue\JobResult.cs" />
<Compile Include="DTOs\Metric.cs" />
@ -91,6 +94,7 @@
<Compile Include="Server.cs" />
<Compile Include="Heartbeat.cs" />
<Compile Include="ServerConfig\BuildServerConfig.cs" />
<Compile Include="ServerConfig\IndexerConfig.cs" />
<Compile Include="ServerConfig\MongoConfig.cs" />
<Compile Include="TestingEndpoints.cs" />
</ItemGroup>
@ -109,6 +113,10 @@
<Project>{0a820830-a298-497d-85e0-e9a89efef5fe}</Project>
<Name>Wabbajack.Lib</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.VirtualFileSystem\Wabbajack.VirtualFileSystem.csproj">
<Project>{5d6a2eaf-6604-4c51-8ae2-a746b4bc5e3e}</Project>
<Name>Wabbajack.VirtualFileSystem</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<PackageReference Include="MongoDB.Driver">

28
Wabbajack.CacheServer/config.yaml
View File

@ -10,6 +10,28 @@ ListValidation:
JobQueue:
Host: internal.test.mongodb
Database: wabbajack
Collection: job_queue
Collection: job_queue
IndexedFiles:
Host: internal.test.mongodb
Database: wabbajack
Collection: indexed_files
NexusModInfos:
Host: internal.test.mongodb
Database: wabbajack
Collection: nexus_mod_infos
NexusModFiles:
Host: internal.test.mongodb
Database: wabbajack
Collection: nexus_mod_files
NexusFileInfos:
Host: internal.test.mongodb
Database: wabbajack
Collection: nexus_file_infos
DownloadStates:
Host: internal.test.mongodb
Database: wabbajack
Collection: download_states
Indexer:
DownloadDir: c:\tmp\downloads
TempDir: c:\tmp\tmp
ArchiveDir: c:\archives

1
Wabbajack.Common/Consts.cs
View File

@ -92,5 +92,6 @@ namespace Wabbajack.Common
public static string LocalAppDataPath => Path.Combine(KnownFolders.LocalAppData.Path, "Wabbajack");
public static string WabbajackCacheLocation = "http://build.wabbajack.org/nexus_api_cache/";
public static string WabbajackCacheHostname = "build.wabbajack.org";
}
}

2
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -40,6 +40,8 @@ namespace Wabbajack.Lib.Downloaders
TypeToName = NameToType.ToDictionary(k => k.Value, k => k.Key);
}
public abstract object[] PrimaryKey { get; }
/// <summary>

2
Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
View File

@ -52,6 +52,8 @@ namespace Wabbajack.Lib.Downloaders
internal string SourcePath => Path.Combine(Game.MetaData().GameLocation(), GameFile);
public override object[] PrimaryKey { get => new object[] {Game, GameFile}; }
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
return true;

2
Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs
View File

@ -36,6 +36,8 @@ namespace Wabbajack.Lib.Downloaders
public class State : AbstractDownloadState
{
public string Id { get; set; }
public override object[] PrimaryKey { get => new object[] {Id}; }
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
return whitelist.GoogleIDs.Contains(Id);

2
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -63,6 +63,8 @@ namespace Wabbajack.Lib.Downloaders
[Exclude]
public HttpClient Client { get; set; }
public override object[] PrimaryKey { get => new object[] {Url};}
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p));

2
Wabbajack.Lib/Downloaders/LoversLabDownloader.cs
View File

@ -118,6 +118,8 @@ namespace Wabbajack.Lib.Downloaders
public string FileID { get; set; }
public string FileName { get; set; }
public override object[] PrimaryKey { get => new object[] {FileID, FileName}; }
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
return true;

2
Wabbajack.Lib/Downloaders/ManualDownloader.cs
View File

@ -73,6 +73,8 @@ namespace Wabbajack.Lib.Downloaders
public class State : AbstractDownloadState
{
public string Url { get; set; }
public override object[] PrimaryKey { get => new object[] {Url}; }
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
return true;

2
Wabbajack.Lib/Downloaders/MediaFireDownloader.cs
View File

@ -24,6 +24,8 @@ namespace Wabbajack.Lib.Downloaders
{
public string Url { get; set; }
public override object[] PrimaryKey { get => new object[] {Url};}
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p));

5
Wabbajack.Lib/Downloaders/ModDBDownloader.cs
View File

@ -1,4 +1,5 @@
using System.Net.Http;
using System;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Wabbajack.Common;
@ -34,6 +35,8 @@ namespace Wabbajack.Lib.Downloaders
public class State : AbstractDownloadState
{
public string Url { get; set; }
public override object[] PrimaryKey { get => new object[]{Url}; }
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
// Everything from Moddb is whitelisted

26
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -112,18 +112,20 @@ namespace Wabbajack.Lib.Downloaders
public class State : AbstractDownloadState
{
public string Author;
public string FileID;
public string GameName;
public string ModID;
public string UploadedBy;
public string UploaderProfile;
public string Version;
public string SlideShowPic;
public string ModName;
public string NexusURL;
public string Summary;
public bool Adult;
public string Author { get; set; }
public string FileID { get; set; }
public string GameName { get; set; }
public string ModID { get; set; }
public string UploadedBy { get; set; }
public string UploaderProfile { get; set; }
public string Version { get; set; }
public string SlideShowPic { get; set; }
public string ModName { get; set; }
public string NexusURL { get; set; }
public string Summary { get; set; }
public bool Adult { get; set; }
public override object[] PrimaryKey { get => new object[]{GameName, ModID, FileID};}
public override bool IsWhitelisted(ServerWhitelist whitelist)
{

2
Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs
View File

@ -41,6 +41,8 @@ namespace Wabbajack.Lib.Downloaders
public class State : AbstractDownloadState
{
public SteamWorkshopItem Item { get; set; }
public override object[] PrimaryKey { get => new object[] {Item.Game, Item.ItemID}; }
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
return true;

53
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -201,6 +201,7 @@ namespace Wabbajack.Lib.NexusApi
{
var dailyRemaining = int.Parse(response.Headers.GetValues("x-rl-daily-remaining").First());
var hourlyRemaining = int.Parse(response.Headers.GetValues("x-rl-hourly-remaining").First());
Utils.Log($"Nexus Requests Remaining: {dailyRemaining} daily - {hourlyRemaining} hourly");
lock (RemainingLock)
{
@ -222,6 +223,7 @@ namespace Wabbajack.Lib.NexusApi
{
ApiKey = apiKey;
HttpClient.BaseAddress = new Uri("https://api.nexusmods.com");
// set default headers for all requests to the Nexus API
var headers = HttpClient.DefaultRequestHeaders;
headers.Add("User-Agent", Consts.UserAgent);
@ -240,10 +242,12 @@ namespace Wabbajack.Lib.NexusApi
return new NexusApiClient(apiKey);
}
private async Task<T> Get<T>(string url)
public async Task<T> Get<T>(string url)
{
var response = await HttpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
UpdateRemaining(response);
if (!response.IsSuccessStatusCode)
throw new HttpRequestException($"{response.StatusCode} - {response.ReasonPhrase}");
using (var stream = await response.Content.ReadAsStreamAsync())
{
@ -253,41 +257,10 @@ namespace Wabbajack.Lib.NexusApi
private async Task<T> GetCached<T>(string url)
{
var code = Encoding.UTF8.GetBytes(url).ToHex() + ".json";
if (UseLocalCache)
{
var cache_file = Path.Combine(LocalCacheDir, code);
lock (_diskLock)
{
if (!Directory.Exists(LocalCacheDir))
Directory.CreateDirectory(LocalCacheDir);
if (File.Exists(cache_file))
{
return cache_file.FromJSON<T>();
}
}
var result = await Get<T>(url);
if (result == null)
return result;
lock (_diskLock)
{
result.ToJSON(cache_file);
}
return result;
}
try
{
return await Get<T>(Consts.WabbajackCacheLocation + code);
var builder = new UriBuilder(url) { Host = Consts.WabbajackCacheHostname, Port = 80, Scheme = "http" };
return await Get<T>(builder.ToString());
}
catch (Exception)
{
@ -388,15 +361,7 @@ namespace Wabbajack.Lib.NexusApi
}
private static bool? _useLocalCache;
public static bool UseLocalCache
{
get
{
if (_useLocalCache == null) return LocalCacheDir != null;
return _useLocalCache ?? false;
}
set => _useLocalCache = value;
}
public static MethodInfo CacheMethod { get; set; }
private static string _localCacheDir;
public static string LocalCacheDir
@ -413,7 +378,7 @@ namespace Wabbajack.Lib.NexusApi
public async Task ClearUpdatedModsInCache()
{
if (!UseLocalCache) return;
if (NexusApiClient.CacheMethod == null) return;
using (var queue = new WorkQueue())
{
var invalid_json = (await Directory.EnumerateFiles(LocalCacheDir, "*.json")

6
Wabbajack.Test/DownloaderTests.cs
View File

@ -215,10 +215,10 @@ namespace Wabbajack.Test
[TestMethod]
public async Task NexusDownload()
{
var old_val = NexusApiClient.UseLocalCache;
var old_val = NexusApiClient.CacheMethod;
try
{
NexusApiClient.UseLocalCache = false;
NexusApiClient.CacheMethod = null;
var ini = @"[General]
gameName=SkyrimSE
modID = 12604
@ -244,7 +244,7 @@ namespace Wabbajack.Test
}
finally
{
NexusApiClient.UseLocalCache = old_val;
NexusApiClient.CacheMethod = old_val;
}
}

5
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -35,12 +35,15 @@ namespace Wabbajack.VirtualFileSystem
public StatusUpdateTracker UpdateTracker { get; set; } = new StatusUpdateTracker(1);
public WorkQueue Queue { get; }
public bool UseExtendedHashes { get; set; }
public Context(WorkQueue queue)
public Context(WorkQueue queue, bool extendedHashes = false)
{
Queue = queue;
UseExtendedHashes = extendedHashes;
}
public TemporaryDirectory GetTemporaryFolder()
{
return new TemporaryDirectory(Path.Combine(_stagingFolder, Guid.NewGuid().ToString()));

42
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -3,8 +3,10 @@ using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using K4os.Hash.Crc;
using Wabbajack.Common;
using Wabbajack.Common.CSP;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
@ -40,6 +42,7 @@ namespace Wabbajack.VirtualFileSystem
}
public string Hash { get; internal set; }
public ExtendedHashes ExtendedHashes { get; set; }
public long Size { get; internal set; }
public long LastModified { get; internal set; }
@ -143,6 +146,8 @@ namespace Wabbajack.VirtualFileSystem
LastAnalyzed = DateTime.Now.Ticks,
Hash = abs_path.FileHash()
};
if (context.UseExtendedHashes)
self.ExtendedHashes = ExtendedHashes.FromFile(abs_path);
if (FileExtractor.CanExtract(abs_path))
{
@ -162,6 +167,7 @@ namespace Wabbajack.VirtualFileSystem
return self;
}
public void Write(MemoryStream ms)
{
using (var bw = new BinaryWriter(ms, Encoding.UTF8, true))
@ -265,6 +271,42 @@ namespace Wabbajack.VirtualFileSystem
}
}
public class ExtendedHashes
{
public static ExtendedHashes FromFile(string file)
{
var hashes = new ExtendedHashes();
using (var stream = File.OpenRead(file))
{
hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex();
stream.Position = 0;
hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex();
stream.Position = 0;
hashes.MD5 = System.Security.Cryptography.MD5.Create().ComputeHash(stream).ToHex();
stream.Position = 0;
var bytes = new byte[1024 * 8];
var crc = new Crc32();
while (true)
{
var read = stream.Read(bytes, 0, bytes.Length);
if (read == 0) break;
crc.Update(bytes, 0, read);
}
hashes.CRC = crc.DigestBytes().ToHex();
}
return hashes;
}
public string SHA256 { get; set; }
public string SHA1 { get; set; }
public string MD5 { get; set; }
public string CRC { get; set; }
}
public class CannotStageNativeFile : Exception
{
public CannotStageNativeFile(string cannotStageANativeFile) : base(cannotStageANativeFile)

3
Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj
View File

@ -88,6 +88,9 @@
<PackageReference Include="AlphaFS">
<Version>2.2.6</Version>
</PackageReference>
<PackageReference Include="K4os.Hash.Crc">
<Version>1.1.4</Version>
</PackageReference>
<PackageReference Include="System.Collections.Immutable">
<Version>1.7.0</Version>
</PackageReference>