Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

delete old server

Browse Source
This commit is contained in:
Timothy Baldridge 2020-01-09 16:20:03 -07:00
parent ad844e8545
commit f004595b5f
31 changed files with 0 additions and 2014 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Wabbajack.CacheServer/App.config
View File

@ -1,6 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.8"/>
</startup>
</configuration>

23
Wabbajack.CacheServer/DTOs/DownloadState.cs
View File

@ -1,23 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.CacheServer.DTOs
{
public class DownloadState
{
[BsonId]
public string Key { get; set; }
public string Hash { get; set; }
public AbstractDownloadState State { get; set; }
public bool IsValid { get; set; }
public DateTime LastValidationTime { get; set; } = DateTime.Now;
public DateTime FirstValidationTime { get; set; } = DateTime.Now;
}
}

31
Wabbajack.CacheServer/DTOs/IndexedFile.cs
View File

@ -1,31 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.RightsManagement;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.CacheServer.DTOs
{
public class IndexedFile
{
[BsonId]
public string Hash { get; set; }
public string SHA256 { get; set; }
public string SHA1 { get; set; }
public string MD5 { get; set; }
public string CRC { get; set; }
public long Size { get; set; }
public bool IsArchive { get; set; }
public List<ChildFile> Children { get; set; } = new List<ChildFile>();
}
public class ChildFile
{
public string Name;
public string Extension;
public string Hash;
}
}

32
Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs
View File

@ -1,32 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.CacheServer.Jobs;
namespace Wabbajack.CacheServer.DTOs.JobQueue
{
public abstract class AJobPayload
{
public static List<Type> KnownSubTypes = new List<Type> {typeof(IndexJob)};
public static Dictionary<Type, string> TypeToName { get; set; }
public static Dictionary<string, Type> NameToType { get; set; }
[BsonIgnore]
public abstract string Description { get; }
public virtual bool UsesNexus { get; } = false;
public abstract Task<JobResult> Execute();
static AJobPayload()
{
NameToType = KnownSubTypes.ToDictionary(t => t.FullName.Substring(t.Namespace.Length + 1), t => t);
TypeToName = NameToType.ToDictionary(k => k.Value, k => k.Key);
}
}
}

67
Wabbajack.CacheServer/DTOs/JobQueue/Job.cs
View File

@ -1,67 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
namespace Wabbajack.CacheServer.DTOs.JobQueue
{
public class Job
{
public enum JobPriority : int
{
Low,
Normal,
High,
}
[BsonId]
public Guid Id { get; set; }
public DateTime? Started { get; set; }
public DateTime? Ended { get; set; }
public DateTime Created { get; set; } = DateTime.Now;
public JobPriority Priority { get; set; } = JobPriority.Normal;
public JobResult Result { get; set; }
public bool RequiresNexus { get; set; } = true;
public AJobPayload Payload { get; set; }
public static async Task<Guid> Enqueue(Job job)
{
await Server.Config.JobQueue.Connect().InsertOneAsync(job);
return job.Id;
}
public static async Task<Job> GetNext()
{
var filter = new BsonDocument
{
{"Started", BsonNull.Value}
};
var update = new BsonDocument
{
{"$set", new BsonDocument {{"Started", DateTime.Now}}}
};
var sort = new {Priority=-1, Created=1}.ToBsonDocument();
var job = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update, new FindOneAndUpdateOptions<Job>{Sort = sort});
return job;
}
public static async Task<Job> Finish(Job job, JobResult jobResult)
{
var filter = new BsonDocument
{
{"query", new BsonDocument {{"Id", job.Id}}},
};
var update = new BsonDocument
{
{"$set", new BsonDocument {{"Ended", DateTime.Now}, {"Result", jobResult.ToBsonDocument()}}}
};
var result = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update);
return result;
}
}
}

36
Wabbajack.CacheServer/DTOs/JobQueue/JobResult.cs
View File

@ -1,36 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
namespace Wabbajack.CacheServer.DTOs.JobQueue
{
public class JobResult
{
public JobResultType ResultType { get; set; }
[BsonIgnoreIfNull]
public string Message { get; set; }
[BsonIgnoreIfNull]
public string Stacktrace { get; set; }
public static JobResult Success()
{
return new JobResult { ResultType = JobResultType.Success };
}
public static JobResult Error(Exception ex)
{
return new JobResult {ResultType = JobResultType.Error, Stacktrace = ex.ToString()};
}
}
public enum JobResultType
{
Success,
Error
}
}

17
Wabbajack.CacheServer/DTOs/Metric.cs
View File

@ -1,17 +0,0 @@
using System;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace Wabbajack.CacheServer.DTOs
{
public class Metric
{
[BsonId]
public ObjectId Id;
public DateTime Timestamp;
public string Action;
public string Subject;
public string MetricsKey;
}
}

70
Wabbajack.CacheServer/DTOs/ModListStatus.cs
View File

@ -1,70 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.Lib;
using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.CacheServer.DTOs
{
public class ModListStatus
{
[BsonId]
public string Id { get; set; }
public ModlistSummary Summary { get; set; }
public ModlistMetadata Metadata { get; set; }
public DetailedStatus DetailedStatus { get; set; }
public static async Task Update(ModListStatus status)
{
var id = status.Metadata.Links.MachineURL;
await Server.Config.ListValidation.Connect().FindOneAndReplaceAsync<ModListStatus>(s => s.Id == id, status, new FindOneAndReplaceOptions<ModListStatus> {IsUpsert = true});
}
public static IQueryable<ModListStatus> AllSummaries
{
get
{
return null;
}
}
public static async Task<ModListStatus> ByName(string name)
{
var result = await Server.Config.ListValidation.Connect()
.AsQueryable()
.Where(doc => doc.Metadata.Links.MachineURL == name || doc.Metadata.Title == name)
.ToListAsync();
return result.First();
}
public static IMongoQueryable<ModListStatus> All
{
get
{
return Server.Config.ListValidation.Connect().AsQueryable();
}
}
}
public class DetailedStatus
{
public string Name { get; set; }
public DateTime Checked { get; set; } = DateTime.Now;
public List<DetailedStatusItem> Archives { get; set; }
public DownloadMetadata DownloadMetaData { get; set; }
public bool HasFailures { get; set; }
public string MachineName { get; set; }
}
public class DetailedStatusItem
{
public bool IsFailing { get; set; }
public Archive Archive { get; set; }
}
}

14
Wabbajack.CacheServer/DTOs/MongoDoc.cs
View File

@ -1,14 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson;
namespace Wabbajack.CacheServer.DTOs
{
public class MongoDoc
{
public ObjectId _id { get; set; } = ObjectId.Empty;
}
}

24
Wabbajack.CacheServer/DTOs/NexusCacheData.cs
View File

@ -1,24 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
namespace Wabbajack.CacheServer.DTOs
{
public class NexusCacheData<T>
{
[BsonId]
public string Path { get; set; }
public T Data { get; set; }
public string Game { get; set; }
public string ModId { get; set; }
public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow;
[BsonIgnoreIfNull]
public string FileId { get; set; }
}
}

74
Wabbajack.CacheServer/DTOs/SerializerSettings.cs
View File

@ -1,74 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Bson.IO;
using MongoDB.Bson.Serialization;
using MongoDB.Bson.Serialization.Conventions;
using Wabbajack.CacheServer.DTOs.JobQueue;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.CacheServer.DTOs
{
public static class SerializerSettings
{
public static void Init()
{
var dis = new TypeDiscriminator(typeof(AbstractDownloadState), AbstractDownloadState.NameToType,
AbstractDownloadState.TypeToName);
BsonSerializer.RegisterDiscriminatorConvention(typeof(AbstractDownloadState), dis);
BsonClassMap.RegisterClassMap<AbstractDownloadState>(cm => cm.SetIsRootClass(true));
dis = new TypeDiscriminator(typeof(AJobPayload), AJobPayload.NameToType, AJobPayload.TypeToName);
BsonSerializer.RegisterDiscriminatorConvention(typeof(AJobPayload), dis);
BsonClassMap.RegisterClassMap<AJobPayload>(cm => cm.SetIsRootClass(true));
}
}
public class TypeDiscriminator : IDiscriminatorConvention
{
private readonly Type defaultType;
private readonly Dictionary<string, Type> typeMap;
private Dictionary<Type, string> revMap;
public TypeDiscriminator(Type defaultType,
Dictionary<string, Type> typeMap, Dictionary<Type, string> revMap)
{
this.defaultType = defaultType;
this.typeMap = typeMap;
this.revMap = revMap;
}
/// <summary>
/// Element Name
/// </summary>
public string ElementName => "_wjType";
public Type GetActualType(IBsonReader bsonReader, Type nominalType)
{
Type type = null;
var bookmark = bsonReader.GetBookmark();
bsonReader.ReadStartDocument();
if (bsonReader.FindElement(ElementName))
{
var value = bsonReader.ReadString();
if (typeMap.ContainsKey(value))
type = typeMap[value];
}
bsonReader.ReturnToBookmark(bookmark);
if (type == null)
throw new Exception($"Type mis-configuration can't find bson type for ${nominalType}");
return type;
}
public BsonValue GetDiscriminator(Type nominalType, Type actualType)
{
return revMap[actualType];
}
}
}

19
Wabbajack.CacheServer/Extensions.cs
View File

@ -1,19 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
namespace Wabbajack.CacheServer
{
public static class Extensions
{
public static async Task<T> FindOneAsync<T>(this IMongoCollection<T> coll, Expression<Func<T, bool>> expr)
{
return (await coll.AsQueryable().Where(expr).Take(1).ToListAsync()).FirstOrDefault();
}
}
}

3
Wabbajack.CacheServer/FodyWeavers.xml
View File

@ -1,3 +0,0 @@
<Weavers xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="FodyWeavers.xsd">
<Costura />
</Weavers>

111
Wabbajack.CacheServer/FodyWeavers.xsd
View File

@ -1,111 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
<!-- This file was generated by Fody. Manual changes to this file will be lost when your project is rebuilt. -->
<xs:element name="Weavers">
<xs:complexType>
<xs:all>
<xs:element name="Costura" minOccurs="0" maxOccurs="1">
<xs:complexType>
<xs:all>
<xs:element minOccurs="0" maxOccurs="1" name="ExcludeAssemblies" type="xs:string">
<xs:annotation>
<xs:documentation>A list of assembly names to exclude from the default action of "embed all Copy Local references", delimited with line breaks</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element minOccurs="0" maxOccurs="1" name="IncludeAssemblies" type="xs:string">
<xs:annotation>
<xs:documentation>A list of assembly names to include from the default action of "embed all Copy Local references", delimited with line breaks.</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element minOccurs="0" maxOccurs="1" name="Unmanaged32Assemblies" type="xs:string">
<xs:annotation>
<xs:documentation>A list of unmanaged 32 bit assembly names to include, delimited with line breaks.</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element minOccurs="0" maxOccurs="1" name="Unmanaged64Assemblies" type="xs:string">
<xs:annotation>
<xs:documentation>A list of unmanaged 64 bit assembly names to include, delimited with line breaks.</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element minOccurs="0" maxOccurs="1" name="PreloadOrder" type="xs:string">
<xs:annotation>
<xs:documentation>The order of preloaded assemblies, delimited with line breaks.</xs:documentation>
</xs:annotation>
</xs:element>
</xs:all>
<xs:attribute name="CreateTemporaryAssemblies" type="xs:boolean">
<xs:annotation>
<xs:documentation>This will copy embedded files to disk before loading them into memory. This is helpful for some scenarios that expected an assembly to be loaded from a physical file.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="IncludeDebugSymbols" type="xs:boolean">
<xs:annotation>
<xs:documentation>Controls if .pdbs for reference assemblies are also embedded.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="DisableCompression" type="xs:boolean">
<xs:annotation>
<xs:documentation>Embedded assemblies are compressed by default, and uncompressed when they are loaded. You can turn compression off with this option.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="DisableCleanup" type="xs:boolean">
<xs:annotation>
<xs:documentation>As part of Costura, embedded assemblies are no longer included as part of the build. This cleanup can be turned off.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="LoadAtModuleInit" type="xs:boolean">
<xs:annotation>
<xs:documentation>Costura by default will load as part of the module initialization. This flag disables that behavior. Make sure you call CosturaUtility.Initialize() somewhere in your code.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="IgnoreSatelliteAssemblies" type="xs:boolean">
<xs:annotation>
<xs:documentation>Costura will by default use assemblies with a name like 'resources.dll' as a satellite resource and prepend the output path. This flag disables that behavior.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="ExcludeAssemblies" type="xs:string">
<xs:annotation>
<xs:documentation>A list of assembly names to exclude from the default action of "embed all Copy Local references", delimited with |</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="IncludeAssemblies" type="xs:string">
<xs:annotation>
<xs:documentation>A list of assembly names to include from the default action of "embed all Copy Local references", delimited with |.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="Unmanaged32Assemblies" type="xs:string">
<xs:annotation>
<xs:documentation>A list of unmanaged 32 bit assembly names to include, delimited with |.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="Unmanaged64Assemblies" type="xs:string">
<xs:annotation>
<xs:documentation>A list of unmanaged 64 bit assembly names to include, delimited with |.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="PreloadOrder" type="xs:string">
<xs:annotation>
<xs:documentation>The order of preloaded assemblies, delimited with |.</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
</xs:element>
</xs:all>
<xs:attribute name="VerifyAssembly" type="xs:boolean">
<xs:annotation>
<xs:documentation>'true' to run assembly verification (PEVerify) on the target assembly after all weavers have been executed.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="VerifyIgnoreCodes" type="xs:string">
<xs:annotation>
<xs:documentation>A comma-separated list of error codes that can be safely ignored in assembly verification.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="GenerateXsd" type="xs:boolean">
<xs:annotation>
<xs:documentation>'false' to turn off automatic generation of the XML Schema file.</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
</xs:element>
</xs:schema>

24
Wabbajack.CacheServer/Heartbeat.cs
View File

@ -1,24 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Nancy;
namespace Wabbajack.CacheServer
{
public class Heartbeat : NancyModule
{
private static DateTime startTime = DateTime.Now;
public Heartbeat() : base("/")
{
Get("/heartbeat", HandleHeartbeat);
}
private object HandleHeartbeat(object arg)
{
return $"Service is live for: {DateTime.Now - startTime}";
}
}
}

144
Wabbajack.CacheServer/JobQueueEndpoints.cs
View File

@ -1,144 +0,0 @@
using System;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Security.Policy;
using System.Threading.Tasks;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nancy;
using Nettle;
using Wabbajack.CacheServer.DTOs.JobQueue;
using Wabbajack.CacheServer.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.Downloaders;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.CacheServer
{
public class JobQueueEndpoints : NancyModule
{
public JobQueueEndpoints() : base ("/jobs")
{
Get("/", HandleListJobs);
Get("/enqueue_curated_for_indexing", HandleEnqueueAllCurated);
Get("/enqueue_game_files_for_indexing", HandleEnqueueAllGameFiles);
}
private readonly Func<object, string> HandleListJobsTemplate = NettleEngine.GetCompiler().Compile(@"
<html><head/><body>
<h2>Jobs - {{$.jobs.Count}} Pending</h2>
<h3>{{$.time}}</h3>
<ol>
{{each $.jobs}}
<li>{{$.Description}}</li>
{{/each}}
</ol>
<script>
setTimeout(function() { location.reload();}, 10000);
</script>
</body></html>");
private async Task<Response> HandleListJobs(object arg)
{
var jobs = await Server.Config.JobQueue.Connect()
.AsQueryable<Job>()
.Where(j => j.Ended == null)
.OrderByDescending(j => j.Priority)
.ThenBy(j => j.Created)
.ToListAsync();
var response = (Response)HandleListJobsTemplate(new {jobs, time = DateTime.Now});
response.ContentType = "text/html";
return response;
}
private async Task<string> HandleEnqueueAllCurated(object arg)
{
var states = await Server.Config.ListValidation.Connect()
.AsQueryable()
.SelectMany(lst => lst.DetailedStatus.Archives)
.Select(a => a.Archive)
.ToListAsync();
var jobs = states.Select(state => new IndexJob {Archive = state})
.Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus})
.ToList();
if (jobs.Count > 0)
await Server.Config.JobQueue.Connect().InsertManyAsync(jobs);
return $"Enqueued {states.Count} jobs";
}
private async Task<string> HandleEnqueueAllGameFiles(object arg)
{
using (var queue = new WorkQueue(4))
{
var states = GameRegistry.Games.Values
.Where(game => game.GameLocation() != null && game.MainExecutable != null)
.SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", SearchOption.AllDirectories)
.Select(file => new GameFileSourceDownloader.State
{
Game = game.Game,
GameVersion = game.InstalledVersion,
GameFile = file.RelativeTo(game.GameLocation()),
}))
.ToList();
await states.PMap(queue, state =>
{
state.Hash = Path.Combine(state.Game.MetaData().GameLocation(), state.GameFile).FileHash();
});
var jobs = states.Select(state => new IndexJob {Archive = new Archive {Name = Path.GetFileName(state.GameFile), State = state}})
.Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus})
.ToList();
if (jobs.Count > 0)
await Server.Config.JobQueue.Connect().InsertManyAsync(jobs);
return $"Enqueued {states.Count} Jobs";
}
}
public static async Task StartJobQueue()
{
foreach (var task in Enumerable.Range(0, 4))
{
var tsk = StartJobQueueInner();
}
}
private static async Task StartJobQueueInner()
{
while (true)
{
try
{
var job = await Job.GetNext();
if (job == null)
{
await Task.Delay(5000);
continue;
}
var result = await job.Payload.Execute();
await Job.Finish(job, result);
}
catch (Exception ex)
{
}
}
}
}
}

106
Wabbajack.CacheServer/Jobs/IndexJob.cs
View File

@ -1,106 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.CacheServer.DTOs;
using Wabbajack.CacheServer.DTOs.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.CacheServer.Jobs
{
public class IndexJob : AJobPayload
{
public Archive Archive { get; set; }
public override string Description { get; } = "Validate and index an archive";
public override bool UsesNexus { get => Archive.State is NexusDownloader.State; }
public override async Task<JobResult> Execute()
{
var pk = new List<object>();
pk.Add(AbstractDownloadState.TypeToName[Archive.State.GetType()]);
pk.AddRange(Archive.State.PrimaryKey);
var pk_str = string.Join("|",pk.Select(p => p.ToString()));
var found = await Server.Config.DownloadStates.Connect().AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync();
if (found.Count > 0)
return JobResult.Success();
string fileName = Archive.Name;
string folder = Guid.NewGuid().ToString();
Utils.Log($"Indexer is downloading {fileName}");
var downloadDest = Path.Combine(Server.Config.Indexer.DownloadDir, folder, fileName);
await Archive.State.Download(downloadDest);
using (var queue = new WorkQueue())
{
var vfs = new Context(queue, true);
await vfs.AddRoot(Path.Combine(Server.Config.Indexer.DownloadDir, folder));
var archive = vfs.Index.ByRootPath.First();
var converted = ConvertArchive(new List<IndexedFile>(), archive.Value);
try
{
await Server.Config.IndexedFiles.Connect().InsertManyAsync(converted, new InsertManyOptions {IsOrdered = false});
}
catch (MongoBulkWriteException)
{
}
await Server.Config.DownloadStates.Connect().InsertOneAsync(new DownloadState
{
Key = pk_str,
Hash = archive.Value.Hash,
State = Archive.State,
IsValid = true
});
var to_path = Path.Combine(Server.Config.Indexer.ArchiveDir,
$"{Path.GetFileName(fileName)}_{archive.Value.Hash.FromBase64().ToHex()}_{Path.GetExtension(fileName)}");
if (File.Exists(to_path))
File.Delete(downloadDest);
else
File.Move(downloadDest, to_path);
Utils.DeleteDirectory(Path.Combine(Server.Config.Indexer.DownloadDir, folder));
}
return JobResult.Success();
}
private List<IndexedFile> ConvertArchive(List<IndexedFile> files, VirtualFile file, bool isTop = true)
{
var name = isTop ? Path.GetFileName(file.Name) : file.Name;
var ifile = new IndexedFile
{
Hash = file.Hash,
SHA256 = file.ExtendedHashes.SHA256,
SHA1 = file.ExtendedHashes.SHA1,
MD5 = file.ExtendedHashes.MD5,
CRC = file.ExtendedHashes.CRC,
Size = file.Size,
Children = file.Children != null ? file.Children.Select(
f =>
{
ConvertArchive(files, f, false);
return new ChildFile
{
Hash = f.Hash,
Name = f.Name.ToLowerInvariant(),
Extension = Path.GetExtension(f.Name.ToLowerInvariant())
};
}).ToList() : new List<ChildFile>()
};
ifile.IsArchive = ifile.Children.Count > 0;
files.Add(ifile);
return files;
}
}
}

252
Wabbajack.CacheServer/ListValidationService.cs
View File

@ -1,252 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using MongoDB.Driver;
using Nancy;
using Wabbajack.CacheServer.DTOs;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using MongoDB.Driver.Linq;
using Nettle;
using Nettle.Functions;
namespace Wabbajack.CacheServer
{
public class ListValidationService : NancyModule
{
public ListValidationService() : base("/lists")
{
Get("/status", HandleGetLists);
Get("/force_recheck", HandleForceRecheck);
Get("/status/{Name}.json", HandleGetListJson);
Get("/status/{Name}.html", HandleGetListHtml);
Get("/status/{Name}/broken.rss", HandleGetRSSFeed);
}
private async Task<string> HandleForceRecheck(object arg)
{
await ValidateLists(false);
return "done";
}
private async Task<string> HandleGetLists(object arg)
{
var summaries = await ModListStatus.All.Select(m => m.Summary).ToListAsync();
return summaries.ToJSON();
}
public class ArchiveSummary
{
public string Name;
public AbstractDownloadState State;
}
public class DetailedSummary
{
public string Name;
public DateTime Checked;
public List<ArchiveSummary> Failed;
public List<ArchiveSummary> Passed;
}
private async Task<string> HandleGetListJson(dynamic arg)
{
var metric = Metrics.Log("list_validation.get_list_json", (string)arg.Name);
var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
return lst.ToJSON();
}
private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
<html><body>
<h2>{{lst.Name}} - {{lst.Checked}}</h2>
<h3>Failed ({{failed.Count}}):</h3>
<ul>
{{each $.failed }}
<li>{{$.Archive.Name}}</li>
{{/each}}
</ul>
<h3>Passed ({{passed.Count}}):</h3>
<ul>
{{each $.passed }}
<li>{{$.Archive.Name}}</li>
{{/each}}
</ul>
</body></html>
");
private async Task<Response> HandleGetListHtml(dynamic arg)
{
var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
var response = (Response)HandleGetListTemplate(new
{
lst,
failed = lst.Archives.Where(a => a.IsFailing).ToList(),
passed = lst.Archives.Where(a => !a.IsFailing).ToList()
});
response.ContentType = "text/html";
return response;
}
private static readonly Func<object, string> HandleGetRSSFeedTemplate = NettleEngine.GetCompiler().Compile(@"
<?xml version=""1.0""?>
<rss version=""2.0"">
<channel>
<title>{{lst.Name}} - Broken Mods</title>
<link>http://build.wabbajack.org/status/{{lst.Name}}.html</link>
<description>These are mods that are broken and need updating</description>
{{ each $.failed }}
<item>
<title>{{$.Archive.Name}}</title>
<link>{{$.Archive.Name}}</link>
</item>
{{/each}}
</channel>
</rss>
");
public async Task<Response> HandleGetRSSFeed(dynamic arg)
{
var metric = Metrics.Log("failed_rss", arg.Name);
var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
var response = (Response)HandleGetRSSFeedTemplate(new
{
lst,
failed = lst.Archives.Where(a => a.IsFailing).ToList(),
passed = lst.Archives.Where(a => !a.IsFailing).ToList()
});
response.ContentType = "application/rss+xml";
await metric;
return response;
}
public static void Start()
{
Task.Run(async () =>
{
while (true)
{
try
{
await ValidateLists();
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
}
// Sleep for two hours
await Task.Delay(1000 * 60 * 60 * 2);
}
}).FireAndForget();
}
public static async Task ValidateLists(bool skipIfNewer = true)
{
Utils.Log("Cleaning Nexus Cache");
var client = new HttpClient();
//await client.GetAsync("http://build.wabbajack.org/nexus_api_cache/update");
Utils.Log("Starting Modlist Validation");
var modlists = await ModlistMetadata.LoadFromGithub();
using (var queue = new WorkQueue())
{
foreach (var list in modlists)
{
try
{
await ValidateList(list, queue, skipIfNewer);
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
}
}
}
Utils.Log($"Done validating {modlists.Count} lists");
}
private static async Task ValidateList(ModlistMetadata list, WorkQueue queue, bool skipIfNewer = true)
{
var existing = await Server.Config.ListValidation.Connect().FindOneAsync(l => l.Id == list.Links.MachineURL);
if (skipIfNewer && existing != null && DateTime.UtcNow - existing.DetailedStatus.Checked < TimeSpan.FromHours(2))
return;
var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ExtensionManager.Extension);
if (list.NeedsDownload(modlist_path))
{
if (File.Exists(modlist_path))
File.Delete(modlist_path);
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlist_path);
}
else
{
Utils.Log($"No changes detected from downloaded modlist");
}
Utils.Log($"Loading {modlist_path}");
var installer = AInstaller.LoadFromFile(modlist_path);
Utils.Log($"{installer.Archives.Count} archives to validate");
DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
var validated = (await installer.Archives
.PMap(queue, async archive =>
{
Utils.Log($"Validating: {archive.Name}");
bool is_failed;
try
{
is_failed = !(await archive.State.Verify());
}
catch (Exception)
{
is_failed = false;
}
return new DetailedStatusItem {IsFailing = is_failed, Archive = archive};
}))
.ToList();
var status = new DetailedStatus
{
Name = list.Title,
Archives = validated.OrderBy(v => v.Archive.Name).ToList(),
DownloadMetaData = list.DownloadMetadata,
HasFailures = validated.Any(v => v.IsFailing)
};
var dto = new ModListStatus
{
Id = list.Links.MachineURL,
Summary = new ModlistSummary
{
Name = status.Name,
Checked = status.Checked,
Failed = status.Archives.Count(a => a.IsFailing),
Passed = status.Archives.Count(a => !a.IsFailing),
},
DetailedStatus = status,
Metadata = list
};
await ModListStatus.Update(dto);
}
}
}

140
Wabbajack.CacheServer/Metrics.cs
View File

@ -1,140 +0,0 @@
using System;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nancy;
using Wabbajack.CacheServer.DTOs;
using Wabbajack.Common;
namespace Wabbajack.CacheServer
{
/// <summary>
/// Extremely
/// </summary>
public class Metrics : NancyModule
{
private static SemaphoreSlim _lockObject = new SemaphoreSlim(1);
public static async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
{
var msg = new[] {string.Join("\t", new[]{timestamp.ToString(), metricsKey, action, subject})};
Utils.Log(msg.First());
var db = Server.Config.Metrics.Connect();
await db.InsertOneAsync(new Metric {Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey});
}
public static Task Log(string action, string subject)
{
return Log(DateTime.Now, action, subject);
}
public Metrics() : base("/")
{
Get("/metrics/{Action}/{Value}", HandleMetrics);
Get("/metrics/chart/", HandleChart);
Get("/metrics/chart/{Action}/", HandleChart);
Get("/metrics/chart/{Action}/{Value}/", HandleChart);
Get("/metrics/ingest/{filename}", HandleBulkIngest);
}
private async Task<string> HandleBulkIngest(dynamic arg)
{
Log("Bulk Loading " + arg.filename.ToString());
var lines = File.ReadAllLines(Path.Combine(@"c:\tmp", (string)arg.filename));
var db = Server.Config.Metrics.Connect();
var data = lines.Select(line => line.Split('\t'))
.Where(line => line.Length == 3)
.Select(line => new Metric{ Timestamp = DateTime.Parse(line[0]), Action = line[1], Subject = line[2] })
.ToList();
foreach (var metric in data)
await db.InsertOneAsync(metric);
return $"Processed {lines.Length} records";
}
private async Task<string> HandleMetrics(dynamic arg)
{
var date = DateTime.UtcNow;
await Log(date, arg.Action, arg.Value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault());
return date.ToString();
}
private async Task<Response> HandleChart(dynamic arg)
{
/*var data = (await GetData()).Select(line => line.Split('\t'))
.Where(line => line.Length == 3)
.Select(line => new {date = DateTime.Parse(line[0]), Action = line[1], Value = line[2]});*/
var q = Server.Config.Metrics.Connect().AsQueryable();
// Remove guids / Default, which come from testing
if (arg?.Action != null)
{
var action = (string)arg.Action;
q = q.Where(d => d.Action == action);
}
if (arg?.Value != null)
{
var value = (string)arg.Value;
q = q.Where(d => d.Subject.StartsWith(value));
}
var data = (await q.Take(Int32.MaxValue).ToListAsync()).AsEnumerable();
data = data.Where(d => !Guid.TryParse(d.Subject ?? "", out Guid v) && (d.Subject ?? "") != "Default");
var grouped_and_counted = data.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd"))
.OrderBy(d => d.Key)
.Select(d => new {Day = d.Key, Count = d.Count()})
.ToList();
var sb = new StringBuilder();
sb.Append("<html><head><script src=\"https://cdn.jsdelivr.net/npm/chart.js@2.8.0\"></script></head>");
sb.Append("<body><canvas id=\"myChart\"></canvas>");
sb.Append("<script language='javascript'>");
var script = @"var ctx = document.getElementById('myChart').getContext('2d');
var chart = new Chart(ctx, {
// The type of chart we want to create
type: 'line',
// The data for our dataset
data: {
labels: [{{LABELS}}],
datasets: [{
label: '{{DATASET}}',
backgroundColor: 'rgb(255, 99, 132)',
borderColor: 'rgb(255, 99, 132)',
data: [{{DATA}}]
}]
},
// Configuration options go here
options: {}
});";
sb.Append(script.Replace("{{LABELS}}", string.Join(",", grouped_and_counted.Select(e => "'"+e.Day+"'")))
.Replace("{{DATA}}", string.Join(",", grouped_and_counted.Select(e => e.Count.ToString())))
.Replace("{{DATASET}}", (arg.Action ?? "*") + " - " + (arg.Value ?? "*")));
sb.Append("</script>");
sb.Append("</body></html>");
var response = (Response)sb.ToString();
response.ContentType = "text/html";
return response;
}
public void Log(string l)
{
Utils.Log("Metrics: " + l);
}
}
}

344
Wabbajack.CacheServer/NexusCacheModule.cs
View File

@ -1,344 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Driver;
using Nancy;
using Newtonsoft.Json;
using Wabbajack.CacheServer.DTOs;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.CacheServer
{
public class NexusCacheModule : NancyModule
{
public NexusCacheModule() : base("/")
{
Get("/v1/games/{GameName}/mods/{ModID}/files/{FileID}.json", HandleFileID);
Get("/v1/games/{GameName}/mods/{ModID}/files.json", HandleGetFiles);
Get("/v1/games/{GameName}/mods/{ModID}.json", HandleModInfo);
Get("/nexus_api_cache/{request}.json", HandleCacheCall);
Get("/nexus_api_cache", ListCache);
Get("/nexus_api_cache/update", UpdateCache);
Get("/nexus_api_cache/ingest/{Folder}", HandleIngestCache);
}
class UpdatedMod
{
public long mod_id;
public long latest_file_update;
public long latest_mod_activity;
}
public async Task<object> UpdateCache(object arg)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var gameTasks = GameRegistry.Games.Values
.Where(game => game.NexusName != null)
.Select(async game =>
{
return (game,
mods: await api.Get<List<UpdatedMod>>(
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"));
})
.Select(async rTask =>
{
var (game, mods) = await rTask;
return mods.Select(mod => new { game = game, mod = mod });
}).ToList();
Utils.Log($"Getting update list for {gameTasks.Count} games");
var purge = (await Task.WhenAll(gameTasks))
.SelectMany(i => i)
.ToList();
Utils.Log($"Found {purge.Count} updated mods in the last month");
using (var queue = new WorkQueue())
{
var collected = await purge.Select(d =>
{
var a = d.mod.latest_file_update.AsUnixTime();
// Mod activity could hide files
var b = d.mod.latest_mod_activity.AsUnixTime();
return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.mod_id.ToString()};
}).PMap(queue, async t =>
{
var resultA = await Server.Config.NexusModInfos.Connect().DeleteManyAsync(f =>
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
var resultB = await Server.Config.NexusModFiles.Connect().DeleteManyAsync(f =>
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
var resultC = await Server.Config.NexusFileInfos.Connect().DeleteManyAsync(f =>
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount;
});
Utils.Log($"Purged {collected.Sum()} cache entries");
}
return "Done";
}
private string ListCache(object arg)
{
Utils.Log($"{DateTime.Now} - List Cache");
return String.Join("",
Directory.EnumerateFiles(NexusApiClient.LocalCacheDir)
.Select(f => new FileInfo(f))
.OrderByDescending(fi => fi.LastWriteTime)
.Select(fi =>
{
var decoded = Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(fi.Name).FromHex());
return $"{fi.LastWriteTime} \t {fi.Length.ToFileSizeString()} \t {decoded} \n";
}));
}
private async Task<Response> HandleModInfo(dynamic arg)
{
Utils.Log($"{DateTime.Now} - Mod Info - {arg.GameName}/{arg.ModID}/");
string gameName = arg.GameName;
string modId = arg.ModId;
var result = await Server.Config.NexusModInfos.Connect()
.FindOneAsync(info => info.Game == gameName && info.ModId == modId);
string method = "CACHED";
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"/v1/games/{gameName}/mods/{modId}.json";
var body = await api.Get<ModInfo>(path);
result = new NexusCacheData<ModInfo>
{
Data = body,
Path = path,
Game = gameName,
ModId = modId
};
try
{
await Server.Config.NexusModInfos.Connect().InsertOneAsync(result);
}
catch (MongoWriteException)
{
}
method = "NOT_CACHED";
}
Response response = result.Data.ToJSON();
response.Headers.Add("WABBAJACK_CACHE_FROM", method);
response.ContentType = "application/json";
return response;
}
private async Task<Response> HandleFileID(dynamic arg)
{
Utils.Log($"{DateTime.Now} - File Info - {arg.GameName}/{arg.ModID}/{arg.FileID}");
string gameName = arg.GameName;
string modId = arg.ModId;
string fileId = arg.FileId;
var result = await Server.Config.NexusFileInfos.Connect()
.FindOneAsync(info => info.Game == gameName && info.ModId == modId && info.FileId == fileId);
string method = "CACHED";
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"/v1/games/{gameName}/mods/{modId}/files/{fileId}.json";
var body = await api.Get<NexusFileInfo>(path);
result = new NexusCacheData<NexusFileInfo>
{
Data = body,
Path = path,
Game = gameName,
ModId = modId,
FileId = fileId
};
try
{
await Server.Config.NexusFileInfos.Connect().InsertOneAsync(result);
}
catch (MongoWriteException)
{
}
method = "NOT_CACHED";
}
Response response = result.Data.ToJSON();
response.Headers.Add("WABBAJACK_CACHE_FROM", method);
response.ContentType = "application/json";
return response;
}
private async Task<Response> HandleGetFiles(dynamic arg)
{
Utils.Log($"{DateTime.Now} - Mod Files - {arg.GameName} {arg.ModID}");
string gameName = arg.GameName;
string modId = arg.ModId;
var result = await Server.Config.NexusModFiles.Connect()
.FindOneAsync(info => info.Game == gameName && info.ModId == modId);
string method = "CACHED";
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"/v1/games/{gameName}/mods/{modId}/files.json";
var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
result = new NexusCacheData<NexusApiClient.GetModFilesResponse>
{
Data = body,
Path = path,
Game = gameName,
ModId = modId
};
try
{
await Server.Config.NexusModFiles.Connect().InsertOneAsync(result);
}
catch (MongoWriteException)
{
}
method = "NOT_CACHED";
}
Response response = result.Data.ToJSON();
response.Headers.Add("WABBAJACK_CACHE_FROM", method);
response.ContentType = "application/json";
return response;
}
private async Task<string> HandleCacheCall(dynamic arg)
{
try
{
string param = (string)arg.request;
var url = new Uri(Encoding.UTF8.GetString(param.FromHex()));
var client = new HttpClient();
var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 8080, Scheme = "http"};
client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]);
return await client.GetStringAsync(builder.Uri.ToString());
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
return "ERROR";
}
}
private async Task<string> HandleIngestCache(dynamic arg)
{
int count = 0;
int failed = 0;
using (var queue = new WorkQueue())
{
await Directory.EnumerateFiles(Path.Combine(Server.Config.Settings.TempDir, (string)arg.Folder)).PMap(queue,
async file =>
{
Utils.Log($"Ingesting {file}");
if (!file.EndsWith(".json")) return;
var fileInfo = new FileInfo(file);
count++;
var url = new Url(
Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(file).FromHex()));
var split = url.Path.Split(new[] {'/'}, StringSplitOptions.RemoveEmptyEntries);
try
{
switch (split.Length)
{
case 5 when split[3] == "mods":
{
var body = file.FromJSON<ModInfo>();
var payload = new NexusCacheData<ModInfo>();
payload.Data = body;
payload.Game = split[2];
payload.Path = url.Path;
payload.ModId = body.mod_id;
payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc;
try
{
await Server.Config.NexusModInfos.Connect().InsertOneAsync(payload);
}
catch (MongoWriteException ex)
{
}
break;
}
case 6 when split[5] == "files.json":
{
var body = file.FromJSON<NexusApiClient.GetModFilesResponse>();
var payload = new NexusCacheData<NexusApiClient.GetModFilesResponse>();
payload.Path = url.Path;
payload.Data = body;
payload.Game = split[2];
payload.ModId = split[4];
payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc;
try
{
await Server.Config.NexusModFiles.Connect().InsertOneAsync(payload);
}
catch (MongoWriteException ex)
{
}
break;
}
case 7 when split[5] == "files":
{
var body = file.FromJSON<NexusFileInfo>();
var payload = new NexusCacheData<NexusFileInfo>();
payload.Data = body;
payload.Path = url.Path;
payload.Game = split[2];
payload.FileId = Path.GetFileNameWithoutExtension(split[6]);
payload.ModId = split[4];
payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc;
try
{
await Server.Config.NexusFileInfos.Connect().InsertOneAsync(payload);
}
catch (MongoWriteException ex)
{
}
break;
}
}
}
catch (Exception ex)
{
failed++;
}
});
}
return $"Inserted {count} caches, {failed} failed";
}
}
}

29
Wabbajack.CacheServer/Program.cs
View File

@ -1,29 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Nancy.Hosting.Self;
using Wabbajack.Common;
namespace Wabbajack.CacheServer
{
class Program
{
static void Main(string[] args)
{
Utils.LogMessages.Subscribe(Console.WriteLine);
using (var server = new Server("http://localhost:8080"))
{
Consts.WabbajackCacheHostname = "localhost";
Consts.WabbajackCachePort = 8080;
server.Start();
ListValidationService.Start();
var tsk = JobQueueEndpoints.StartJobQueue();
Console.ReadLine();
}
}
}
}

36
Wabbajack.CacheServer/Properties/AssemblyInfo.cs
View File

@ -1,36 +0,0 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Wabbajack.CacheServer")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("Wabbajack.CacheServer")]
[assembly: AssemblyCopyright("Copyright © 2019")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("bdc9a094-d235-47cd-83ca-44199b60ab20")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

3
Wabbajack.CacheServer/Readme.md
View File

@ -1,3 +0,0 @@
# Wabbajack.CacheServer
CacheServer for caching mod information to reduce the amount of API calls a user has to account for when using Wabbajack to compiler/install a ModList.

79
Wabbajack.CacheServer/Server.cs
View File

@ -1,79 +0,0 @@
using System;
using Alphaleonis.Win32.Filesystem;
using Nancy;
using Nancy.Bootstrapper;
using Nancy.Configuration;
using Nancy.Hosting.Self;
using Nancy.TinyIoc;
using Wabbajack.CacheServer.DTOs;
using Wabbajack.CacheServer.ServerConfig;
using Wabbajack.Common;
namespace Wabbajack.CacheServer
{
public class Server : IDisposable
{
private NancyHost _server;
private HostConfiguration _config;
public static BuildServerConfig Config;
static Server()
{
SerializerSettings.Init();
}
public Server(string address)
{
Address = address;
_config = new HostConfiguration {MaximumConnectionCount = 200, RewriteLocalhost = true};
//_config.UrlReservations.CreateAutomatically = true;
_server = new NancyHost(_config, new Uri(address));
Config = File.ReadAllText("config.yaml").FromYaml<BuildServerConfig>();
}
public string Address { get; }
public void Start()
{
_server.Start();
}
public void Dispose()
{
_server?.Dispose();
}
}
public class CachingBootstrapper : DefaultNancyBootstrapper
{
protected override void ApplicationStartup(TinyIoCContainer container, IPipelines pipelines)
{
pipelines.AfterRequest.AddItemToEndOfPipeline(ctx =>
{
ctx.Response.WithHeader("Access-Control-Allow-Origin", "*")
.WithHeader("Access-Control-Allow-Methods", "POST, GET")
.WithHeader("Access-Control-Allow-Headers", "Accept, Origin, Content-type")
.WithHeader("Cache-Control","no-store");
});
}
public override void Configure(INancyEnvironment environment)
{
environment.Tracing(
enabled: true,
displayErrorTraces: true);
}
protected override void ConfigureApplicationContainer(TinyIoCContainer container)
{
container.Register<Heartbeat>();
container.Register<JobQueueEndpoints>();
container.Register<ListValidationService>();
container.Register<Metrics>();
container.Register<NexusCacheModule>();
container.Register<TestingEndpoints>();
}
}
}

31
Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs
View File

@ -1,31 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Driver.Core.Configuration;
using Wabbajack.CacheServer.DTOs;
using Wabbajack.CacheServer.DTOs.JobQueue;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.CacheServer.ServerConfig
{
public class BuildServerConfig
{
public MongoConfig<Metric> Metrics { get; set; }
public MongoConfig<ModListStatus> ListValidation { get; set; }
public MongoConfig<Job> JobQueue { get; set; }
public MongoConfig<IndexedFile> IndexedFiles { get; set; }
public MongoConfig<DownloadState> DownloadStates { get; set; }
public MongoConfig<NexusCacheData<ModInfo>> NexusModInfos { get; set; }
public MongoConfig<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles { get; set; }
public MongoConfig<NexusCacheData<NexusFileInfo>> NexusFileInfos { get; set; }
public IndexerConfig Indexer { get; set; }
public Settings Settings { get; set; }
}
}

16
Wabbajack.CacheServer/ServerConfig/IndexerConfig.cs
View File

@ -1,16 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.CacheServer.ServerConfig
{
public class IndexerConfig
{
public string DownloadDir { get; set; }
public string TempDir { get; set; }
public string ArchiveDir { get; set; }
}
}

34
Wabbajack.CacheServer/ServerConfig/MongoConfig.cs
View File

@ -1,34 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Driver;
using Wabbajack.CacheServer.DTOs;
namespace Wabbajack.CacheServer.ServerConfig
{
public class MongoConfig<T>
{
public string Host { get; set; }
public string Database { get; set; }
public string Collection { get; set; }
public string Username { get; set; }
public string Password { get; set; }
private IMongoDatabase Client
{
get
{
if (Username != null && Password != null)
return new MongoClient($"mongodb://{Username}:{Password}@{Host}").GetDatabase(Database);
return new MongoClient($"mongodb://{Host}").GetDatabase(Database);
}
}
public IMongoCollection<T> Connect()
{
return Client.GetCollection<T>(Collection);
}
}
}

13
Wabbajack.CacheServer/ServerConfig/Settings.cs
View File

@ -1,13 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.CacheServer.ServerConfig
{
public class Settings
{
public string TempDir { get; set; }
}
}

28
Wabbajack.CacheServer/TestingEndpoints.cs
View File

@ -1,28 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Nancy;
using Nancy.Responses;
namespace Wabbajack.CacheServer
{
/// <summary>
/// These endpoints are used by the testing service to verify that manual and direct
/// downloading works as expected.
/// </summary>
public class TestingEndpoints : NancyModule
{
public TestingEndpoints() : base("/")
{
Get("/WABBAJACK_TEST_FILE.txt", _ => "Cheese for Everyone!");
Get("/WABBAJACK_TEST_FILE.zip", _ =>
{
var response = new StreamResponse(() => new MemoryStream(Encoding.UTF8.GetBytes("Cheese for Everyone!")), "application/zip");
return response.AsAttachment("WABBAJACK_TEST_FILE.zip");
});
}
}
}

169
Wabbajack.CacheServer/Wabbajack.CacheServer.csproj
View File

@ -1,169 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{BDC9A094-D235-47CD-83CA-44199B60AB20}</ProjectGuid>
<OutputType>Exe</OutputType>
<RootNamespace>Wabbajack.CacheServer</RootNamespace>
<AssemblyName>Wabbajack.CacheServer</AssemblyName>
<TargetFrameworkVersion>v4.8</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<Deterministic>true</Deterministic>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>x64</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<NoWarn>CS1998</NoWarn>
<WarningsAsErrors>CS4014</WarningsAsErrors>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<NoWarn>CS1998</NoWarn>
<WarningsAsErrors>CS4014</WarningsAsErrors>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>x64</PlatformTarget>
<LangVersion>7.3</LangVersion>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
<OutputPath>bin\x64\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x64</PlatformTarget>
<LangVersion>7.3</LangVersion>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x86'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x86\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<NoWarn>CS1998</NoWarn>
<DebugType>full</DebugType>
<PlatformTarget>x86</PlatformTarget>
<WarningsAsErrors>CS4014</WarningsAsErrors>
<LangVersion>7.3</LangVersion>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x86'">
<OutputPath>bin\x86\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<NoWarn>CS1998</NoWarn>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x86</PlatformTarget>
<WarningsAsErrors>CS4014</WarningsAsErrors>
<LangVersion>7.3</LangVersion>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Windows" />
<Reference Include="System.Windows.Forms" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
<Reference Include="WindowsBase" />
</ItemGroup>
<ItemGroup>
<Compile Include="DTOs\DownloadState.cs" />
<Compile Include="DTOs\IndexedFile.cs" />
<Compile Include="DTOs\JobQueue\AJobPayload.cs" />
<Compile Include="DTOs\NexusCacheData.cs" />
<Compile Include="Jobs\IndexJob.cs" />
<Compile Include="DTOs\JobQueue\Job.cs" />
<Compile Include="DTOs\JobQueue\JobResult.cs" />
<Compile Include="DTOs\Metric.cs" />
<Compile Include="DTOs\ModListStatus.cs" />
<Compile Include="DTOs\MongoDoc.cs" />
<Compile Include="DTOs\SerializerSettings.cs" />
<Compile Include="Extensions.cs" />
<Compile Include="JobQueueEndpoints.cs" />
<Compile Include="ListValidationService.cs" />
<Compile Include="Metrics.cs" />
<Compile Include="NexusCacheModule.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Server.cs" />
<Compile Include="Heartbeat.cs" />
<Compile Include="ServerConfig\BuildServerConfig.cs" />
<Compile Include="ServerConfig\IndexerConfig.cs" />
<Compile Include="ServerConfig\MongoConfig.cs" />
<Compile Include="TestingEndpoints.cs" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
<None Include="config.yaml">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
<Project>{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}</Project>
<Name>Wabbajack.Common</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.Lib\Wabbajack.Lib.csproj">
<Project>{0a820830-a298-497d-85e0-e9a89efef5fe}</Project>
<Name>Wabbajack.Lib</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.VirtualFileSystem\Wabbajack.VirtualFileSystem.csproj">
<Project>{5d6a2eaf-6604-4c51-8ae2-a746b4bc5e3e}</Project>
<Name>Wabbajack.VirtualFileSystem</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<PackageReference Include="MongoDB.Driver">
<Version>2.10.0</Version>
</PackageReference>
<PackageReference Include="Nancy.Hosting.Self">
<Version>2.0.0</Version>
</PackageReference>
<PackageReference Include="Nettle">
<Version>1.3.0</Version>
</PackageReference>
<PackageReference Include="Newtonsoft.Json">
<Version>12.0.3</Version>
</PackageReference>
<PackageReference Include="ReactiveUI">
<Version>11.1.6</Version>
</PackageReference>
<PackageReference Include="System.Reactive">
<Version>4.3.2</Version>
</PackageReference>
</ItemGroup>
<ItemGroup />
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>

39
Wabbajack.CacheServer/config.yaml
View File

@ -1,39 +0,0 @@
---
Metrics:
Host: internal.test.mongodb
Database: wabbajack
Collection: metrics
ListValidation:
Host: internal.test.mongodb
Database: wabbajack
Collection: mod_lists
JobQueue:
Host: internal.test.mongodb
Database: wabbajack
Collection: job_queue
IndexedFiles:
Host: internal.test.mongodb
Database: wabbajack
Collection: indexed_files
NexusModInfos:
Host: internal.test.mongodb
Database: wabbajack
Collection: nexus_mod_infos
NexusModFiles:
Host: internal.test.mongodb
Database: wabbajack
Collection: nexus_mod_files
NexusFileInfos:
Host: internal.test.mongodb
Database: wabbajack
Collection: nexus_file_infos
DownloadStates:
Host: internal.test.mongodb
Database: wabbajack
Collection: download_states
Indexer:
DownloadDir: c:\tmp\downloads
TempDir: c:\tmp\tmp
ArchiveDir: c:\archives
Settings:
TempDir: c:\tmp\tmp