diff --git a/Wabbajack.CacheServer/DTOs/DownloadState.cs b/Wabbajack.CacheServer/DTOs/DownloadState.cs new file mode 100644 index 00000000..9dc2a02c --- /dev/null +++ b/Wabbajack.CacheServer/DTOs/DownloadState.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson.Serialization.Attributes; +using Wabbajack.Lib.Downloaders; + +namespace Wabbajack.CacheServer.DTOs +{ + public class DownloadState + { + [BsonId] + public string Key { get; set; } + public string Hash { get; set; } + + public AbstractDownloadState State { get; set; } + + public bool IsValid { get; set; } + public DateTime LastValidationTime { get; set; } = DateTime.Now; + public DateTime FirstValidationTime { get; set; } = DateTime.Now; + } +} diff --git a/Wabbajack.CacheServer/DTOs/IndexedFile.cs b/Wabbajack.CacheServer/DTOs/IndexedFile.cs new file mode 100644 index 00000000..5f56926c --- /dev/null +++ b/Wabbajack.CacheServer/DTOs/IndexedFile.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.RightsManagement; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson.Serialization.Attributes; +using Wabbajack.VirtualFileSystem; + +namespace Wabbajack.CacheServer.DTOs +{ + public class IndexedFile + { + [BsonId] + public string Hash { get; set; } + public string SHA256 { get; set; } + public string SHA1 { get; set; } + public string MD5 { get; set; } + public string CRC { get; set; } + public long Size { get; set; } + public bool IsArchive { get; set; } + public List Children { get; set; } = new List(); + } + + public class ChildFile + { + public string Name; + public string Extension; + public string Hash; + } +} diff --git a/Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs b/Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs new file mode 100644 index 00000000..fac78dfa --- /dev/null +++ b/Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson.Serialization.Attributes; +using Wabbajack.CacheServer.Jobs; + +namespace Wabbajack.CacheServer.DTOs.JobQueue +{ + public abstract class AJobPayload + { + public static List KnownSubTypes = new List {typeof(IndexJob)}; + public static Dictionary TypeToName { get; set; } + public static Dictionary NameToType { get; set; } + + + [BsonIgnore] + public abstract string Description { get; } + + public virtual bool UsesNexus { get; } = false; + + public abstract Task Execute(); + + static AJobPayload() + { + NameToType = KnownSubTypes.ToDictionary(t => t.FullName.Substring(t.Namespace.Length + 1), t => t); + TypeToName = NameToType.ToDictionary(k => k.Value, k => k.Key); + } + + } +} diff --git a/Wabbajack.CacheServer/DTOs/JobQueue/Job.cs b/Wabbajack.CacheServer/DTOs/JobQueue/Job.cs new file mode 100644 index 00000000..a859e80c --- /dev/null +++ b/Wabbajack.CacheServer/DTOs/JobQueue/Job.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; + +namespace Wabbajack.CacheServer.DTOs.JobQueue +{ + public class Job + { + public enum JobPriority : int + { + Low, + Normal, + High, + } + + [BsonId] + public Guid Id { get; set; } + public DateTime? Started { get; set; } + public DateTime? Ended { get; set; } + public DateTime Created { get; set; } = DateTime.Now; + public JobPriority Priority { get; set; } = JobPriority.Normal; + + public JobResult Result { get; set; } + public bool RequiresNexus { get; set; } = true; + public AJobPayload Payload { get; set; } + + public static async Task Enqueue(Job job) + { + await Server.Config.JobQueue.Connect().InsertOneAsync(job); + return job.Id; + } + + public static async Task GetNext() + { + var filter = new BsonDocument + { + {"Started", BsonNull.Value} + }; + var update = new BsonDocument + { + {"$set", new BsonDocument {{"Started", DateTime.Now}}} + }; + var sort = new {Priority=-1, Created=1}.ToBsonDocument(); + var job = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync(filter, update, new FindOneAndUpdateOptions{Sort = sort}); + return job; + } + + public static async Task Finish(Job job, JobResult jobResult) + { + var filter = new BsonDocument + { + {"query", new BsonDocument {{"Id", job.Id}}}, + }; + var update = new BsonDocument + { + {"$set", new BsonDocument {{"Ended", DateTime.Now}, {"Result", jobResult.ToBsonDocument()}}} + }; + var result = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync(filter, update); + return result; + } + } +} diff --git a/Wabbajack.CacheServer/DTOs/JobQueue/JobResult.cs b/Wabbajack.CacheServer/DTOs/JobQueue/JobResult.cs new file mode 100644 index 00000000..241908c6 --- /dev/null +++ b/Wabbajack.CacheServer/DTOs/JobQueue/JobResult.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson.Serialization.Attributes; + +namespace Wabbajack.CacheServer.DTOs.JobQueue +{ + public class JobResult + { + public JobResultType ResultType { get; set; } + [BsonIgnoreIfNull] + public string Message { get; set; } + + [BsonIgnoreIfNull] + public string Stacktrace { get; set; } + + public static JobResult Success() + { + return new JobResult { ResultType = JobResultType.Success }; + } + + public static JobResult Error(Exception ex) + { + return new JobResult {ResultType = JobResultType.Error, Stacktrace = ex.ToString()}; + } + + } + + public enum JobResultType + { + Success, + Error + } +} diff --git a/Wabbajack.CacheServer/DTOs/Metric.cs b/Wabbajack.CacheServer/DTOs/Metric.cs new file mode 100644 index 00000000..9b090214 --- /dev/null +++ b/Wabbajack.CacheServer/DTOs/Metric.cs @@ -0,0 +1,16 @@ +using System; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + + +namespace Wabbajack.CacheServer.DTOs +{ + public class Metric + { + [BsonId] + public ObjectId Id; + public DateTime Timestamp; + public string Action; + public string Subject; + } +} diff --git a/Wabbajack.CacheServer/DTOs/ModListStatus.cs b/Wabbajack.CacheServer/DTOs/ModListStatus.cs new file mode 100644 index 00000000..ddf41f43 --- /dev/null +++ b/Wabbajack.CacheServer/DTOs/ModListStatus.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Wabbajack.Lib; +using Wabbajack.Lib.ModListRegistry; + +namespace Wabbajack.CacheServer.DTOs +{ + public class ModListStatus + { + + [BsonId] + public string Id { get; set; } + public ModlistSummary Summary { get; set; } + + public ModlistMetadata Metadata { get; set; } + public DetailedStatus DetailedStatus { get; set; } + + public static async Task Update(ModListStatus status) + { + var id = status.Metadata.Links.MachineURL; + await Server.Config.ListValidation.Connect().FindOneAndReplaceAsync(s => s.Id == id, status, new FindOneAndReplaceOptions {IsUpsert = true}); + } + + public static IQueryable AllSummaries + { + get + { + return null; + } + } + + public static async Task ByName(string name) + { + var result = await Server.Config.ListValidation.Connect() + .AsQueryable() + .Where(doc => doc.Metadata.Links.MachineURL == name || doc.Metadata.Title == name) + .ToListAsync(); + return result.First(); + } + + public static IMongoQueryable All + { + get + { + return Server.Config.ListValidation.Connect().AsQueryable(); + } + } + } + + public class DetailedStatus + { + public string Name { get; set; } + public DateTime Checked { get; set; } = DateTime.Now; + public List Archives { get; set; } + public DownloadMetadata DownloadMetaData { get; set; } + public bool HasFailures { get; set; } + public string MachineName { get; set; } + } + + public class DetailedStatusItem + { + public bool IsFailing { get; set; } + public Archive Archive { get; set; } + } +} diff --git a/Wabbajack.CacheServer/DTOs/MongoDoc.cs b/Wabbajack.CacheServer/DTOs/MongoDoc.cs new file mode 100644 index 00000000..0d91b231 --- /dev/null +++ b/Wabbajack.CacheServer/DTOs/MongoDoc.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson; + +namespace Wabbajack.CacheServer.DTOs +{ + public class MongoDoc + { + public ObjectId _id { get; set; } = ObjectId.Empty; + } +} diff --git a/Wabbajack.CacheServer/DTOs/NexusCacheData.cs b/Wabbajack.CacheServer/DTOs/NexusCacheData.cs new file mode 100644 index 00000000..68096fd4 --- /dev/null +++ b/Wabbajack.CacheServer/DTOs/NexusCacheData.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson.Serialization.Attributes; + +namespace Wabbajack.CacheServer.DTOs +{ + public class NexusCacheData + { + [BsonId] + public string Path { get; set; } + public T Data { get; set; } + public string Game { get; set; } + public string ModId { get; set; } + + public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow; + + [BsonIgnoreIfNull] + public string FileId { get; set; } + + } +} diff --git a/Wabbajack.CacheServer/DTOs/SerializerSettings.cs b/Wabbajack.CacheServer/DTOs/SerializerSettings.cs new file mode 100644 index 00000000..41ecc746 --- /dev/null +++ b/Wabbajack.CacheServer/DTOs/SerializerSettings.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Conventions; +using Wabbajack.CacheServer.DTOs.JobQueue; +using Wabbajack.Lib.Downloaders; + +namespace Wabbajack.CacheServer.DTOs +{ + public static class SerializerSettings + { + public static void Init() + { + var dis = new TypeDiscriminator(typeof(AbstractDownloadState), AbstractDownloadState.NameToType, + AbstractDownloadState.TypeToName); + BsonSerializer.RegisterDiscriminatorConvention(typeof(AbstractDownloadState), dis); + BsonClassMap.RegisterClassMap(cm => cm.SetIsRootClass(true)); + + dis = new TypeDiscriminator(typeof(AJobPayload), AJobPayload.NameToType, AJobPayload.TypeToName); + BsonSerializer.RegisterDiscriminatorConvention(typeof(AJobPayload), dis); + BsonClassMap.RegisterClassMap(cm => cm.SetIsRootClass(true)); + } + } + + + public class TypeDiscriminator : IDiscriminatorConvention + { + private readonly Type defaultType; + private readonly Dictionary typeMap; + private Dictionary revMap; + + public TypeDiscriminator(Type defaultType, + Dictionary typeMap, Dictionary revMap) + { + this.defaultType = defaultType; + this.typeMap = typeMap; + this.revMap = revMap; + } + + + /// + /// Element Name + /// + public string ElementName => "_wjType"; + + public Type GetActualType(IBsonReader bsonReader, Type nominalType) + { + Type type = null; + var bookmark = bsonReader.GetBookmark(); + bsonReader.ReadStartDocument(); + if (bsonReader.FindElement(ElementName)) + { + var value = bsonReader.ReadString(); + if (typeMap.ContainsKey(value)) + type = typeMap[value]; + } + + bsonReader.ReturnToBookmark(bookmark); + if (type == null) + throw new Exception($"Type mis-configuration can't find bson type for ${nominalType}"); + return type; + } + + public BsonValue GetDiscriminator(Type nominalType, Type actualType) + { + return revMap[actualType]; + } + } +} diff --git a/Wabbajack.CacheServer/Extensions.cs b/Wabbajack.CacheServer/Extensions.cs new file mode 100644 index 00000000..c961925f --- /dev/null +++ b/Wabbajack.CacheServer/Extensions.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Driver; +using MongoDB.Driver.Linq; + +namespace Wabbajack.CacheServer +{ + public static class Extensions + { + public static async Task FindOneAsync(this IMongoCollection coll, Expression> expr) + { + return (await coll.AsQueryable().Where(expr).Take(1).ToListAsync()).FirstOrDefault(); + } + } +} diff --git a/Wabbajack.CacheServer/JobQueueEndpoints.cs b/Wabbajack.CacheServer/JobQueueEndpoints.cs new file mode 100644 index 00000000..a6615404 --- /dev/null +++ b/Wabbajack.CacheServer/JobQueueEndpoints.cs @@ -0,0 +1,105 @@ +using System; +using System.Linq; +using System.Linq.Expressions; +using System.Security.Policy; +using System.Threading.Tasks; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Nancy; +using Nettle; +using Wabbajack.CacheServer.DTOs.JobQueue; +using Wabbajack.CacheServer.Jobs; + +namespace Wabbajack.CacheServer +{ + public class JobQueueEndpoints : NancyModule + { + public JobQueueEndpoints() : base ("/jobs") + { + Get("/", HandleListJobs); + Get("/enqueue_curated_for_indexing", HandleEnqueueAllCurated); + } + + private readonly Func HandleListJobsTemplate = NettleEngine.GetCompiler().Compile(@" + + +

Jobs - {{$.jobs.Count}} Pending

+

{{$.time}}

+
    + {{each $.jobs}} +
  1. {{$.Description}}
    2. + {{/each}} +
+ + + + "); + + private async Task HandleListJobs(object arg) + { + var jobs = await Server.Config.JobQueue.Connect() + .AsQueryable() + .Where(j => j.Ended == null) + .OrderByDescending(j => j.Priority) + .ThenBy(j => j.Created) + .ToListAsync(); + + var response = (Response)HandleListJobsTemplate(new {jobs, time = DateTime.Now}); + response.ContentType = "text/html"; + return response; + } + + + private async Task HandleEnqueueAllCurated(object arg) + { + var states = await Server.Config.ListValidation.Connect() + .AsQueryable() + .SelectMany(lst => lst.DetailedStatus.Archives) + .Select(a => a.Archive) + .ToListAsync(); + + var jobs = states.Select(state => new IndexJob {Archive = state}) + .Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus}) + .ToList(); + + if (jobs.Count > 0) + await Server.Config.JobQueue.Connect().InsertManyAsync(jobs); + + return $"Enqueued {states.Count} jobs"; + } + + public static async Task StartJobQueue() + { + foreach (var task in Enumerable.Range(0, 4)) + { + var tsk = StartJobQueueInner(); + } + } + + private static async Task StartJobQueueInner() + { + while (true) + { + try + { + var job = await Job.GetNext(); + if (job == null) + { + await Task.Delay(5000); + continue; + } + + var result = await job.Payload.Execute(); + await Job.Finish(job, result); + } + catch (Exception ex) + { + + } + + } + } + } +} diff --git a/Wabbajack.CacheServer/Jobs/IndexJob.cs b/Wabbajack.CacheServer/Jobs/IndexJob.cs new file mode 100644 index 00000000..c8777dc3 --- /dev/null +++ b/Wabbajack.CacheServer/Jobs/IndexJob.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Wabbajack.CacheServer.DTOs; +using Wabbajack.CacheServer.DTOs.JobQueue; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.VirtualFileSystem; + +namespace Wabbajack.CacheServer.Jobs +{ + public class IndexJob : AJobPayload + { + public Archive Archive { get; set; } + public override string Description { get; } = "Validate and index an archive"; + public override bool UsesNexus { get => Archive.State is NexusDownloader.State; } + public override async Task Execute() + { + + var pk = new List(); + pk.Add(AbstractDownloadState.TypeToName[Archive.State.GetType()]); + pk.AddRange(Archive.State.PrimaryKey); + var pk_str = string.Join("|",pk.Select(p => p.ToString())); + + var found = await Server.Config.DownloadStates.Connect().AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync(); + if (found.Count > 0) + return JobResult.Success(); + + string fileName = Archive.Name; + string folder = Guid.NewGuid().ToString(); + Utils.Log($"Indexer is downloading ${fileName}"); + var downloadDest = Path.Combine(Server.Config.Indexer.DownloadDir, folder, fileName); + await Archive.State.Download(downloadDest); + + using (var queue = new WorkQueue()) + { + var vfs = new Context(queue, true); + await vfs.AddRoot(Path.Combine(Server.Config.Indexer.DownloadDir, folder)); + var archive = vfs.Index.ByRootPath.First(); + var converted = ConvertArchive(new List(), archive.Value); + try + { + await Server.Config.IndexedFiles.Connect().InsertManyAsync(converted, new InsertManyOptions {IsOrdered = false}); + } + catch (MongoBulkWriteException) + { + } + + await Server.Config.DownloadStates.Connect().InsertOneAsync(new DownloadState + { + Key = pk_str, + Hash = archive.Value.Hash, + State = Archive.State, + IsValid = true + }); + + var to_path = Path.Combine(Server.Config.Indexer.ArchiveDir, + $"{Path.GetFileName(fileName)}_{archive.Value.Hash.FromBase64().ToHex()}_{Path.GetExtension(fileName)}"); + if (File.Exists(to_path)) + File.Delete(downloadDest); + else + File.Move(downloadDest, to_path); + Utils.DeleteDirectory(Path.Combine(Server.Config.Indexer.DownloadDir, folder)); + } + + return JobResult.Success(); + } + + private List ConvertArchive(List files, VirtualFile file, bool isTop = true) + { + var name = isTop ? Path.GetFileName(file.Name) : file.Name; + var ifile = new IndexedFile + { + Hash = file.Hash, + SHA256 = file.ExtendedHashes.SHA256, + SHA1 = file.ExtendedHashes.SHA1, + MD5 = file.ExtendedHashes.MD5, + CRC = file.ExtendedHashes.CRC, + Size = file.Size, + Children = file.Children != null ? file.Children.Select( + f => + { + ConvertArchive(files, f, false); + + return new ChildFile + { + Hash = f.Hash, + Name = f.Name.ToLowerInvariant(), + Extension = Path.GetExtension(f.Name.ToLowerInvariant()) + }; + }).ToList() : new List() + }; + ifile.IsArchive = ifile.Children.Count > 0; + files.Add(ifile); + return files; + } + + + } +} diff --git a/Wabbajack.CacheServer/ListValidationService.cs b/Wabbajack.CacheServer/ListValidationService.cs index bc27b784..8fb58a7e 100644 --- a/Wabbajack.CacheServer/ListValidationService.cs +++ b/Wabbajack.CacheServer/ListValidationService.cs @@ -1,52 +1,41 @@ using System; -using System.Collections; using System.Collections.Generic; -using System.Collections.Immutable; using System.Linq; using System.Net.Http; -using System.Text; -using System.Threading; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; +using MongoDB.Driver; using Nancy; -using Nancy.Responses; +using Wabbajack.CacheServer.DTOs; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.ModListRegistry; +using MongoDB.Driver.Linq; +using Nettle; namespace Wabbajack.CacheServer { public class ListValidationService : NancyModule { - public class ModListStatus - { - public string Name; - public DateTime Checked = DateTime.Now; - public List<(Archive archive, bool)> Archives { get; set; } - public DownloadMetadata DownloadMetaData { get; set; } - public bool HasFailures { get; set; } - public string MachineName { get; set; } - } - - public static Dictionary ModLists { get; set; } - public ListValidationService() : base("/lists") { Get("/status", HandleGetLists); + Get("/force_recheck", HandleForceRecheck); Get("/status/{Name}.json", HandleGetListJson); Get("/status/{Name}.html", HandleGetListHtml); + } - private object HandleGetLists(object arg) + private async Task HandleForceRecheck(object arg) { - var summaries = ModLists.Values.Select(m => new ModlistSummary - { - Name = m.Name, - Checked = m.Checked, - Failed = m.Archives.Count(a => a.Item2), - Passed = m.Archives.Count(a => !a.Item2), - }).ToList(); + await ValidateLists(false); + return "done"; + } + + private async Task HandleGetLists(object arg) + { + var summaries = await ModListStatus.All.Select(m => m.Summary).ToListAsync(); return summaries.ToJSON(); } @@ -63,49 +52,42 @@ namespace Wabbajack.CacheServer public List Passed; } - private object HandleGetListJson(dynamic arg) + private async Task HandleGetListJson(dynamic arg) { - var lst = ModLists[(string)arg.Name]; - var summary = new DetailedSummary - { - Name = lst.Name, - Checked = lst.Checked, - Failed = lst.Archives.Where(a => a.Item2) - .Select(a => new ArchiveSummary {Name = a.archive.Name, State = a.archive.State}).ToList(), - Passed = lst.Archives.Where(a => !a.Item2) - .Select(a => new ArchiveSummary { Name = a.archive.Name, State = a.archive.State }).ToList(), - }; - return summary.ToJSON(); + var metric = Metrics.Log("list_validation.get_list_json", (string)arg.Name); + var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus; + return lst.ToJSON(); } - private object HandleGetListHtml(dynamic arg) + + private static readonly Func HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@" + +

{{lst.Name}} - {{lst.Checked}}

+

Failed ({{failed.Count}}):

+
    + {{each $.failed }} +
  • {{$.Archive.Name}}
    • + {{/each}} +
+

Passed ({{passed.Count}}):

+
    + {{each $.passed }} +
  • {{$.Archive.Name}}
    • + {{/each}} +
+ + "); + + private async Task HandleGetListHtml(dynamic arg) { - var lst = ModLists[(string)arg.Name]; - var sb = new StringBuilder(); - - sb.Append(""); - sb.Append($"

{lst.Name} - {lst.Checked}

"); - - var failed_list = lst.Archives.Where(a => a.Item2).ToList(); - sb.Append($"

Failed ({failed_list.Count}):

"); - sb.Append("
    "); - foreach (var archive in failed_list) + + var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus; + var response = (Response)HandleGetListTemplate(new { - sb.Append($"
  • {archive.archive.Name}
    • "); - } - sb.Append("
"); - - var pased_list = lst.Archives.Where(a => !a.Item2).ToList(); - sb.Append($"

Passed ({pased_list.Count}):

"); - sb.Append("
    "); - foreach (var archive in pased_list.OrderBy(f => f.archive.Name)) - { - sb.Append($"
  • {archive.archive.Name}
    • "); - } - sb.Append("
"); - - sb.Append(""); - var response = (Response)sb.ToString(); + lst, + failed = lst.Archives.Where(a => a.IsFailing).ToList(), + passed = lst.Archives.Where(a => !a.IsFailing).ToList() + }); response.ContentType = "text/html"; return response; } @@ -130,79 +112,104 @@ namespace Wabbajack.CacheServer } }).FireAndForget(); } - public static async Task ValidateLists() + public static async Task ValidateLists(bool skipIfNewer = true) { Utils.Log("Cleaning Nexus Cache"); var client = new HttpClient(); - await client.GetAsync("http://build.wabbajack.org/nexus_api_cache/update"); + //await client.GetAsync("http://build.wabbajack.org/nexus_api_cache/update"); Utils.Log("Starting Modlist Validation"); var modlists = await ModlistMetadata.LoadFromGithub(); - var statuses = new Dictionary(); - using (var queue = new WorkQueue()) { foreach (var list in modlists) { - var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ExtensionManager.Extension); - - if (list.NeedsDownload(modlist_path)) + try { - if (File.Exists(modlist_path)) - File.Delete(modlist_path); - - var state = DownloadDispatcher.ResolveArchive(list.Links.Download); - Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}"); - await state.Download(modlist_path); + await ValidateList(list, queue, skipIfNewer); } - else + catch (Exception ex) { - Utils.Log($"No changes detected from downloaded modlist"); } - - - Utils.Log($"Loading {modlist_path}"); - - var installer = AInstaller.LoadFromFile(modlist_path); - - Utils.Log($"{installer.Archives.Count} archives to validate"); - - DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State)); - - var validated = (await installer.Archives - .PMap(queue, async archive => - { - Utils.Log($"Validating: {archive.Name}"); - bool is_failed; - try - { - is_failed = !(await archive.State.Verify()); - } - catch (Exception) - { - is_failed = false; - } - - return (archive, is_failed); - })) - .ToList(); - - - var status = new ModListStatus - { - Name = list.Title, - Archives = validated.OrderBy(v => v.archive.Name).ToList(), - DownloadMetaData = list.DownloadMetadata, - HasFailures = validated.Any(v => v.is_failed) - }; - - statuses.Add(status.Name, status); - } + } } - Utils.Log($"Done validating {statuses.Count} lists"); - ModLists = statuses; + Utils.Log($"Done validating {modlists.Count} lists"); + } + + private static async Task ValidateList(ModlistMetadata list, WorkQueue queue, bool skipIfNewer = true) + { + var existing = await Server.Config.ListValidation.Connect().FindOneAsync(l => l.Id == list.Links.MachineURL); + if (skipIfNewer && existing != null && DateTime.Now - existing.DetailedStatus.Checked < TimeSpan.FromHours(2)) + return; + + var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ExtensionManager.Extension); + + if (list.NeedsDownload(modlist_path)) + { + if (File.Exists(modlist_path)) + File.Delete(modlist_path); + + var state = DownloadDispatcher.ResolveArchive(list.Links.Download); + Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}"); + await state.Download(modlist_path); + } + else + { + Utils.Log($"No changes detected from downloaded modlist"); + } + + + Utils.Log($"Loading {modlist_path}"); + + var installer = AInstaller.LoadFromFile(modlist_path); + + Utils.Log($"{installer.Archives.Count} archives to validate"); + + DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State)); + + var validated = (await installer.Archives + .PMap(queue, async archive => + { + Utils.Log($"Validating: {archive.Name}"); + bool is_failed; + try + { + is_failed = !(await archive.State.Verify()); + } + catch (Exception) + { + is_failed = false; + } + + return new DetailedStatusItem {IsFailing = is_failed, Archive = archive}; + })) + .ToList(); + + + var status = new DetailedStatus + { + Name = list.Title, + Archives = validated.OrderBy(v => v.Archive.Name).ToList(), + DownloadMetaData = list.DownloadMetadata, + HasFailures = validated.Any(v => v.IsFailing) + }; + + var dto = new ModListStatus + { + Id = list.Links.MachineURL, + Summary = new ModlistSummary + { + Name = status.Name, + Checked = status.Checked, + Failed = status.Archives.Count(a => a.IsFailing), + Passed = status.Archives.Count(a => !a.IsFailing), + }, + DetailedStatus = status, + Metadata = list + }; + await ModListStatus.Update(dto); } } } diff --git a/Wabbajack.CacheServer/Metrics.cs b/Wabbajack.CacheServer/Metrics.cs index 140681d0..12b4a799 100644 --- a/Wabbajack.CacheServer/Metrics.cs +++ b/Wabbajack.CacheServer/Metrics.cs @@ -1,13 +1,13 @@ using System; -using System.Collections.Generic; using System.Linq; -using System.Net; using System.Text; using System.Threading; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; +using MongoDB.Driver; +using MongoDB.Driver.Linq; using Nancy; -using ReactiveUI; +using Wabbajack.CacheServer.DTOs; using Wabbajack.Common; namespace Wabbajack.CacheServer @@ -19,19 +19,17 @@ namespace Wabbajack.CacheServer { private static SemaphoreSlim _lockObject = new SemaphoreSlim(1); - public static async Task Log(params object[] args) + public static async Task Log(DateTime timestamp, string action, string subject) { - var msg = new[] {string.Join("\t", args.Select(a => a.ToString()))}; + var msg = new[] {string.Join("\t", new[]{timestamp.ToString(), action, subject})}; Utils.Log(msg.First()); - await _lockObject.WaitAsync(); - try - { - File.AppendAllLines("stats.tsv", msg); - } - finally - { - _lockObject.Release(); - } + var db = Server.Config.Metrics.Connect(); + await db.InsertOneAsync(new Metric {Timestamp = timestamp, Action = action, Subject = subject}); + } + + public static Task Log(string action, string subject) + { + return Log(DateTime.Now, action, subject); } public Metrics() : base("/") @@ -40,6 +38,26 @@ namespace Wabbajack.CacheServer Get("/metrics/chart/", HandleChart); Get("/metrics/chart/{Action}/", HandleChart); Get("/metrics/chart/{Action}/{Value}/", HandleChart); + Get("/metrics/ingest/{filename}", HandleBulkIngest); + } + + private async Task HandleBulkIngest(dynamic arg) + { + Log("Bulk Loading " + arg.filename.ToString()); + + var lines = File.ReadAllLines(Path.Combine(@"c:\tmp", (string)arg.filename)); + + var db = Server.Config.Metrics.Connect(); + + var data = lines.Select(line => line.Split('\t')) + .Where(line => line.Length == 3) + .Select(line => new Metric{ Timestamp = DateTime.Parse(line[0]), Action = line[1], Subject = line[2] }) + .ToList(); + + foreach (var metric in data) + await db.InsertOneAsync(metric); + + return $"Processed {lines.Length} records"; } private async Task HandleMetrics(dynamic arg) @@ -49,36 +67,33 @@ namespace Wabbajack.CacheServer return date.ToString(); } - private static async Task GetData() - { - await _lockObject.WaitAsync(); - try - { - return File.ReadAllLines("stats.tsv"); - } - finally - { - _lockObject.Release(); - } - } - private async Task HandleChart(dynamic arg) { - var data = (await GetData()).Select(line => line.Split('\t')) + /*var data = (await GetData()).Select(line => line.Split('\t')) .Where(line => line.Length == 3) - .Select(line => new {date = DateTime.Parse(line[0]), Action = line[1], Value = line[2]}); + .Select(line => new {date = DateTime.Parse(line[0]), Action = line[1], Value = line[2]});*/ + + var q = Server.Config.Metrics.Connect().AsQueryable(); // Remove guids / Default, which come from testing - data = data.Where(d => !Guid.TryParse(d.Value ?? "", out _) && (d.Value ?? "") != "Default"); if (arg?.Action != null) - data = data.Where(d => d.Action == arg.Action); + { + var action = (string)arg.Action; + q = q.Where(d => d.Action == action); + } if (arg?.Value != null) - data = data.Where(d => d.Value.StartsWith(arg.Value)); + { + var value = (string)arg.Value; + q = q.Where(d => d.Subject.StartsWith(value)); + } - var grouped_and_counted = data.GroupBy(d => d.date.ToString("yyyy-MM-dd")) + var data = (await q.Take(Int32.MaxValue).ToListAsync()).AsEnumerable(); + data = data.Where(d => !Guid.TryParse(d.Subject ?? "", out Guid v) && (d.Subject ?? "") != "Default"); + + var grouped_and_counted = data.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd")) .OrderBy(d => d.Key) .Select(d => new {Day = d.Key, Count = d.Count()}) .ToList(); @@ -116,5 +131,10 @@ namespace Wabbajack.CacheServer response.ContentType = "text/html"; return response; } + + public void Log(string l) + { + Utils.Log("Metrics: " + l); + } } } diff --git a/Wabbajack.CacheServer/NexusCacheModule.cs b/Wabbajack.CacheServer/NexusCacheModule.cs index 7e001ace..a334bb9c 100644 --- a/Wabbajack.CacheServer/NexusCacheModule.cs +++ b/Wabbajack.CacheServer/NexusCacheModule.cs @@ -3,10 +3,13 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; +using System.Reflection; using System.Text; using System.Threading.Tasks; +using MongoDB.Driver; using Nancy; -using Nancy.Helpers; +using Newtonsoft.Json; +using Wabbajack.CacheServer.DTOs; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.NexusApi; @@ -24,12 +27,65 @@ namespace Wabbajack.CacheServer Get("/nexus_api_cache/{request}.json", HandleCacheCall); Get("/nexus_api_cache", ListCache); Get("/nexus_api_cache/update", UpdateCache); + Get("/nexus_api_cache/ingest/{Folder}", HandleIngestCache); + } + + class UpdatedMod + { + public long mod_id; + public long latest_file_update; + public long latest_mod_activity; } public async Task UpdateCache(object arg) { var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - await api.ClearUpdatedModsInCache(); + + var gameTasks = GameRegistry.Games.Values + .Where(game => game.NexusName != null) + .Select(async game => + { + return (game, + mods: await api.Get>( + $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m")); + }) + .Select(async rTask => + { + var (game, mods) = await rTask; + return mods.Select(mod => new { game = game, mod = mod }); + }).ToList(); + + Utils.Log($"Getting update list for {gameTasks.Count} games"); + + var purge = (await Task.WhenAll(gameTasks)) + .SelectMany(i => i) + .ToList(); + + Utils.Log($"Found {purge.Count} updated mods in the last month"); + using (var queue = new WorkQueue()) + { + var collected = await purge.Select(d => + { + var a = d.mod.latest_file_update.AsUnixTime(); + // Mod activity could hide files + var b = d.mod.latest_mod_activity.AsUnixTime(); + + return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.mod_id.ToString()}; + }).PMap(queue, async t => + { + var resultA = await Server.Config.NexusModInfos.Connect().DeleteManyAsync(f => + f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); + var resultB = await Server.Config.NexusModFiles.Connect().DeleteManyAsync(f => + f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); + var resultC = await Server.Config.NexusFileInfos.Connect().DeleteManyAsync(f => + f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); + + return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount; + }); + + Utils.Log($"Purged {collected.Sum()} cache entries"); + } + return "Done"; } @@ -47,30 +103,123 @@ namespace Wabbajack.CacheServer })); } - private async Task HandleModInfo(dynamic arg) + private async Task HandleModInfo(dynamic arg) { Utils.Log($"{DateTime.Now} - Mod Info - {arg.GameName}/{arg.ModID}/"); - var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - return api.GetModInfo(GameRegistry.GetByNexusName((string)arg.GameName).Game, (string)arg.ModID).ToJSON(); + string gameName = arg.GameName; + string modId = arg.ModId; + var result = await Server.Config.NexusModInfos.Connect() + .FindOneAsync(info => info.Game == gameName && info.ModId == modId); + + string method = "CACHED"; + if (result == null) + { + var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); + var path = $"/v1/games/{gameName}/mods/{modId}.json"; + var body = await api.Get(path); + result = new NexusCacheData + { + Data = body, + Path = path, + Game = gameName, + ModId = modId + }; + try + { + await Server.Config.NexusModInfos.Connect().InsertOneAsync(result); + } + catch (MongoWriteException) + { + + } + + method = "NOT_CACHED"; + } + + Response response = result.Data.ToJSON(); + response.Headers.Add("WABBAJACK_CACHE_FROM", method); + response.ContentType = "application/json"; + return response; } - private async Task HandleFileID(dynamic arg) + private async Task HandleFileID(dynamic arg) { Utils.Log($"{DateTime.Now} - File Info - {arg.GameName}/{arg.ModID}/{arg.FileID}"); - var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - return api.GetFileInfo(new NexusDownloader.State + string gameName = arg.GameName; + string modId = arg.ModId; + string fileId = arg.FileId; + var result = await Server.Config.NexusFileInfos.Connect() + .FindOneAsync(info => info.Game == gameName && info.ModId == modId && info.FileId == fileId); + + string method = "CACHED"; + if (result == null) { - GameName = arg.GameName, - ModID = arg.ModID, - FileID = arg.FileID - }).ToJSON(); + var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); + var path = $"/v1/games/{gameName}/mods/{modId}/files/{fileId}.json"; + var body = await api.Get(path); + result = new NexusCacheData + { + Data = body, + Path = path, + Game = gameName, + ModId = modId, + FileId = fileId + }; + try + { + await Server.Config.NexusFileInfos.Connect().InsertOneAsync(result); + } + catch (MongoWriteException) + { + + } + + method = "NOT_CACHED"; + } + + Response response = result.Data.ToJSON(); + response.Headers.Add("WABBAJACK_CACHE_FROM", method); + response.ContentType = "application/json"; + return response; } - private async Task HandleGetFiles(dynamic arg) + private async Task HandleGetFiles(dynamic arg) { Utils.Log($"{DateTime.Now} - Mod Files - {arg.GameName} {arg.ModID}"); - var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - return api.GetModFiles(GameRegistry.GetByNexusName((string)arg.GameName).Game, (int)arg.ModID).ToJSON(); + string gameName = arg.GameName; + string modId = arg.ModId; + var result = await Server.Config.NexusModFiles.Connect() + .FindOneAsync(info => info.Game == gameName && info.ModId == modId); + + string method = "CACHED"; + if (result == null) + { + var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); + var path = $"/v1/games/{gameName}/mods/{modId}/files.json"; + var body = await api.Get(path); + result = new NexusCacheData + { + Data = body, + Path = path, + Game = gameName, + ModId = modId + }; + try + { + await Server.Config.NexusModFiles.Connect().InsertOneAsync(result); + } + catch (MongoWriteException) + { + + } + + method = "NOT_CACHED"; + } + + Response response = result.Data.ToJSON(); + response.Headers.Add("WABBAJACK_CACHE_FROM", method); + response.ContentType = "application/json"; + return response; } private async Task HandleCacheCall(dynamic arg) @@ -79,27 +228,11 @@ namespace Wabbajack.CacheServer { string param = (string)arg.request; var url = new Uri(Encoding.UTF8.GetString(param.FromHex())); - var path = Path.Combine(NexusApiClient.LocalCacheDir, arg.request + ".json"); - if (!File.Exists(path)) - { - Utils.Log($"{DateTime.Now} - Not Cached - {url}"); - var client = new HttpClient(); - var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 8080, Scheme = "http"}; - client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]); - await client.GetStringAsync(builder.Uri.ToString()); - if (!File.Exists(path)) - { - Utils.Log($"Still not cached : {path}"); - throw new InvalidDataException("Invalid Data"); - } - - Utils.Log($"Is Now Cached : {path}"); - - } - - Utils.Log($"{DateTime.Now} - From Cached - {url}"); - return File.ReadAllText(path); + var client = new HttpClient(); + var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 8080, Scheme = "http"}; + client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]); + return await client.GetStringAsync(builder.Uri.ToString()); } catch (Exception ex) { @@ -107,5 +240,105 @@ namespace Wabbajack.CacheServer return "ERROR"; } } + + private async Task HandleIngestCache(dynamic arg) + { + int count = 0; + int failed = 0; + + using (var queue = new WorkQueue()) + { + await Directory.EnumerateFiles(Path.Combine(Server.Config.Settings.TempDir, (string)arg.Folder)).PMap(queue, + async file => + { + Utils.Log($"Ingesting {file}"); + if (!file.EndsWith(".json")) return; + + var fileInfo = new FileInfo(file); + count++; + + var url = new Url( + Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(file).FromHex())); + var split = url.Path.Split(new[] {'/'}, StringSplitOptions.RemoveEmptyEntries); + try + { + switch (split.Length) + { + case 5 when split[3] == "mods": + { + var body = file.FromJSON(); + + var payload = new NexusCacheData(); + payload.Data = body; + payload.Game = split[2]; + payload.Path = url.Path; + payload.ModId = body.mod_id; + payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc; + + try + { + await Server.Config.NexusModInfos.Connect().InsertOneAsync(payload); + } + catch (MongoWriteException ex) + { + + } + + break; + } + case 6 when split[5] == "files.json": + { + var body = file.FromJSON(); + var payload = new NexusCacheData(); + payload.Path = url.Path; + payload.Data = body; + payload.Game = split[2]; + payload.ModId = split[4]; + payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc; + + try + { + await Server.Config.NexusModFiles.Connect().InsertOneAsync(payload); + } + catch (MongoWriteException ex) + { + + } + + break; + } + case 7 when split[5] == "files": + { + var body = file.FromJSON(); + var payload = new NexusCacheData(); + payload.Data = body; + payload.Path = url.Path; + payload.Game = split[2]; + payload.FileId = Path.GetFileNameWithoutExtension(split[6]); + payload.ModId = split[4]; + payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc; + + try + { + await Server.Config.NexusFileInfos.Connect().InsertOneAsync(payload); + } + catch (MongoWriteException ex) + { + + } + + break; + } + } + } + catch (Exception ex) + { + failed++; + } + }); + } + + return $"Inserted {count} caches, {failed} failed"; + } } } diff --git a/Wabbajack.CacheServer/Program.cs b/Wabbajack.CacheServer/Program.cs index 5e0b4930..049c7ada 100644 --- a/Wabbajack.CacheServer/Program.cs +++ b/Wabbajack.CacheServer/Program.cs @@ -15,8 +15,13 @@ namespace Wabbajack.CacheServer Utils.LogMessages.Subscribe(Console.WriteLine); using (var server = new Server("http://localhost:8080")) { - ListValidationService.Start(); + Consts.WabbajackCacheHostname = "localhost"; + Consts.WabbajackCachePort = 8080; server.Start(); + + ListValidationService.Start(); + var tsk = JobQueueEndpoints.StartJobQueue(); + Console.ReadLine(); } } diff --git a/Wabbajack.CacheServer/Server.cs b/Wabbajack.CacheServer/Server.cs index 76d7867c..edf64462 100644 --- a/Wabbajack.CacheServer/Server.cs +++ b/Wabbajack.CacheServer/Server.cs @@ -1,17 +1,13 @@ using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Net; -using System.Net.Sockets; -using System.Text; -using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; using Nancy; using Nancy.Bootstrapper; using Nancy.Configuration; -using Nancy.Diagnostics; using Nancy.Hosting.Self; using Nancy.TinyIoc; +using Wabbajack.CacheServer.DTOs; +using Wabbajack.CacheServer.ServerConfig; +using Wabbajack.Common; namespace Wabbajack.CacheServer { @@ -19,15 +15,22 @@ namespace Wabbajack.CacheServer { private NancyHost _server; private HostConfiguration _config; + public static BuildServerConfig Config; + + static Server() + { + SerializerSettings.Init(); + } + public Server(string address) { Address = address; - _config = new HostConfiguration {MaximumConnectionCount = 24, RewriteLocalhost = true}; + _config = new HostConfiguration {MaximumConnectionCount = 200, RewriteLocalhost = true}; //_config.UrlReservations.CreateAutomatically = true; _server = new NancyHost(_config, new Uri(address)); - - + + Config = File.ReadAllText("config.yaml").FromYaml(); } public string Address { get; } diff --git a/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs b/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs new file mode 100644 index 00000000..10f6cebd --- /dev/null +++ b/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Driver.Core.Configuration; +using Wabbajack.CacheServer.DTOs; +using Wabbajack.CacheServer.DTOs.JobQueue; +using Wabbajack.Lib.NexusApi; + +namespace Wabbajack.CacheServer.ServerConfig +{ + public class BuildServerConfig + { + public MongoConfig Metrics { get; set; } + public MongoConfig ListValidation { get; set; } + + public MongoConfig JobQueue { get; set; } + + public MongoConfig IndexedFiles { get; set; } + public MongoConfig DownloadStates { get; set; } + + public MongoConfig> NexusModInfos { get; set; } + public MongoConfig> NexusModFiles { get; set; } + public MongoConfig> NexusFileInfos { get; set; } + + public IndexerConfig Indexer { get; set; } + + public Settings Settings { get; set; } + } +} diff --git a/Wabbajack.CacheServer/ServerConfig/IndexerConfig.cs b/Wabbajack.CacheServer/ServerConfig/IndexerConfig.cs new file mode 100644 index 00000000..e84208de --- /dev/null +++ b/Wabbajack.CacheServer/ServerConfig/IndexerConfig.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Wabbajack.CacheServer.ServerConfig +{ + public class IndexerConfig + { + public string DownloadDir { get; set; } + public string TempDir { get; set; } + + public string ArchiveDir { get; set; } + } +} diff --git a/Wabbajack.CacheServer/ServerConfig/MongoConfig.cs b/Wabbajack.CacheServer/ServerConfig/MongoConfig.cs new file mode 100644 index 00000000..519b2684 --- /dev/null +++ b/Wabbajack.CacheServer/ServerConfig/MongoConfig.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Driver; +using Wabbajack.CacheServer.DTOs; + +namespace Wabbajack.CacheServer.ServerConfig +{ + public class MongoConfig + { + public string Host { get; set; } + public string Database { get; set; } + public string Collection { get; set; } + public string Username { get; set; } + public string Password { get; set; } + + private IMongoDatabase Client + { + get + { + if (Username != null && Password != null) + return new MongoClient($"mongodb://{Username}:{Password}@{Host}").GetDatabase(Database); + return new MongoClient($"mongodb://{Host}").GetDatabase(Database); + } + } + + public IMongoCollection Connect() + { + return Client.GetCollection(Collection); + } + } +} diff --git a/Wabbajack.CacheServer/ServerConfig/Settings.cs b/Wabbajack.CacheServer/ServerConfig/Settings.cs new file mode 100644 index 00000000..26ccfe0b --- /dev/null +++ b/Wabbajack.CacheServer/ServerConfig/Settings.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Wabbajack.CacheServer.ServerConfig +{ + public class Settings + { + public string TempDir { get; set; } + } +} diff --git a/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj b/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj index 32e91d3a..48111bc2 100644 --- a/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj +++ b/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj @@ -73,6 +73,19 @@ + + + + + + + + + + + + + @@ -80,10 +93,17 @@ + + + + + + PreserveNewest + @@ -94,11 +114,21 @@ {0a820830-a298-497d-85e0-e9a89efef5fe} Wabbajack.Lib + + {5d6a2eaf-6604-4c51-8ae2-a746b4bc5e3e} + Wabbajack.VirtualFileSystem + + + 2.10.0 + 2.0.0 + + 1.3.0 + 12.0.3 @@ -106,8 +136,9 @@ 11.0.6 - 4.3.1 + 4.3.2 + \ No newline at end of file diff --git a/Wabbajack.CacheServer/config.yaml b/Wabbajack.CacheServer/config.yaml new file mode 100644 index 00000000..95b7811e --- /dev/null +++ b/Wabbajack.CacheServer/config.yaml @@ -0,0 +1,39 @@ +--- +Metrics: + Host: internal.test.mongodb + Database: wabbajack + Collection: metrics +ListValidation: + Host: internal.test.mongodb + Database: wabbajack + Collection: mod_lists +JobQueue: + Host: internal.test.mongodb + Database: wabbajack + Collection: job_queue +IndexedFiles: + Host: internal.test.mongodb + Database: wabbajack + Collection: indexed_files +NexusModInfos: + Host: internal.test.mongodb + Database: wabbajack + Collection: nexus_mod_infos +NexusModFiles: + Host: internal.test.mongodb + Database: wabbajack + Collection: nexus_mod_files +NexusFileInfos: + Host: internal.test.mongodb + Database: wabbajack + Collection: nexus_file_infos +DownloadStates: + Host: internal.test.mongodb + Database: wabbajack + Collection: download_states +Indexer: + DownloadDir: c:\tmp\downloads + TempDir: c:\tmp\tmp + ArchiveDir: c:\archives +Settings: + TempDir: c:\tmp\tmp \ No newline at end of file diff --git a/Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj b/Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj index 19a4e8d6..9aa98d40 100644 --- a/Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj +++ b/Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj @@ -4,7 +4,7 @@ netstandard2.0 - + diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index c95ee467..b2f4a121 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -92,5 +92,8 @@ namespace Wabbajack.Common public static string LocalAppDataPath => Path.Combine(KnownFolders.LocalAppData.Path, "Wabbajack"); public static string WabbajackCacheLocation = "http://build.wabbajack.org/nexus_api_cache/"; + + public static string WabbajackCacheHostname = "build.wabbajack.org"; + public static int WabbajackCachePort = 80; } } diff --git a/Wabbajack.Common/SteamHandler.cs b/Wabbajack.Common/SteamHandler.cs index 9fe0c695..86c60e31 100644 --- a/Wabbajack.Common/SteamHandler.cs +++ b/Wabbajack.Common/SteamHandler.cs @@ -65,6 +65,8 @@ namespace Wabbajack.Common { var steamKey = Registry.CurrentUser.OpenSubKey(SteamRegKey); SteamPath = steamKey?.GetValue("SteamPath").ToString(); + if(string.IsNullOrWhiteSpace(SteamPath) || steamKey == null || !Directory.Exists(SteamPath)) + Utils.ErrorThrow(new Exception("Could not find the Steam folder!")); if(!init) return; LoadInstallFolders(); LoadAllSteamGames(); @@ -92,10 +94,15 @@ namespace Wabbajack.Common if (!l.Contains("BaseInstallFolder_")) return; var s = GetVdfValue(l); s = Path.Combine(s, "steamapps"); - if(Directory.Exists(s)) - paths.Add(s); + if (!Directory.Exists(s)) + return; + + paths.Add(s); + Utils.Log($"Steam Library found at {s}"); }); + Utils.Log($"Total number of Steam Libraries found: {paths.Count}"); + // Default path in the Steam folder isn't in the configs if(Directory.Exists(Path.Combine(SteamPath, "steamapps"))) paths.Add(Path.Combine(SteamPath, "steamapps")); @@ -145,9 +152,13 @@ namespace Wabbajack.Common g.RequiredFiles.TrueForAll(s => File.Exists(Path.Combine(steamGame.InstallDir, s))) )?.Game; games.Add(steamGame); + + Utils.Log($"Found Game: {steamGame.Name} ({steamGame.AppId}) at {steamGame.InstallDir}"); }); }); + Utils.Log($"Total number of Steam Games found: {games.Count}"); + Games = games; } diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 4ab1f37d..994b6f31 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -95,10 +95,8 @@ namespace Wabbajack.Lib ModList.Readme = $"readme{readme.Extension}"; } - // DISABLED FOR THIS RELEASE //ModList.ReadmeIsWebsite = ReadmeIsWebsite; - //ModList.ToJSON(Path.Combine(ModListOutputFolder, "modlist.json")); ModList.ToCERAS(Path.Combine(ModListOutputFolder, "modlist"), CerasConfig.Config); if (File.Exists(ModListOutputFile)) diff --git a/Wabbajack.Lib/CerasConfig.cs b/Wabbajack.Lib/CerasConfig.cs index ac82ccda..05d16e4c 100644 --- a/Wabbajack.Lib/CerasConfig.cs +++ b/Wabbajack.Lib/CerasConfig.cs @@ -32,7 +32,6 @@ namespace Wabbajack.Lib }, }; - // DISABLED FOR THIS RELEASE //Config.VersionTolerance.Mode = VersionToleranceMode.Standard; } } diff --git a/Wabbajack.Lib/Data.cs b/Wabbajack.Lib/Data.cs index adf76b3d..df2ae470 100644 --- a/Wabbajack.Lib/Data.cs +++ b/Wabbajack.Lib/Data.cs @@ -119,7 +119,6 @@ namespace Wabbajack.Lib /// /// Whether readme is a website /// - /// DISABLED FOR THIS RELEASE //public bool ReadmeIsWebsite; } @@ -246,19 +245,19 @@ namespace Wabbajack.Lib /// /// MurMur3 Hash of the archive /// - public string Hash; + public string Hash { get; set; } /// /// Meta INI for the downloaded archive /// - public string Meta; + public string Meta { get; set; } /// /// Human friendly name of this archive /// - public string Name; + public string Name { get; set; } - public long Size; + public long Size { get; set; } public AbstractDownloadState State { get; set; } } diff --git a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs index 2feca218..2731fe19 100644 --- a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs +++ b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs @@ -1,5 +1,9 @@ -using System.Threading.Tasks; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; +using MongoDB.Bson.Serialization.Attributes; using Wabbajack.Lib.Validation; namespace Wabbajack.Lib.Downloaders @@ -7,8 +11,39 @@ namespace Wabbajack.Lib.Downloaders /// /// Base for all abstract downloaders /// + [BsonDiscriminator(RootClass = true)] + [BsonKnownTypes(typeof(HTTPDownloader.State), typeof(GameFileSourceDownloader.State), typeof(GoogleDriveDownloader.State), + typeof(LoversLabDownloader.State), typeof(ManualDownloader.State), typeof(MediaFireDownloader.State), typeof(MegaDownloader.State), + typeof(ModDBDownloader.State), typeof(NexusDownloader.State), typeof(SteamWorkshopDownloader.State))] public abstract class AbstractDownloadState { + + public static List KnownSubTypes = new List() + { + typeof(HTTPDownloader.State), + typeof(GameFileSourceDownloader.State), + typeof(GoogleDriveDownloader.State), + typeof(LoversLabDownloader.State), + typeof(ManualDownloader.State), + typeof(MediaFireDownloader.State), + typeof(MegaDownloader.State), + typeof(ModDBDownloader.State), + typeof(NexusDownloader.State), + typeof(SteamWorkshopDownloader.State) + }; + public static Dictionary NameToType { get; set; } + public static Dictionary TypeToName { get; set; } + + static AbstractDownloadState() + { + NameToType = KnownSubTypes.ToDictionary(t => t.FullName.Substring(t.Namespace.Length + 1), t => t); + TypeToName = NameToType.ToDictionary(k => k.Value, k => k.Key); + } + + public abstract object[] PrimaryKey { get; } + + + /// /// Returns true if this file is allowed to be downloaded via whitelist /// diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs index 8f389fbd..680ae6bf 100644 --- a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs @@ -52,6 +52,8 @@ namespace Wabbajack.Lib.Downloaders internal string SourcePath => Path.Combine(Game.MetaData().GameLocation(), GameFile); + public override object[] PrimaryKey { get => new object[] {Game, GameFile}; } + public override bool IsWhitelisted(ServerWhitelist whitelist) { return true; diff --git a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs index ecd8e23e..6a63f931 100644 --- a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs @@ -36,6 +36,8 @@ namespace Wabbajack.Lib.Downloaders public class State : AbstractDownloadState { public string Id { get; set; } + public override object[] PrimaryKey { get => new object[] {Id}; } + public override bool IsWhitelisted(ServerWhitelist whitelist) { return whitelist.GoogleIDs.Contains(Id); diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index 7f7d5e59..911776b8 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -63,6 +63,8 @@ namespace Wabbajack.Lib.Downloaders [Exclude] public HttpClient Client { get; set; } + public override object[] PrimaryKey { get => new object[] {Url};} + public override bool IsWhitelisted(ServerWhitelist whitelist) { return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p)); diff --git a/Wabbajack.Lib/Downloaders/LoversLabDownloader.cs b/Wabbajack.Lib/Downloaders/LoversLabDownloader.cs index 2b304706..1b8b795e 100644 --- a/Wabbajack.Lib/Downloaders/LoversLabDownloader.cs +++ b/Wabbajack.Lib/Downloaders/LoversLabDownloader.cs @@ -117,6 +117,8 @@ namespace Wabbajack.Lib.Downloaders public string FileID { get; set; } public string FileName { get; set; } + public override object[] PrimaryKey { get => new object[] {FileID, FileName}; } + public override bool IsWhitelisted(ServerWhitelist whitelist) { return true; diff --git a/Wabbajack.Lib/Downloaders/ManualDownloader.cs b/Wabbajack.Lib/Downloaders/ManualDownloader.cs index ad316936..c3352514 100644 --- a/Wabbajack.Lib/Downloaders/ManualDownloader.cs +++ b/Wabbajack.Lib/Downloaders/ManualDownloader.cs @@ -73,6 +73,8 @@ namespace Wabbajack.Lib.Downloaders public class State : AbstractDownloadState { public string Url { get; set; } + public override object[] PrimaryKey { get => new object[] {Url}; } + public override bool IsWhitelisted(ServerWhitelist whitelist) { return true; diff --git a/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs b/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs index 3f089f7d..609bf713 100644 --- a/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs +++ b/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs @@ -24,6 +24,8 @@ namespace Wabbajack.Lib.Downloaders { public string Url { get; set; } + public override object[] PrimaryKey { get => new object[] {Url};} + public override bool IsWhitelisted(ServerWhitelist whitelist) { return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p)); diff --git a/Wabbajack.Lib/Downloaders/ModDBDownloader.cs b/Wabbajack.Lib/Downloaders/ModDBDownloader.cs index 72a1c0ee..4a9d3ebd 100644 --- a/Wabbajack.Lib/Downloaders/ModDBDownloader.cs +++ b/Wabbajack.Lib/Downloaders/ModDBDownloader.cs @@ -1,4 +1,5 @@ -using System.Net.Http; +using System; +using System.Net.Http; using System.Text.RegularExpressions; using System.Threading.Tasks; using Wabbajack.Common; @@ -34,6 +35,8 @@ namespace Wabbajack.Lib.Downloaders public class State : AbstractDownloadState { public string Url { get; set; } + public override object[] PrimaryKey { get => new object[]{Url}; } + public override bool IsWhitelisted(ServerWhitelist whitelist) { // Everything from Moddb is whitelisted diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index b589cf47..e96cf160 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -112,18 +112,20 @@ namespace Wabbajack.Lib.Downloaders public class State : AbstractDownloadState { - public string Author; - public string FileID; - public string GameName; - public string ModID; - public string UploadedBy; - public string UploaderProfile; - public string Version; - public string SlideShowPic; - public string ModName; - public string NexusURL; - public string Summary; - public bool Adult; + public string Author { get; set; } + public string FileID { get; set; } + public string GameName { get; set; } + public string ModID { get; set; } + public string UploadedBy { get; set; } + public string UploaderProfile { get; set; } + public string Version { get; set; } + public string SlideShowPic { get; set; } + public string ModName { get; set; } + public string NexusURL { get; set; } + public string Summary { get; set; } + public bool Adult { get; set; } + + public override object[] PrimaryKey { get => new object[]{GameName, ModID, FileID};} public override bool IsWhitelisted(ServerWhitelist whitelist) { @@ -137,7 +139,7 @@ namespace Wabbajack.Lib.Downloaders try { var client = await NexusApiClient.Get(); - url = await client.GetNexusDownloadLink(this, false); + url = await client.GetNexusDownloadLink(this); } catch (Exception ex) { diff --git a/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs b/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs index 503e8563..dc871383 100644 --- a/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs +++ b/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs @@ -41,6 +41,8 @@ namespace Wabbajack.Lib.Downloaders public class State : AbstractDownloadState { public SteamWorkshopItem Item { get; set; } + public override object[] PrimaryKey { get => new object[] {Item.Game, Item.ItemID}; } + public override bool IsWhitelisted(ServerWhitelist whitelist) { return true; diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 0688d105..5234f302 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -320,7 +320,7 @@ namespace Wabbajack.Lib Utils.Log( $"Removing {remove.Count} archives from the compilation state, this is probably not an issue but reference this if you have compilation failures"); - remove.Do(r => Utils.Log($"Resolution failed for: {r.File}")); + remove.Do(r => Utils.Log($"Resolution failed for: {r.File.FullPath}")); IndexedArchives.RemoveAll(a => remove.Contains(a)); } diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index 1904724f..5eb1b9c2 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -179,6 +179,7 @@ namespace Wabbajack.Lib.NexusApi { var dailyRemaining = int.Parse(response.Headers.GetValues("x-rl-daily-remaining").First()); var hourlyRemaining = int.Parse(response.Headers.GetValues("x-rl-hourly-remaining").First()); + Utils.Log($"Nexus Requests Remaining: {dailyRemaining} daily - {hourlyRemaining} hourly"); lock (RemainingLock) { @@ -200,6 +201,7 @@ namespace Wabbajack.Lib.NexusApi { ApiKey = apiKey; + HttpClient.BaseAddress = new Uri("https://api.nexusmods.com"); // set default headers for all requests to the Nexus API var headers = HttpClient.DefaultRequestHeaders; headers.Add("User-Agent", Consts.UserAgent); @@ -218,10 +220,12 @@ namespace Wabbajack.Lib.NexusApi return new NexusApiClient(apiKey); } - private async Task Get(string url) + public async Task Get(string url) { var response = await HttpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); UpdateRemaining(response); + if (!response.IsSuccessStatusCode) + throw new HttpRequestException($"{response.StatusCode} - {response.ReasonPhrase}"); using (var stream = await response.Content.ReadAsStreamAsync()) { @@ -231,83 +235,26 @@ namespace Wabbajack.Lib.NexusApi private async Task GetCached(string url) { - var code = Encoding.UTF8.GetBytes(url).ToHex() + ".json"; - - if (UseLocalCache) - { - var cache_file = Path.Combine(LocalCacheDir, code); - - lock (_diskLock) - { - if (!Directory.Exists(LocalCacheDir)) - Directory.CreateDirectory(LocalCacheDir); - - - if (File.Exists(cache_file)) - { - return cache_file.FromJSON(); - } - } - - var result = await Get(url); - - if (result == null) - return result; - - lock (_diskLock) - { - result.ToJSON(cache_file); - } - - - return result; - } - try { - return await Get(Consts.WabbajackCacheLocation + code); + var builder = new UriBuilder(url) { Host = Consts.WabbajackCacheHostname, Port = Consts.WabbajackCachePort, Scheme = "http" }; + return await Get(builder.ToString()); } - catch (Exception) + catch (Exception ex) { return await Get(url); } } - public async Task GetNexusDownloadLink(NexusDownloader.State archive, bool cache = false) + public async Task GetNexusDownloadLink(NexusDownloader.State archive) { - if (cache) - { - var result = await TryGetCachedLink(archive); - if (result.Succeeded) - { - return result.Value; - } - } - ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(archive.GameName)}/mods/{archive.ModID}/files/{archive.FileID}/download_link.json"; return (await Get>(url)).First().URI; } - private async Task> TryGetCachedLink(NexusDownloader.State archive) - { - if (!Directory.Exists(Consts.NexusCacheDirectory)) - Directory.CreateDirectory(Consts.NexusCacheDirectory); - - var path = Path.Combine(Consts.NexusCacheDirectory, $"link-{archive.GameName}-{archive.ModID}-{archive.FileID}.txt"); - if (!File.Exists(path) || (DateTime.Now - new FileInfo(path).LastWriteTime).TotalHours > 24) - { - File.Delete(path); - var result = await GetNexusDownloadLink(archive); - File.WriteAllText(path, result); - return GetResponse.Succeed(result); - } - - return GetResponse.Succeed(File.ReadAllText(path)); - } - public async Task GetFileInfo(NexusDownloader.State mod) { var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(mod.GameName)}/mods/{mod.ModID}/files/{mod.FileID}.json"; @@ -358,23 +305,8 @@ namespace Wabbajack.Lib.NexusApi public string URI { get; set; } } - private class UpdatedMod - { - public long mod_id; - public long latest_file_update; - public long latest_mod_activity; - } - private static bool? _useLocalCache; - public static bool UseLocalCache - { - get - { - if (_useLocalCache == null) return LocalCacheDir != null; - return _useLocalCache ?? false; - } - set => _useLocalCache = value; - } + public static MethodInfo CacheMethod { get; set; } private static string _localCacheDir; public static string LocalCacheDir @@ -387,88 +319,5 @@ namespace Wabbajack.Lib.NexusApi } set => _localCacheDir = value; } - - - public async Task ClearUpdatedModsInCache() - { - if (!UseLocalCache) return; - using (var queue = new WorkQueue()) - { - var invalid_json = (await Directory.EnumerateFiles(LocalCacheDir, "*.json") - .PMap(queue, f => - { - var s = JsonSerializer.Create(); - try - { - using (var tr = File.OpenText(f)) - s.Deserialize(new JsonTextReader(tr)); - return null; - } - catch (JsonReaderException) - { - return f; - } - })).Where(f => f != null).ToList(); - Utils.Log($"Found {invalid_json.Count} bad json files"); - foreach (var file in invalid_json) - File.Delete(file); - } - - var gameTasks = GameRegistry.Games.Values - .Where(game => game.NexusName != null) - .Select(async game => - { - return (game, - mods: await Get>( - $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m")); - }) - .Select(async rTask => - { - var (game, mods) = await rTask; - return mods.Select(mod => new { game = game, mod = mod }); - }); - var purge = (await Task.WhenAll(gameTasks)) - .SelectMany(i => i) - .ToList(); - - Utils.Log($"Found {purge.Count} updated mods in the last month"); - using (var queue = new WorkQueue()) - { - var to_purge = (await Directory.EnumerateFiles(LocalCacheDir, "*.json") - .PMap(queue, f => - { - Utils.Status("Cleaning Nexus cache for"); - var uri = new Uri(Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(f).FromHex())); - var parts = uri.PathAndQuery.Split('/', '.').ToHashSet(); - var found = purge.FirstOrDefault(p => - parts.Contains(p.game.NexusName) && parts.Contains(p.mod.mod_id.ToString())); - if (found != null) - { - var a = found.mod.latest_file_update.AsUnixTime(); - // Mod activity could hide files - var b = found.mod.latest_mod_activity.AsUnixTime(); - var should_remove = File.GetLastWriteTimeUtc(f) <= (a > b ? a : b); - return (should_remove, f); - } - - // ToDo - // Can improve to not read the entire file to see if it starts with null - if (File.ReadAllText(f).StartsWith("null")) - return (true, f); - - return (false, f); - })) - .Where(p => p.Item1) - .ToList(); - - Utils.Log($"Purging {to_purge.Count} cache entries"); - await to_purge.PMap(queue, f => - { - var uri = new Uri(Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(f.f).FromHex())); - Utils.Log($"Purging {uri}"); - File.Delete(f.f); - }); - } - } } } diff --git a/Wabbajack.Lib/UI/FilePickerVM.cs b/Wabbajack.Lib/UI/FilePickerVM.cs index c7e49d17..b087fe19 100644 --- a/Wabbajack.Lib/UI/FilePickerVM.cs +++ b/Wabbajack.Lib/UI/FilePickerVM.cs @@ -1,4 +1,4 @@ -using DynamicData; +using DynamicData; using Microsoft.WindowsAPICodePack.Dialogs; using ReactiveUI; using ReactiveUI.Fody.Helpers; @@ -80,7 +80,7 @@ namespace Wabbajack.Lib this.WhenAny(x => x.TargetPath) // Dont want to debounce the initial value, because we know it's null .Skip(1) - .Debounce(TimeSpan.FromMilliseconds(200)) + .Debounce(TimeSpan.FromMilliseconds(200), RxApp.TaskpoolScheduler) .StartWith(default(string)), resultSelector: (existsOption, type, path) => (ExistsOption: existsOption, Type: type, Path: path)) .StartWith((ExistsOption: ExistCheckOption, Type: PathType, Path: TargetPath)) @@ -107,7 +107,7 @@ namespace Wabbajack.Lib .Replay(1) .RefCount(); - _exists = Observable.Interval(TimeSpan.FromSeconds(3)) + _exists = Observable.Interval(TimeSpan.FromSeconds(3), RxApp.TaskpoolScheduler) // Only check exists on timer if desired .FilterSwitch(doExistsCheck) .Unit() @@ -119,6 +119,7 @@ namespace Wabbajack.Lib .CombineLatest(existsCheckTuple, resultSelector: (_, tuple) => tuple) // Refresh exists + .ObserveOn(RxApp.TaskpoolScheduler) .Select(t => { switch (t.ExistsOption) @@ -146,7 +147,7 @@ namespace Wabbajack.Lib } }) .DistinctUntilChanged() - .ObserveOn(RxApp.MainThreadScheduler) + .ObserveOnGuiThread() .StartWith(false) .ToProperty(this, nameof(Exists)); @@ -217,6 +218,7 @@ namespace Wabbajack.Lib if (filter.Failed) return filter; return ErrorResponse.Convert(err); }) + .ObserveOnGuiThread() .ToProperty(this, nameof(ErrorState)); _inError = this.WhenAny(x => x.ErrorState) @@ -242,6 +244,7 @@ namespace Wabbajack.Lib if (!string.IsNullOrWhiteSpace(filters)) return filters; return err?.Reason; }) + .ObserveOnGuiThread() .ToProperty(this, nameof(ErrorTooltip)); } diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 9447a0b6..588372bb 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -57,6 +57,9 @@ MinimumRecommendedRules.ruleset + + ..\..\..\Users\tbald\.nuget\packages\mongodb.bson\2.10.0\lib\net452\MongoDB.Bson.dll + @@ -214,7 +217,7 @@ 11.0.6 - 11.0.1 + 11.0.6 0.24.0 @@ -223,7 +226,7 @@ 1.2.1 - 4.3.1 + 4.3.2 1.0.4 diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 03cc47a6..b97fc1d9 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -222,10 +222,10 @@ namespace Wabbajack.Test [TestMethod] public async Task NexusDownload() { - var old_val = NexusApiClient.UseLocalCache; + var old_val = NexusApiClient.CacheMethod; try { - NexusApiClient.UseLocalCache = false; + NexusApiClient.CacheMethod = null; var ini = @"[General] gameName=SkyrimSE modID = 12604 @@ -251,7 +251,7 @@ namespace Wabbajack.Test } finally { - NexusApiClient.UseLocalCache = old_val; + NexusApiClient.CacheMethod = old_val; } } diff --git a/Wabbajack.Test/Wabbajack.Test.csproj b/Wabbajack.Test/Wabbajack.Test.csproj index 1fd503d4..0b27b547 100644 --- a/Wabbajack.Test/Wabbajack.Test.csproj +++ b/Wabbajack.Test/Wabbajack.Test.csproj @@ -188,7 +188,7 @@ 11.0.6 - 4.3.1 + 4.3.2 diff --git a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj index e39038a0..9df4add4 100644 --- a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj +++ b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj @@ -106,7 +106,7 @@ 2.0.0 - 4.3.1 + 4.3.2 diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index 14bc9308..d10154c6 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -35,12 +35,15 @@ namespace Wabbajack.VirtualFileSystem public StatusUpdateTracker UpdateTracker { get; set; } = new StatusUpdateTracker(1); public WorkQueue Queue { get; } + public bool UseExtendedHashes { get; set; } - public Context(WorkQueue queue) + public Context(WorkQueue queue, bool extendedHashes = false) { Queue = queue; + UseExtendedHashes = extendedHashes; } + public TemporaryDirectory GetTemporaryFolder() { return new TemporaryDirectory(Path.Combine(_stagingFolder, Guid.NewGuid().ToString())); diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index aa0a4aec..d3dfa9dd 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -3,8 +3,10 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.IO; using System.Linq; +using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; +using K4os.Hash.Crc; using Wabbajack.Common; using Wabbajack.Common.CSP; using Directory = Alphaleonis.Win32.Filesystem.Directory; @@ -40,6 +42,7 @@ namespace Wabbajack.VirtualFileSystem } public string Hash { get; internal set; } + public ExtendedHashes ExtendedHashes { get; set; } public long Size { get; internal set; } public long LastModified { get; internal set; } @@ -143,6 +146,8 @@ namespace Wabbajack.VirtualFileSystem LastAnalyzed = DateTime.Now.Ticks, Hash = abs_path.FileHash() }; + if (context.UseExtendedHashes) + self.ExtendedHashes = ExtendedHashes.FromFile(abs_path); if (FileExtractor.CanExtract(abs_path)) { @@ -162,6 +167,7 @@ namespace Wabbajack.VirtualFileSystem return self; } + public void Write(MemoryStream ms) { using (var bw = new BinaryWriter(ms, Encoding.UTF8, true)) @@ -265,6 +271,42 @@ namespace Wabbajack.VirtualFileSystem } } + public class ExtendedHashes + { + public static ExtendedHashes FromFile(string file) + { + var hashes = new ExtendedHashes(); + using (var stream = File.OpenRead(file)) + { + hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex(); + stream.Position = 0; + hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex(); + stream.Position = 0; + hashes.MD5 = System.Security.Cryptography.MD5.Create().ComputeHash(stream).ToHex(); + stream.Position = 0; + + var bytes = new byte[1024 * 8]; + var crc = new Crc32(); + while (true) + { + var read = stream.Read(bytes, 0, bytes.Length); + if (read == 0) break; + crc.Update(bytes, 0, read); + } + + hashes.CRC = crc.DigestBytes().ToHex(); + } + + return hashes; + } + + public string SHA256 { get; set; } + public string SHA1 { get; set; } + public string MD5 { get; set; } + public string CRC { get; set; } + } + + public class CannotStageNativeFile : Exception { public CannotStageNativeFile(string cannotStageANativeFile) : base(cannotStageANativeFile) diff --git a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj index 2639ee58..46b8cfc1 100644 --- a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj +++ b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj @@ -88,6 +88,9 @@ 2.2.6 + + 1.1.4 + 1.7.0 diff --git a/Wabbajack.sln b/Wabbajack.sln index f46b08e3..6286c990 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -34,7 +34,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VirtualFileSystem EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.CacheServer", "Wabbajack.CacheServer\Wabbajack.CacheServer.csproj", "{BDC9A094-D235-47CD-83CA-44199B60AB20}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OMODExtractor", "OMODExtractor\OMODExtractor.csproj", "{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OMODExtractor", "OMODExtractor\OMODExtractor.csproj", "{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -225,8 +225,8 @@ Global {BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug|x86.Build.0 = Debug|Any CPU {BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|Any CPU.ActiveCfg = Release|Any CPU {BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|Any CPU.Build.0 = Release|Any CPU - {BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x64.ActiveCfg = Release|Any CPU - {BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x64.Build.0 = Release|Any CPU + {BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x64.ActiveCfg = Release|x64 + {BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x64.Build.0 = Release|x64 {BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x86.ActiveCfg = Release|Any CPU {BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x86.Build.0 = Release|Any CPU {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU diff --git a/Wabbajack/View Models/Compilers/CompilerVM.cs b/Wabbajack/View Models/Compilers/CompilerVM.cs index 136cc742..86ae7666 100644 --- a/Wabbajack/View Models/Compilers/CompilerVM.cs +++ b/Wabbajack/View Models/Compilers/CompilerVM.cs @@ -60,6 +60,9 @@ namespace Wabbajack [Reactive] public ErrorResponse? Completed { get; set; } + private readonly ObservableAsPropertyHelper _progressTitle; + public string ProgressTitle => _progressTitle.Value; + public CompilerVM(MainWindowVM mainWindowVM) { MWVM = mainWindowVM; @@ -114,8 +117,9 @@ namespace Wabbajack _image = this.WhenAny(x => x.CurrentModlistSettings.ImagePath.TargetPath) // Throttle so that it only loads image after any sets of swaps have completed - .Throttle(TimeSpan.FromMilliseconds(50), RxApp.MainThreadScheduler) + .Throttle(TimeSpan.FromMilliseconds(50), RxApp.TaskpoolScheduler) .DistinctUntilChanged() + .ObserveOn(RxApp.MainThreadScheduler) .Select(path => { if (string.IsNullOrWhiteSpace(path)) return UIUtils.BitmapImageFromResource("Resources/Wabba_Mouth_No_Text.png"); @@ -157,7 +161,7 @@ namespace Wabbajack return ret; }) .ToObservableChangeSet(x => x.Status.ID) - .Batch(TimeSpan.FromMilliseconds(250), RxApp.TaskpoolScheduler) + .Batch(TimeSpan.FromMilliseconds(50), RxApp.TaskpoolScheduler) .EnsureUniqueChanges() .Filter(i => i.Status.IsWorking && i.Status.ID != WorkQueue.UnassignedCpuId) .ObserveOn(RxApp.MainThreadScheduler) @@ -246,6 +250,22 @@ namespace Wabbajack Process.Start("explorer.exe", OutputLocation.TargetPath); } }); + + _progressTitle = Observable.CombineLatest( + this.WhenAny(x => x.Compiling), + this.WhenAny(x => x.StartedCompilation), + resultSelector: (compiling, started) => + { + if (compiling) + { + return "Compiling"; + } + else + { + return started ? "Compiled" : "Configuring"; + } + }) + .ToProperty(this, nameof(ProgressTitle)); } } } diff --git a/Wabbajack/View Models/Installers/InstallerVM.cs b/Wabbajack/View Models/Installers/InstallerVM.cs index 94db86eb..dd6d37ef 100644 --- a/Wabbajack/View Models/Installers/InstallerVM.cs +++ b/Wabbajack/View Models/Installers/InstallerVM.cs @@ -210,7 +210,7 @@ namespace Wabbajack _image = Observable.CombineLatest( this.WhenAny(x => x.ModList.Error), this.WhenAny(x => x.ModList) - .Select(x => x?.ImageObservable ?? Observable.Empty()) + .Select(x => x?.ImageObservable ?? Observable.Return(WabbajackLogo)) .Switch() .StartWith(WabbajackLogo), this.WhenAny(x => x.Slideshow.Image) @@ -228,21 +228,24 @@ namespace Wabbajack .Select(x => x) .ToProperty(this, nameof(Image)); _titleText = Observable.CombineLatest( - this.WhenAny(x => x.ModList.Name), + this.WhenAny(x => x.ModList) + .Select(modList => modList?.Name ?? string.Empty), this.WhenAny(x => x.Slideshow.TargetMod.ModName) .StartWith(default(string)), this.WhenAny(x => x.Installing), resultSelector: (modList, mod, installing) => installing ? mod : modList) .ToProperty(this, nameof(TitleText)); _authorText = Observable.CombineLatest( - this.WhenAny(x => x.ModList.Author), + this.WhenAny(x => x.ModList) + .Select(modList => modList?.Author ?? string.Empty), this.WhenAny(x => x.Slideshow.TargetMod.ModAuthor) .StartWith(default(string)), this.WhenAny(x => x.Installing), resultSelector: (modList, mod, installing) => installing ? mod : modList) .ToProperty(this, nameof(AuthorText)); _description = Observable.CombineLatest( - this.WhenAny(x => x.ModList.Description), + this.WhenAny(x => x.ModList) + .Select(modList => modList?.Description ?? string.Empty), this.WhenAny(x => x.Slideshow.TargetMod.ModDescription) .StartWith(default(string)), this.WhenAny(x => x.Installing), @@ -278,8 +281,14 @@ namespace Wabbajack this.WhenAny(x => x.StartedInstallation), resultSelector: (installing, started) => { - if (!installing) return "Configuring"; - return started ? "Installing" : "Installed"; + if (installing) + { + return "Installing"; + } + else + { + return started ? "Installed" : "Configuring"; + } }) .ToProperty(this, nameof(ProgressTitle)); @@ -298,7 +307,7 @@ namespace Wabbajack return ret; }) .ToObservableChangeSet(x => x.Status.ID) - .Batch(TimeSpan.FromMilliseconds(250), RxApp.TaskpoolScheduler) + .Batch(TimeSpan.FromMilliseconds(50), RxApp.TaskpoolScheduler) .EnsureUniqueChanges() .Filter(i => i.Status.IsWorking && i.Status.ID != WorkQueue.UnassignedCpuId) .ObserveOn(RxApp.MainThreadScheduler) diff --git a/Wabbajack/View Models/Installers/MO2InstallerVM.cs b/Wabbajack/View Models/Installers/MO2InstallerVM.cs index 3982a31b..a0909b96 100644 --- a/Wabbajack/View Models/Installers/MO2InstallerVM.cs +++ b/Wabbajack/View Models/Installers/MO2InstallerVM.cs @@ -59,7 +59,7 @@ namespace Wabbajack this.WhenAny(x => x.Location.TargetPath), this.WhenAny(x => x.DownloadLocation.TargetPath), resultSelector: (target, download) => (target, download)) - .ObserveOn(RxApp.MainThreadScheduler) + .ObserveOn(RxApp.TaskpoolScheduler) .Select(i => MO2Installer.CheckValidInstallPath(i.target, i.download)) .ObserveOnGuiThread(); diff --git a/Wabbajack/View Models/MainWindowVM.cs b/Wabbajack/View Models/MainWindowVM.cs index 42efd35d..5975a11b 100644 --- a/Wabbajack/View Models/MainWindowVM.cs +++ b/Wabbajack/View Models/MainWindowVM.cs @@ -40,11 +40,8 @@ namespace Wabbajack public readonly UserInterventionHandlers UserInterventionHandlers; public readonly LoginManagerVM LoginManagerVM; - public readonly List NavigationTrail = new List(); - public Dispatcher ViewDispatcher { get; set; } - public ICommand CopyVersionCommand { get; } public ICommand ShowLoginManagerVM { get; } @@ -54,7 +51,6 @@ namespace Wabbajack public MainWindowVM(MainWindow mainWindow, MainSettings settings) { MainWindow = mainWindow; - ViewDispatcher = MainWindow.Dispatcher; Settings = settings; Installer = new Lazy(() => new InstallerVM(this)); Compiler = new Lazy(() => new CompilerVM(this)); diff --git a/Wabbajack/View Models/ModListVM.cs b/Wabbajack/View Models/ModListVM.cs index 78d4df79..264771d6 100644 --- a/Wabbajack/View Models/ModListVM.cs +++ b/Wabbajack/View Models/ModListVM.cs @@ -87,8 +87,7 @@ namespace Wabbajack public void OpenReadmeWindow() { if (string.IsNullOrEmpty(Readme)) return; - // DISABLED FOR THIS RELEASE - if (false) // SourceModList.ReadmeIsWebsite) + if (false) //SourceModList.ReadmeIsWebsite) { Process.Start(Readme); } diff --git a/Wabbajack/Views/Compilers/CompilerView.xaml b/Wabbajack/Views/Compilers/CompilerView.xaml index 409b9104..7db0c6f2 100644 --- a/Wabbajack/Views/Compilers/CompilerView.xaml +++ b/Wabbajack/Views/Compilers/CompilerView.xaml @@ -77,7 +77,7 @@ Grid.ColumnSpan="5" OverhangShadow="True" ProgressPercent="{Binding PercentCompleted}" - StatePrefixTitle="Compiling" /> + StatePrefixTitle="{Binding ProgressTitle}" />