diff --git a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs index b92ab0a7..c7d18aa1 100644 --- a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs @@ -7,7 +7,7 @@ using MongoDB.Bson; using MongoDB.Driver; using Wabbajack.BuildServer.Models; using Wabbajack.Common; -using Wabbajack.Lib.DTOs; +using Wabbajack.VirtualFileSystem; namespace Wabbajack.BuildServer.Controllers { @@ -20,7 +20,7 @@ namespace Wabbajack.BuildServer.Controllers [HttpGet] [Route("{xxHashAsBase64}")] - public async Task GetFile(string xxHashAsBase64) + public async Task GetFile(string xxHashAsBase64) { var id = xxHashAsBase64.FromHex().ToBase64(); var query = new[] @@ -63,7 +63,10 @@ namespace Wabbajack.BuildServer.Controllers return file; } - return Convert(result.FirstOrDefault()); + var first = result.FirstOrDefault(); + if (first == null) + return NotFound(); + return Ok(Convert(first)); } public class TreeResult : IndexedFile diff --git a/Wabbajack.BuildServer/GraphQL/Query.cs b/Wabbajack.BuildServer/GraphQL/Query.cs index a34b7b16..5b7582be 100644 --- a/Wabbajack.BuildServer/GraphQL/Query.cs +++ b/Wabbajack.BuildServer/GraphQL/Query.cs @@ -49,7 +49,7 @@ namespace Wabbajack.BuildServer.GraphQL new QueryArgument {Name = "id", Description = "Id of the Job"}), resolve: async context => { - var id = Guid.Parse(context.GetArgument("id")); + var id = context.GetArgument("id"); var data = await db.Jobs.AsQueryable().Where(j => j.Id == id).ToListAsync(); return data; }); diff --git a/Wabbajack.BuildServer/Models/JobQueue/Job.cs b/Wabbajack.BuildServer/Models/JobQueue/Job.cs index b4838b16..006da7aa 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/Job.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/Job.cs @@ -18,8 +18,7 @@ namespace Wabbajack.BuildServer.Models.JobQueue High, } - [BsonId] - public Guid Id { get; set; } + [BsonId] public String Id { get; set; } = Guid.NewGuid().ToString(); public DateTime? Started { get; set; } public DateTime? Ended { get; set; } public DateTime Created { get; set; } = DateTime.Now; @@ -29,7 +28,7 @@ namespace Wabbajack.BuildServer.Models.JobQueue public bool RequiresNexus { get; set; } = true; public AJobPayload Payload { get; set; } - public static async Task Enqueue(DBContext db, Job job) + public static async Task Enqueue(DBContext db, Job job) { await db.Jobs.InsertOneAsync(job); return job.Id; @@ -54,7 +53,7 @@ namespace Wabbajack.BuildServer.Models.JobQueue { var filter = new BsonDocument { - {"query", new BsonDocument {{"_id", job.Id}}}, + {"_id", job.Id}, }; var update = new BsonDocument { diff --git a/Wabbajack.Common/FileExtractor.cs b/Wabbajack.Common/FileExtractor.cs index 31aa8ca8..e78ecdf5 100644 --- a/Wabbajack.Common/FileExtractor.cs +++ b/Wabbajack.Common/FileExtractor.cs @@ -259,6 +259,7 @@ namespace Wabbajack.Common return p.ExitCode == 0; } + var testInfo = new ProcessStartInfo { FileName = "7z.exe", @@ -295,5 +296,12 @@ namespace Wabbajack.Common testP.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Can Extract Check {v}"); return testP.ExitCode == 0; } + + + public static bool MightBeArchive(string path) + { + var ext = Path.GetExtension(path.ToLower()); + return ext == ".exe" || Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext); + } } } diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index d10154c6..d8db2cae 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -72,7 +72,7 @@ namespace Wabbajack.VirtualFileSystem return found; } - return await VirtualFile.Analyze(this, null, f, f); + return await VirtualFile.Analyze(this, null, f, f, true); }); var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList()); @@ -109,7 +109,7 @@ namespace Wabbajack.VirtualFileSystem return found; } - return await VirtualFile.Analyze(this, null, f, f); + return await VirtualFile.Analyze(this, null, f, f, true); }); var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList()); diff --git a/Wabbajack.Lib/DTOs/IndexedVirtualFile.cs b/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs similarity index 91% rename from Wabbajack.Lib/DTOs/IndexedVirtualFile.cs rename to Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs index 72179335..28049661 100644 --- a/Wabbajack.Lib/DTOs/IndexedVirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace Wabbajack.Lib.DTOs +namespace Wabbajack.VirtualFileSystem { /// /// Response from the Build server for a indexed file diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index d3dfa9dd..47979868 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.IO; using System.Linq; +using System.Net.Http; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; @@ -133,9 +134,40 @@ namespace Wabbajack.VirtualFileSystem } public static async Task Analyze(Context context, VirtualFile parent, string abs_path, - string rel_path) + string rel_path, bool topLevel) { + var hash = abs_path.FileHash(); var fi = new FileInfo(abs_path); + + if (FileExtractor.MightBeArchive(abs_path)) + { + var result = await TryGetContentsFromServer(hash); + + if (result != null) + { + VirtualFile Convert(IndexedVirtualFile file, string path, VirtualFile vparent) + { + var vself = new VirtualFile + { + Context = context, + Name = path, + Parent = vparent, + Size = file.Size, + LastModified = fi.LastWriteTimeUtc.Ticks, + LastAnalyzed = DateTime.Now.Ticks, + Hash = file.Hash, + + }; + + vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList(); + + return vself; + } + + return Convert(result, rel_path, parent); + } + } + var self = new VirtualFile { Context = context, @@ -144,7 +176,7 @@ namespace Wabbajack.VirtualFileSystem Size = fi.Length, LastModified = fi.LastWriteTimeUtc.Ticks, LastAnalyzed = DateTime.Now.Ticks, - Hash = abs_path.FileHash() + Hash = hash }; if (context.UseExtendedHashes) self.ExtendedHashes = ExtendedHashes.FromFile(abs_path); @@ -157,7 +189,7 @@ namespace Wabbajack.VirtualFileSystem await FileExtractor.ExtractAll(context.Queue, abs_path, tempFolder.FullName); var list = await Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories) - .PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName))); + .PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName), false)); self.Children = list.ToImmutableList(); } @@ -167,6 +199,27 @@ namespace Wabbajack.VirtualFileSystem return self; } + private static async Task TryGetContentsFromServer(string hash) + { + try + { + var client = new HttpClient(); + var response = await client.GetAsync($"http://{Consts.WabbajackCacheHostname}/indexed_files/{hash.FromBase64().ToHex()}"); + if (!response.IsSuccessStatusCode) + return null; + + using (var stream = await response.Content.ReadAsStreamAsync()) + { + return stream.FromJSON(); + } + + } + catch (Exception ex) + { + return null; + } + } + public void Write(MemoryStream ms) {