Integrate build server file indexing into the main app

Wabbajack.BuildServer/Controllers/IndexedFiles.cs

@@ -7,7 +7,7 @@ using MongoDB.Bson;
 using MongoDB.Driver;
 using Wabbajack.BuildServer.Models;
 using Wabbajack.Common;
-using Wabbajack.Lib.DTOs;
+using Wabbajack.VirtualFileSystem;
 
 namespace Wabbajack.BuildServer.Controllers
 {
@@ -20,7 +20,7 @@ namespace Wabbajack.BuildServer.Controllers
 
         [HttpGet]
         [Route("{xxHashAsBase64}")]
-        public async Task<IndexedVirtualFile> GetFile(string xxHashAsBase64)
+        public async Task<IActionResult> GetFile(string xxHashAsBase64)
         {
             var id = xxHashAsBase64.FromHex().ToBase64();
             var query = new[]
@@ -63,7 +63,10 @@ namespace Wabbajack.BuildServer.Controllers
                 return file;
             }
 
-            return Convert(result.FirstOrDefault());
+            var first = result.FirstOrDefault();
+            if (first == null)
+                return NotFound();
+            return Ok(Convert(first));
         }
 
         public class TreeResult : IndexedFile

Wabbajack.BuildServer/GraphQL/Query.cs

@@ -49,7 +49,7 @@ namespace Wabbajack.BuildServer.GraphQL
                     new QueryArgument<IdGraphType> {Name = "id", Description = "Id of the Job"}),
                 resolve: async context =>
                 {
-                    var id = Guid.Parse(context.GetArgument<string>("id"));
+                    var id = context.GetArgument<string>("id");
                     var data = await db.Jobs.AsQueryable().Where(j => j.Id == id).ToListAsync();
                     return data;
                 });

Wabbajack.BuildServer/Models/JobQueue/Job.cs

@@ -18,8 +18,7 @@ namespace Wabbajack.BuildServer.Models.JobQueue
             High,
         }
 
-        [BsonId]
-        public Guid Id { get; set; }
+        [BsonId] public String Id { get; set; } = Guid.NewGuid().ToString();
         public DateTime? Started { get; set; }
         public DateTime? Ended { get; set; }
         public DateTime Created { get; set; } = DateTime.Now;
@@ -29,7 +28,7 @@ namespace Wabbajack.BuildServer.Models.JobQueue
         public bool RequiresNexus { get; set; } = true;
         public AJobPayload Payload { get; set; }
 
-        public static async Task<Guid> Enqueue(DBContext db, Job job)
+        public static async Task<String> Enqueue(DBContext db, Job job)
         {
             await db.Jobs.InsertOneAsync(job);
             return job.Id;
@@ -54,7 +53,7 @@ namespace Wabbajack.BuildServer.Models.JobQueue
         {
             var filter = new BsonDocument
             {
-                {"query", new BsonDocument {{"_id", job.Id}}},
+                {"_id", job.Id},
             };
             var update = new BsonDocument
             {

Wabbajack.Common/FileExtractor.cs

@@ -259,6 +259,7 @@ namespace Wabbajack.Common
                 return p.ExitCode == 0;
             }
 
+
             var testInfo = new ProcessStartInfo
             {
                 FileName = "7z.exe",
@@ -295,5 +296,12 @@ namespace Wabbajack.Common
             testP.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Can Extract Check {v}");
             return testP.ExitCode == 0;
         }
+        
+        
+        public static bool MightBeArchive(string path)
+        {
+            var ext = Path.GetExtension(path.ToLower());
+            return ext == ".exe" || Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext);
+        }
     }
 }

Wabbajack.VirtualFileSystem/Context.cs

@@ -72,7 +72,7 @@ namespace Wabbajack.VirtualFileSystem
                                         return found;
                                 }
 
-                                return await VirtualFile.Analyze(this, null, f, f);
+                                return await VirtualFile.Analyze(this, null, f, f, true);
                             });
 
             var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@@ -109,7 +109,7 @@ namespace Wabbajack.VirtualFileSystem
                             return found;
                     }
 
-                    return await VirtualFile.Analyze(this, null, f, f);
+                    return await VirtualFile.Analyze(this, null, f, f, true);
                 });
 
             var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());

Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs

@@ -1,6 +1,6 @@
 using System.Collections.Generic;
 
-namespace Wabbajack.Lib.DTOs
+namespace Wabbajack.VirtualFileSystem
 {
     /// <summary>
     /// Response from the Build server for a indexed file

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
+using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Tasks;
@@ -133,9 +134,40 @@ namespace Wabbajack.VirtualFileSystem
         }
 
         public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, string abs_path,
-            string rel_path)
+            string rel_path, bool topLevel)
         {
+            var hash = abs_path.FileHash();
             var fi = new FileInfo(abs_path);
+
+            if (FileExtractor.MightBeArchive(abs_path))
+            {
+                var result = await TryGetContentsFromServer(hash);
+
+                if (result != null)
+                {
+                    VirtualFile Convert(IndexedVirtualFile file, string path, VirtualFile vparent)
+                    {
+                        var vself = new VirtualFile
+                        {
+                            Context = context,
+                            Name = path,
+                            Parent = vparent,
+                            Size = file.Size,
+                            LastModified = fi.LastWriteTimeUtc.Ticks,
+                            LastAnalyzed = DateTime.Now.Ticks,
+                            Hash = file.Hash,
+
+                        };
+
+                        vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList();
+
+                        return vself;
+                    }
+
+                    return Convert(result, rel_path, parent);
+                }
+            }
+
             var self = new VirtualFile
             {
                 Context = context,
@@ -144,7 +176,7 @@ namespace Wabbajack.VirtualFileSystem
                 Size = fi.Length,
                 LastModified = fi.LastWriteTimeUtc.Ticks,
                 LastAnalyzed = DateTime.Now.Ticks,
-                Hash = abs_path.FileHash()
+                Hash = hash
             };
             if (context.UseExtendedHashes)
                 self.ExtendedHashes = ExtendedHashes.FromFile(abs_path);
@@ -157,7 +189,7 @@ namespace Wabbajack.VirtualFileSystem
                     await FileExtractor.ExtractAll(context.Queue, abs_path, tempFolder.FullName);
 
                     var list = await Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories)
-                                        .PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName)));
+                                        .PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName), false));
 
                     self.Children = list.ToImmutableList();
                 }
@@ -167,6 +199,27 @@ namespace Wabbajack.VirtualFileSystem
             return self;
         }
 
+        private static async Task<IndexedVirtualFile> TryGetContentsFromServer(string hash)
+        {
+            try
+            {
+                var client = new HttpClient();
+                var response = await client.GetAsync($"http://{Consts.WabbajackCacheHostname}/indexed_files/{hash.FromBase64().ToHex()}");
+                if (!response.IsSuccessStatusCode)
+                    return null;
+
+                using (var stream = await response.Content.ReadAsStreamAsync())
+                {
+                    return stream.FromJSON<IndexedVirtualFile>();
+                }
+
+            }
+            catch (Exception ex)
+            {
+                return null;
+            }
+        }
+
 
         public void Write(MemoryStream ms)
         {