Add ArchiveMaintainer to track archived files

Wabbajack.Server.Test/ABuildServerSystemTest.cs

@@ -32,6 +32,9 @@ namespace Wabbajack.BuildServer.Test
 
         public BuildServerFixture()
         {
+            ServerArchivesFolder.DeleteDirectory();
+            ServerArchivesFolder.CreateDirectory();
+
             var builder = Program.CreateHostBuilder(
                 new[]
                 {
@@ -54,6 +57,7 @@ namespace Wabbajack.BuildServer.Test
 
             "ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllText(
                 "GoogleIDs:\nAllowedPrefixes:\n    - http://localhost");
+
         }
 
         ~BuildServerFixture()
@@ -149,6 +153,7 @@ namespace Wabbajack.BuildServer.Test
 
             Consts.ModlistSummaryURL = MakeURL("lists/status.json");
             Consts.ServerWhitelistURL = MakeURL("ServerWhitelist.yaml");
+           
         }
 
         public WorkQueue Queue { get; set; }

Wabbajack.Server.Test/ArchiveMaintainerTests.cs

@@ -0,0 +1,49 @@
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Server.Services;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Wabbajack.BuildServer.Test
+{
+    public class ArchiveMaintainerTests : ABuildServerSystemTest
+    {
+        public ArchiveMaintainerTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
+        {
+        }
+        
+        [Fact]
+        public async Task CanIngestFiles()
+        {
+            var maintainer = Fixture.GetService<ArchiveMaintainer>();
+            using var tf = new TempFile();
+            using var tf2 = new TempFile();
+            
+            await tf.Path.WriteAllBytesAsync(RandomData(1024));
+            await tf.Path.CopyToAsync(tf2.Path);
+
+            
+            var hash = await tf.Path.FileHashAsync();
+            await maintainer.Ingest(tf.Path);
+            
+            Assert.True(maintainer.TryGetPath(hash, out var found));
+            Assert.Equal(await tf2.Path.ReadAllBytesAsync(), await found.ReadAllBytesAsync());
+        }
+        
+        
+        [Fact]
+        public async Task IngestsExistingFiles()
+        {
+            var maintainer = Fixture.GetService<ArchiveMaintainer>();
+            using var tf = new TempFile();
+            
+            await tf.Path.WriteAllBytesAsync(RandomData(1024));
+            var hash = await tf.Path.FileHashAsync();
+            
+            await tf.Path.CopyToAsync(Fixture.ServerArchivesFolder.Combine(hash.ToHex()));
+            maintainer.Start();
+
+            Assert.True(maintainer.TryGetPath(hash, out var found));
+        }
+    }
+}

Wabbajack.Server/Services/ArchiveMaintainer.cs

@@ -0,0 +1,81 @@
+using System;
+using System.Collections.Concurrent;
+using System.Linq;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer;
+using Wabbajack.Common;
+using File = System.IO.File;
+
+namespace Wabbajack.Server.Services
+{
+    /// <summary>
+    /// Maintains a concurrent cache of all the files we've downloaded, indexed by Hash
+    /// </summary>
+    public class ArchiveMaintainer
+    {
+        private AppSettings _settings;
+        private ILogger<ArchiveMaintainer> _logger;
+        private ConcurrentDictionary<Hash, AbsolutePath> _archives = new ConcurrentDictionary<Hash, AbsolutePath>();
+
+        public ArchiveMaintainer(ILogger<ArchiveMaintainer> logger, AppSettings settings)
+        {
+            _settings = settings;
+            _logger = logger;
+        }
+
+        public void Start()
+        {
+            foreach (var path in _settings.ArchivePath.EnumerateFiles(false))
+            {
+                try
+                {
+                    var hash = Hash.FromHex((string)path.FileNameWithoutExtension);
+                    _archives[hash] = path;
+                }
+                catch (Exception ex)
+                {
+                    _logger.Log(LogLevel.Error, ex.ToString());
+                }
+            }
+            _logger.Log(LogLevel.Information, $"Found {_archives.Count} archives");
+        }
+
+        public async Task<AbsolutePath> Ingest(AbsolutePath file)
+        {
+            var hash = await file.FileHashAsync();
+            if (HaveArchive(hash))
+            {
+                file.Delete();
+                return _archives[hash];
+            }
+            
+            var newPath = _settings.ArchivePath.Combine(hash.ToHex());
+            await file.MoveToAsync(newPath);
+            _archives[hash] = newPath;
+            return _archives[hash];
+        }
+
+        public bool HaveArchive(Hash hash)
+        {
+            return _archives.ContainsKey(hash);
+        }
+
+        public bool TryGetPath(Hash hash, out AbsolutePath path)
+        {
+            return _archives.TryGetValue(hash, out path);
+        }
+    }
+    
+    public static class ArchiveMaintainerExtensions 
+    {
+        public static void UseArchiveMaintainer(this IApplicationBuilder b)
+        {
+            var poll = (ArchiveMaintainer)b.ApplicationServices.GetService(typeof(ArchiveMaintainer));
+            poll.Start();
+        }
+    
+    }
+}

Wabbajack.Server/Startup.cs

@@ -56,7 +56,8 @@ namespace Wabbajack.Server
             services.AddSingleton<AppSettings>();
             services.AddSingleton<SqlService>();
             services.AddSingleton<GlobalInformation>();
-            services.AddSingleton<NexusPoll>();            
+            services.AddSingleton<NexusPoll>();
+            services.AddSingleton<ArchiveMaintainer>();
             services.AddMvc();
             services.AddControllers()
                 .AddNewtonsoftJson(o =>