Updates to smooth out server update features

Wabbajack.Server.Test/MirroredFilesTests.cs

@@ -50,6 +50,13 @@ namespace Wabbajack.Server.Test
             var file2 = new TempFile();
             await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file2.Path);
         }
+
+        [Fact]
+        public async Task CanQueueFiles()
+        {
+            var service = Fixture.GetService<MirrorQueueService>();
+            Assert.Equal(1, await service.Execute());
+        }
         
     }
 }

Wabbajack.Server.Test/ModFileTests.cs

@@ -1,10 +1,12 @@
 using System;
+using System.Linq;
 using System.Threading.Tasks;
 using Wabbajack.BuildServer.Test;
 using Wabbajack.Common;
 using Wabbajack.Lib;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -22,23 +24,38 @@ namespace Wabbajack.Server.Test
         public async Task CanGetDownloadStates()
         {
             var sql = Fixture.GetService<SqlService>();
+            var hash = Hash.FromBase64("eSIyd+KOG3s=");
 
             var archive =
                 new Archive(new WabbajackCDNDownloader.State(new Uri(
                     "https://wabbajack.b-cdn.net/WABBAJACK_TEST_FILE.zip_a1a3e961-5c0b-4ccf-84b4-7aa437d9640d")))
                 {
-                    Size = 20, Hash = Hash.FromBase64("eSIyd+KOG3s=")
+                    Size = 20, Hash = hash
                 };
 
             await sql.EnqueueDownload(archive);
+            await sql.UpsertMirroredFile(new MirroredFile()
+            {
+                Created = DateTime.UtcNow,
+                Uploaded = DateTime.UtcNow,
+                Hash = hash,
+                Rationale = "Test File"
+            });
             var dld = await sql.GetNextPendingDownload();
             await dld.Finish(sql);
 
 
             var state = await ClientAPI.InferDownloadState(archive.Hash);
-            
             Assert.Equal(archive.State.GetMetaIniString(), state!.GetMetaIniString());
 
+            var archives = await (await ClientAPI.GetClient()).GetJsonAsync<Archive[]>(
+                $"{Consts.WabbajackBuildServerUri}mod_files/by_hash/{hash.ToHex()}");
+            
+            Assert.True(archives.Length >= 2);
+            Assert.NotNull(archives.FirstOrDefault(a => a.State is WabbajackCDNDownloader.State));
+
+            await sql.DeleteMirroredFile(hash);
+
         }
     }
 }

Wabbajack.Server/DataLayer/ArchiveDownloads.cs

@@ -241,10 +241,19 @@ namespace Wabbajack.Server.DataLayer
             var files = (await conn.QueryAsync<(long, Hash, AbstractDownloadState)>(
                     @"SELECT Size, Hash,  DownloadState from dbo.ArchiveDownloads WHERE Hash = @Hash AND IsFailed = 0 AND DownloadFinished IS NOT NULL ORDER BY DownloadFinished DESC",
                     new {Hash = hash})
-                );
+                ).Select(e =>
-            return files.Select(e =>
+                    new Archive(e.Item3) {Size = e.Item1, Hash = e.Item2}
-                new Archive(e.Item3) {Size = e.Item1, Hash = e.Item2}
+                ).ToList();
-            ).ToArray();
+
+            if (await HaveMirror(hash) && files.Count > 0)
+            {
+                var ffile = files.First();
+                var url = new Uri($"https://{(await _mirrorCreds).Username}.b-cdn.net/{hash.ToHex()}");
+                files.Add(new Archive(
+                    new WabbajackCDNDownloader.State(url)) {Hash = hash, Size = ffile.Size, Name = ffile.Name});
+            }
+
+            return files.ToArray();
         }
     }
 }

Wabbajack.Server/DataLayer/MirroredFiles.cs

@@ -49,6 +49,13 @@ namespace Wabbajack.Server.DataLayer
             await trans.CommitAsync();
         }
 
+        public async Task DeleteMirroredFile(Hash hash)
+        {
+            await using var conn = await Open();
+            await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash",
+                new {Hash = hash});
+        }
+
         public async Task InsertAllNexusMirrors()
         {
             var permissions = (await GetNexusPermissions()).Where(p => p.Value == HTMLInterface.PermissionValue.Yes);
@@ -82,5 +89,46 @@ namespace Wabbajack.Server.DataLayer
             return await conn.QueryFirstOrDefaultAsync<Hash>("SELECT Hash FROM dbo.MirroredArchives WHERE Hash = @Hash",
                 new {Hash = hash}) != default;
         }
+
+        public async Task QueueMirroredFiles()
+        {
+            await using var conn = await Open();
+
+            await conn.ExecuteAsync(@"
+
+                INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
+
+                SELECT hs.Hash, GETUTCDATE(), 'File has re-upload permissions on the Nexus' FROM
+                (SELECT DISTINCT ad.Hash FROM dbo.NexusModPermissions p
+                INNER JOIN GameMetadata md on md.NexusGameId = p.NexusGameID
+                INNER JOIN dbo.ArchiveDownloads ad on ad.PrimaryKeyString like 'NexusDownloader+State|'+md.WabbajackName+'|'+CAST(p.ModID as nvarchar)+'|%'
+                WHERE p.Permissions = 1
+                AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
+                ) hs
+
+                INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
+                SELECT DISTINCT Hash, GETUTCDATE(), 'File is hosted on GitHub'
+                FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%github.com/%'
+                AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
+
+
+                INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
+                SELECT DISTINCT Hash, GETUTCDATE(), 'File license allows uploading to any Non-nexus site'
+                FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%enbdev.com/%'
+                AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
+
+                INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
+                SELECT DISTINCT Hash, GETUTCDATE(), 'DynDOLOD file' /*, Name*/
+                from dbo.ModListArchives mla WHERE Name like '%DynDoLOD%standalone%'
+                and Hash not in (select Hash from dbo.MirroredArchives)
+
+                INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
+                SELECT DISTINCT Hash, GETUTCDATE(), 'Distribution allowed by author' /*, Name*/
+                from dbo.ModListArchives mla WHERE Name like '%particle%patch%'
+                and Hash not in (select Hash from dbo.MirroredArchives)
+
+
+                ");
+        }
     }
 }

Wabbajack.Server/DataLayer/SqlService.cs

@@ -1,16 +1,19 @@
 using System.Data.SqlClient;
 using System.Threading.Tasks;
 using Wabbajack.BuildServer;
+using Wabbajack.Server.DTOs;
 
 namespace Wabbajack.Server.DataLayer
 {
     public partial class SqlService
     {
         private AppSettings _settings;
+        private Task<BunnyCdnFtpInfo> _mirrorCreds;
 
         public SqlService(AppSettings settings)
         {
             _settings = settings;
+            _mirrorCreds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
 
         }
 

Wabbajack.Server/Services/MirrorQueueService.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer;
+using Wabbajack.Server.DataLayer;
+
+namespace Wabbajack.Server.Services
+{
+    public class MirrorQueueService : AbstractService<MirrorQueueService, int>
+    {
+        private DiscordWebHook _discord;
+        private SqlService _sql;
+
+        public MirrorQueueService(ILogger<MirrorQueueService> logger, AppSettings settings, QuickSync quickSync, 
+            DiscordWebHook discordWebHook, SqlService sqlService) : 
+            base(logger, settings, quickSync, TimeSpan.FromMinutes(5))
+        {
+            _discord = discordWebHook;
+            _sql = sqlService;
+        }
+
+        public override async Task<int> Execute()
+        {
+            await _sql.QueueMirroredFiles();
+            return 1;
+        }
+    }
+}

Wabbajack.Server/Services/MirrorUploader.cs

@@ -23,11 +23,14 @@ namespace Wabbajack.Server.Services
     {
         private SqlService _sql;
         private ArchiveMaintainer _archives;
+        private DiscordWebHook _discord;
 
-        public MirrorUploader(ILogger<MirrorUploader> logger, AppSettings settings, SqlService sql, QuickSync quickSync, ArchiveMaintainer archives) : base(logger, settings, quickSync, TimeSpan.FromHours(1))
+        public MirrorUploader(ILogger<MirrorUploader> logger, AppSettings settings, SqlService sql, QuickSync quickSync, ArchiveMaintainer archives, DiscordWebHook discord)
+            : base(logger, settings, quickSync, TimeSpan.FromHours(1))
         {
             _sql = sql;
             _archives = archives;
+            _discord = discord;
         }
 
         public override async Task<int> Execute()
@@ -61,6 +64,12 @@ namespace Wabbajack.Server.Services
                         goto TOP;
                     }
 
+                    await _discord.Send(Channel.Spam,
+                        new DiscordMessage
+                        {
+                            Content = $"Uploading {toUpload.Hash} - {toUpload.Created} because {toUpload.Rationale}"
+                        });
+
                     var definition = await Client.GenerateFileDefinition(queue, path, (s, percent) => { });
 
                     using (var client = await GetClient(creds))

Wabbajack.Server/Startup.cs

@@ -70,6 +70,7 @@ namespace Wabbajack.Server
             services.AddSingleton<CDNMirrorList>();
             services.AddSingleton<NexusPermissionsUpdater>();
             services.AddSingleton<MirrorUploader>();
+            services.AddSingleton<MirrorQueueService>();
             
             services.AddMvc();
             services.AddControllers()
@@ -127,6 +128,7 @@ namespace Wabbajack.Server
             app.UseService<CDNMirrorList>();
             app.UseService<NexusPermissionsUpdater>();
             app.UseService<MirrorUploader>();
+            app.UseService<MirrorQueueService>();
 
             app.Use(next =>
             {

Wabbajack.Test/EndToEndTests.cs

@@ -63,7 +63,7 @@ namespace Wabbajack.Test
                 DownloadAndInstall(Game.SkyrimSpecialEdition, 4783, "Frost Armor UNP"), 
                 DownloadAndInstall(Game.SkyrimSpecialEdition, 32359, "Frost Armor HDT"),
                 DownloadAndInstall("https://github.com/ShikyoKira/Project-New-Reign---Nemesis-Main/releases/download/v0.84-beta/Nemesis.Unlimited.Behavior.Engine.v0.84-beta.rar", "Nemesis.Unlimited.Behavior.Engine.v0.84-beta.rar", "Nemesis"),
-                DownloadAndInstall(Game.Fallout4, 40136, "RAR test File"));
+                DownloadAndInstall(Game.Fallout4, 40136, "RAR test File")); // ShouldPullFrom Mirror
             
             // We're going to fully patch this mod from another source.
             await modfiles[3].Download.DeleteAsync();

Wabbajack.Test/TestUtils.cs

@@ -172,13 +172,13 @@ namespace Wabbajack.Test
             }
 
             await DownloadsPath.Combine(name + Consts.MetaFileExtension).WriteAllLinesAsync(
-                    "[General]",
+                "[General]",
-                    "manualURL=<TESTING>"
+                "manualURL=<TESTING>"
-                );
+            );
 
             return name;
         }
-
+        
         public async Task VerifyInstalledFile(string mod, string file)
         {
             var src = SourcePath.Combine((string)Consts.MO2ModFolderName, mod, file);