WIP autoheal

Wabbajack.Server.Test/ABuildServerSystemTest.cs

@@ -1,12 +1,18 @@
 using System;
+using System.Collections.Generic;
+using System.IO.Compression;
 using System.Reactive.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Hosting;
 using Wabbajack.Common;
 using Wabbajack.Common.Http;
 using Wabbajack.Common.StatusFeed;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.FileUploader;
+using Wabbajack.Lib.ModListRegistry;
 using Wabbajack.Server;
 using Wabbajack.Server.DataLayer;
 using Xunit;
@@ -182,5 +188,79 @@ namespace Wabbajack.BuildServer.Test
             _unsubErr.Dispose();
             
         }
+        
+        protected async Task<Uri> MakeModList()
+        {
+            var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!");
+            var test_archive_path = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
+            await test_archive_path.WriteAllBytesAsync(archive_data);
+
+
+
+            ModListData = new ModList();
+            ModListData.Archives.Add(
+                new Archive(new HTTPDownloader.State(MakeURL("test_archive.txt")))
+                {
+                    Hash = await test_archive_path.FileHashAsync(),
+                    Name = "test_archive",
+                    Size = test_archive_path.Size,
+                });
+            
+            var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder);
+
+            await using (var fs = modListPath.Create())
+            {
+                using var za = new ZipArchive(fs, ZipArchiveMode.Create);
+                var entry = za.CreateEntry("modlist");
+                await using var es = entry.Open();
+                ModListData.ToJson(es);
+            }
+
+            ModListMetaData = new List<ModlistMetadata>
+            {
+                new ModlistMetadata
+                {
+                    Official = false,
+                    Author = "Test Suite",
+                    Description = "A test",
+                    DownloadMetadata = new DownloadMetadata
+                    {
+                        Hash = await modListPath.FileHashAsync(), 
+                        Size = modListPath.Size
+                    },
+                    Links = new ModlistMetadata.LinksObject
+                    {
+                        MachineURL = "test_list",
+                        Download = MakeURL("test_modlist.wabbajack")
+                    }
+                },
+                new ModlistMetadata
+                {
+                    Official = true,
+                    Author = "Test Suite",
+                    Description = "A list with a broken hash",
+                    DownloadMetadata = new DownloadMetadata()
+                    {
+                        Hash = Hash.FromLong(42),
+                        Size = 42
+                    },
+                    Links = new ModlistMetadata.LinksObject
+                    {
+                        MachineURL = "broken_list",
+                        Download = MakeURL("test_modlist.wabbajack")
+                    }
+                }
+            };
+
+            var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder);
+
+            ModListMetaData.ToJson(metadataPath);
+            
+            return new Uri(MakeURL("test_mod_list_metadata.json"));
+        }
+
+        public ModList ModListData { get; set; }
+
+        public List<ModlistMetadata> ModListMetaData { get; set; }
     }
 }

Wabbajack.Server.Test/ModListValidationTests.cs

@@ -135,79 +135,7 @@ namespace Wabbajack.BuildServer.Test
         }
 
         
-        private async Task<Uri> MakeModList()
-        {
-            var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!");
-            var test_archive_path = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
-            await test_archive_path.WriteAllBytesAsync(archive_data);
 
 
-
-            ModListData = new ModList();
-            ModListData.Archives.Add(
-                new Archive(new HTTPDownloader.State(MakeURL("test_archive.txt")))
-                {
-                    Hash = await test_archive_path.FileHashAsync(),
-                    Name = "test_archive",
-                    Size = test_archive_path.Size,
-                });
-            
-            var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder);
-
-            await using (var fs = modListPath.Create())
-            {
-                using var za = new ZipArchive(fs, ZipArchiveMode.Create);
-                var entry = za.CreateEntry("modlist");
-                await using var es = entry.Open();
-                ModListData.ToJson(es);
-            }
-
-            ModListMetaData = new List<ModlistMetadata>
-            {
-                new ModlistMetadata
-                {
-                    Official = false,
-                    Author = "Test Suite",
-                    Description = "A test",
-                    DownloadMetadata = new DownloadMetadata
-                    {
-                        Hash = await modListPath.FileHashAsync(), 
-                        Size = modListPath.Size
-                    },
-                    Links = new ModlistMetadata.LinksObject
-                    {
-                        MachineURL = "test_list",
-                        Download = MakeURL("test_modlist.wabbajack")
-                    }
-                },
-                new ModlistMetadata
-                {
-                    Official = true,
-                    Author = "Test Suite",
-                    Description = "A list with a broken hash",
-                    DownloadMetadata = new DownloadMetadata()
-                    {
-                        Hash = Hash.FromLong(42),
-                        Size = 42
-                    },
-                    Links = new ModlistMetadata.LinksObject
-                    {
-                        MachineURL = "broken_list",
-                        Download = MakeURL("test_modlist.wabbajack")
-                    }
-                }
-            };
-
-            var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder);
-
-            ModListMetaData.ToJson(metadataPath);
-            
-            return new Uri(MakeURL("test_mod_list_metadata.json"));
-        }
-
-        public ModList ModListData { get; set; }
-
-        public List<ModlistMetadata> ModListMetaData { get; set; }
-
     }
 }

Wabbajack.Server.Test/ModlistUpdater.cs

@@ -0,0 +1,134 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Dapper;
+using Wabbajack.BuildServer;
+using Wabbajack.BuildServer.Test;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.ModListRegistry;
+using Wabbajack.Lib.NexusApi;
+using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
+using Wabbajack.Server.Services;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Wabbajack.Server.Test
+{
+    public class ModlistUpdater : ABuildServerSystemTest
+    {
+        public ModlistUpdater(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output,
+            fixture)
+        {
+        }
+
+        [Fact]
+        public async Task CanIndexAndUpdateFiles()
+        {
+            var validator = Fixture.GetService<ListValidator>();
+            var nonNexus = Fixture.GetService<NonNexusDownloadValidator>();
+            var modLists = await MakeModList();
+            Consts.ModlistMetadataURL = modLists.ToString();
+            
+            
+            var listDownloader = Fixture.GetService<ModListDownloader>();
+            var downloader = Fixture.GetService<ArchiveDownloader>();
+            var archiver = Fixture.GetService<ArchiveMaintainer>();
+            
+            var sql = Fixture.GetService<SqlService>();
+            var modId = long.MaxValue >> 1;
+            var oldFileId = long.MaxValue >> 2;
+            var newFileId = (long.MaxValue >> 2) + 1;
+
+            var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!");
+            var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!");
+            var oldDataHash = oldFileData.xxHash();
+            var newDataHash = newFileData.xxHash();
+
+            Assert.Equal(2, await listDownloader.CheckForNewLists());
+            Assert.Equal(1, await downloader.Execute());
+            Assert.Equal(0, await nonNexus.Execute());
+            Assert.Equal(0, await validator.Execute());
+            
+            
+            Assert.True(archiver.HaveArchive(oldDataHash));
+            Assert.False(archiver.HaveArchive(newDataHash));
+
+            var status = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
+            Assert.Equal(0, status.ValidationSummary.Failed);
+            
+
+            // Update the archive
+            await "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(newFileData);
+            
+            // Nothing new to do
+            Assert.Equal(0, await listDownloader.CheckForNewLists());
+            Assert.Equal(0, await downloader.Execute());
+            
+            // List now fails after we check the manual link
+            Assert.Equal(1, await nonNexus.Execute());
+            Assert.Equal(1, await validator.Execute());
+
+            /*
+            Assert.True(await sql.HaveIndexdFile(oldDataHash));
+            Assert.True(await sql.HaveIndexdFile(newDataHash));
+
+            var settings = Fixture.GetService<AppSettings>();
+            Assert.Equal($"Oldfile_{oldDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(oldDataHash));
+            Assert.Equal($"Newfile_{newDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(newDataHash));
+
+            Utils.Log($"Download Updating {oldDataHash} -> {newDataHash}");
+            await using var conn = await sql.Open();
+
+            await conn.ExecuteAsync("DELETE FROM dbo.DownloadStates WHERE Hash in (@OldHash, @NewHash);",
+                new {OldHash = (long)oldDataHash, NewHash = (long)newDataHash});
+
+            await sql.AddDownloadState(oldDataHash, new NexusDownloader.State
+            {
+                Game = Game.Oblivion,
+                ModID = modId,
+                FileID = oldFileId
+            });
+
+            await sql.AddDownloadState(newDataHash, new NexusDownloader.State
+            {
+                Game = Game.Oblivion,
+                ModID = modId,
+                FileID = newFileId
+            });
+            
+            Assert.NotNull(await sql.GetNexusStateByHash(oldDataHash));
+            Assert.NotNull(await sql.GetNexusStateByHash(newDataHash));
+
+            // No nexus info, so no upgrade
+            var noUpgrade = await ClientAPI.GetModUpgrade(oldDataHash);
+            Assert.Null(noUpgrade);
+
+            // Add Nexus info
+            await sql.AddNexusModFiles(Game.Oblivion, modId, DateTime.Now,
+                new NexusApiClient.GetModFilesResponse
+                {
+                    files = new List<NexusFileInfo>
+                    {
+                        new NexusFileInfo {category_name = "MAIN", file_id = newFileId, file_name = "New File"},
+                        new NexusFileInfo {category_name = null, file_id = oldFileId, file_name = "Old File"}
+                    }
+                });
+
+            
+            var enqueuedUpgrade = await ClientAPI.GetModUpgrade(oldDataHash);
+            
+            // Not Null because upgrade was enqueued
+            Assert.NotNull(enqueuedUpgrade);
+
+            await RunAllJobs();
+
+            Assert.True($"{oldDataHash.ToHex()}_{newDataHash.ToHex()}".RelativeTo(Fixture.ServerUpdatesFolder).IsFile);
+*/
+        }
+    }
+}

Wabbajack.Server/DTOs/Patch.cs

@@ -0,0 +1,14 @@
+using System;
+using Wabbajack.Common;
+
+namespace Wabbajack.Server.DTOs
+{
+    public class Patch
+    {
+        public ArchiveDownload Src { get; set; }
+        public ArchiveDownload Dest { get; set; }
+        public Hash PatchHash { get; set; }
+        public long PatchSize { get; set; }
+        public DateTime? Finished { get; set; }
+    }
+}

Wabbajack.Server/DataLayer/ArchiveDownloads.cs

@@ -37,6 +37,26 @@ namespace Wabbajack.Server.DataLayer
             return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, PrimaryKeyString FROM ArchiveDownloads")).ToHashSet();
         }
 
+        
+        public async Task<ArchiveDownload> GetArchiveDownload(Guid id)
+        {
+            await using var conn = await Open();
+            var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
+                "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE Id = @id",
+                new {Id = id});
+            if (result == default)
+                return null;
+
+            return new ArchiveDownload
+            {
+                Id = result.Item1,
+                IsFailed = result.Item4,
+                DownloadFinished = result.Item6,
+                Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
+            };
+
+        }
+
         public async Task<ArchiveDownload> GetNextPendingDownload(bool ignoreNexus = false)
         {
             await using var conn = await Open();

Wabbajack.Server/DataLayer/Patches.cs

@@ -0,0 +1,65 @@
+using System;
+using System.Threading.Tasks;
+using Dapper;
+using Wabbajack.Common;
+using Wabbajack.Server.DTOs;
+
+namespace Wabbajack.Server.DataLayer
+{
+    public partial class SqlService
+    {
+        /// <summary>
+        /// Adds a patch record
+        /// </summary>
+        /// <param name="patch"></param>
+        /// <returns></returns>
+        public async Task AddPatch(Patch patch)
+        {
+            await using var conn = await Open();
+            await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)",
+                new {SrcId = patch.Src.Id, DestId = patch.Dest.Id});
+        }
+        
+        /// <summary>
+        /// Adds a patch record
+        /// </summary>
+        /// <param name="patch"></param>
+        /// <returns></returns>
+        public async Task FinializePatch(Patch patch)
+        {
+            await using var conn = await Open();
+            await conn.ExecuteAsync("UPDATE dbo.Patches SET PatchSize = @Size, PatchHash = @PatchHash, Finished = @Finished WHERE SrcId = @SrcId AND DestID = @DestId",
+                new
+                {
+                    SrcId = patch.Src.Id, 
+                    DestId = patch.Dest.Id,
+                    PatchHash = patch.PatchHash,
+                    PatchSize = patch.PatchSize,
+                    Finshed = patch.Finished
+                });
+        }
+
+        public async Task<Patch> FindPatch(Guid src, Guid dest)
+        {
+            await using var conn = await Open();
+            var patch = await conn.QueryFirstOrDefaultAsync<(Hash, long, DateTime?)>(
+                "SELECT PatchHash, PatchSize, Finished FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId",
+                new
+                {
+                    SrcId = src,
+                    DestId = dest
+                });
+            if (patch == default)
+                return default(Patch);
+
+            return new Patch {
+                Src = await GetArchiveDownload(src), 
+                Dest = await GetArchiveDownload(dest),
+                PatchHash = patch.Item1,
+                PatchSize = patch.Item2,
+                Finished = patch.Item3
+            };
+
+        }
+    }
+}

Wabbajack.Server/Services/ArchiveDownloader.cs

@@ -67,6 +67,7 @@ namespace Wabbajack.Server.Services
                     
                     if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
                     {
+                        _logger.Log(LogLevel.Warning, $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}");
                         await nextDownload.Fail(_sql, "Invalid Hash");
                         continue;
                     }