Merge pull request #782 from wabbajack-tools/server-fixes

Print the archive being extracted when analysis fails.

Wabbajack.BuildServer.Test/IndexedFilesTests.cs

@@ -86,6 +86,52 @@ namespace Wabbajack.BuildServer.Test
             Assert.Null(await SQL.GetJob());
         }
 
+        [Fact]
+        public async Task CanGetGameFiles()
+        {
+            var sql = Fixture.GetService<SqlService>();
+            await sql.AddDownloadState(Hash.FromLong(1),
+                new GameFileSourceDownloader.State("1.2.3.4")
+                {
+                    Game = Game.SkyrimSpecialEdition, 
+                    Hash = Hash.FromLong(1),
+                    GameFile = (RelativePath)@"Data\foo.bsa",
+                });
+            await sql.AddDownloadState(Hash.FromLong(2),
+                new GameFileSourceDownloader.State("1.2.3.4")
+                {
+                    Game = Game.SkyrimSpecialEdition, 
+                    Hash = Hash.FromLong(2),
+                    GameFile = (RelativePath)@"Data\foo - Textures.bsa",
+                });
+
+            
+            await sql.AddDownloadState(Hash.FromLong(3),
+                new GameFileSourceDownloader.State("1.2.3.4")
+                {
+                    Game = Game.Skyrim, 
+                    Hash = Hash.FromLong(3),
+                    GameFile = (RelativePath)@"Data\foo - Textures.bsa",
+                });
+
+            await sql.AddDownloadState(Hash.FromLong(4),
+                new GameFileSourceDownloader.State("1.9.3.4")
+                {
+                    Game = Game.SkyrimSpecialEdition, 
+                    Hash = Hash.FromLong(4),
+                    GameFile = (RelativePath)@"Data\foo - Textures.bsa",
+                });
+
+            var results = await ClientAPI.GetGameFiles(Game.SkyrimSpecialEdition, Version.Parse("1.2.3.4"));
+            
+            Assert.Equal(new Dictionary<RelativePath, Hash>
+            {
+                {(RelativePath)@"Data\foo.bsa", Hash.FromLong(1)},
+                {(RelativePath)@"Data\foo - Textures.bsa", Hash.FromLong(2)},
+            }, results);
+
+        }
+
         public IndexedFilesTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
         {
             

Wabbajack.BuildServer.Test/UploadedFilesTest.cs

@@ -64,6 +64,28 @@ namespace Wabbajack.BuildServer.Test
             }
 
         }
+        
+        [Fact]
+        public async Task CanDeleteFilesUsingClientApi()
+        {
+            using (var file = new TempFile())
+            {
+                var data = new byte[1024];
+                await using (var fs = file.Path.Create())
+                {
+                    await fs.WriteAsync(data);
+                }
+
+                Utils.Log($"Uploading {file.Path.Size.ToFileSizeString()} file");
+                var result = await AuthorAPI.UploadFile(file.Path,
+                    progress => Utils.Log($"Uploading : {progress * 100}%"), Fixture.APIKey);
+
+                Utils.Log($"Delete {result}");
+                await AuthorAPI.DeleteFile((string)((RelativePath)new Uri(result).AbsolutePath).FileName);
+
+            }
+
+        }
 
         public UploadedFilesTest(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
         {

Wabbajack.BuildServer.Test/sql/wabbajack_db.sql

@@ -460,6 +460,33 @@ CREATE NONCLUSTERED INDEX [ByHash] ON [dbo].[DownloadStates]
     INCLUDE([IniState]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
 GO
 
+/****** Object:  View [dbo].[GameFiles]    Script Date: 4/30/2020 4:23:25 PM ******/
+
+CREATE VIEW [dbo].[GameFiles]
+            WITH SCHEMABINDING
+AS
+
+Select
+    Id,
+    CONVERT(NVARCHAR(20), JSON_VALUE(JsonState,'$.GameVersion')) as GameVersion,
+    CONVERT(NVARCHAR(32),JSON_VALUE(JsonState,'$.Game')) as Game,
+    JSON_VALUE(JsonState,'$.GameFile') as Path,
+    Hash as Hash
+FROM dbo.DownloadStates
+WHERE PrimaryKey like 'GameFileSourceDownloader+State|%'
+  AND JSON_VALUE(JsonState,'$.GameFile') NOT LIKE '%.xxhash'
+GO
+
+CREATE UNIQUE CLUSTERED INDEX [ByGameAndVersion] ON [dbo].[GameFiles]
+    (
+     [Game] ASC,
+     [GameVersion] ASC,
+     [Id] ASC
+        )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+GO
+
+
+
 /****** Object:  Index [IX_Child]    Script Date: 3/28/2020 4:58:59 PM ******/
 CREATE NONCLUSTERED INDEX [IX_Child] ON [dbo].[AllFilesInArchive]
     (

Wabbajack.BuildServer/Controllers/IndexedFiles.cs

@@ -131,6 +131,14 @@ namespace Wabbajack.BuildServer.Controllers
             return Ok(result);
         }
 
+        [HttpGet]
+        [Route("/game_files/{game}/{version}")]
+        public async Task<IActionResult> GetGameFiles(string game, string version)
+        {
+            var result = await _sql.GameFiles(GameRegistry.GetByFuzzyName(game).Game, Version.Parse(version));
+            return Ok(result.ToDictionary(k => k.Item1, k => k.Item2));
+        }
+
         public class TreeResult : IndexedFile
         {
             public List<TreeResult> ChildFiles { get; set; }

Wabbajack.BuildServer/Controllers/ListValidation.cs

@@ -114,10 +114,14 @@ namespace Wabbajack.BuildServer.Controllers
                         DownloadMetaData = metadata.DownloadMetadata,
                         HasFailures = failedCount > 0,
                         MachineName = metadata.Links.MachineURL,
-                        Archives = archives.Select(a => new DetailedStatusItem
+                        Archives = archives.Select(a =>
                         {
-                            Archive = a.Item1,
-                            IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
+                            a.Item1.Meta = "";
+                            return new DetailedStatusItem
+                            {
+                                Archive = a.Item1,
+                                IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
+                            };
                         }).ToList()
                     };
 
@@ -172,33 +176,59 @@ namespace Wabbajack.BuildServer.Controllers
             var mod = await SQL.GetNexusModInfoString(ns.Game, ns.ModID);
             var files = await SQL.GetModFiles(ns.Game, ns.ModID);
 
-            if (mod == null)
+            try
             {
-                Utils.Log($"Found missing Nexus mod info {ns.Game} {ns.ModID}");
-                mod = await (await _nexusClient).GetModInfo(ns.Game, ns.ModID, false);
-                try
+                if (mod == null)
                 {
-                    await SQL.AddNexusModInfo(ns.Game, ns.ModID, mod.updated_time, mod);
+                    Utils.Log($"Found missing Nexus mod info {ns.Game} {ns.ModID}");
+                    try
+                    {
+                        mod = await (await _nexusClient).GetModInfo(ns.Game, ns.ModID, false);
+                    }
+                    catch
+                    {
+                        mod = new ModInfo
+                        {
+                            mod_id = ns.ModID.ToString(), game_id = ns.Game.MetaData().NexusGameId, available = false
+                        };
+                    }
+
+                    try
+                    {
+                        await SQL.AddNexusModInfo(ns.Game, ns.ModID, mod.updated_time, mod);
+                    }
+                    catch (Exception _)
+                    {
+                        // Could be a PK constraint failure
+                    }
+
                 }
-                catch (Exception _)
+
+                if (files == null)
                 {
-                    // Could be a PK constraint failure
+                    Utils.Log($"Found missing Nexus mod file infos {ns.Game} {ns.ModID}");
+                    try
+                    {
+                        files = await (await _nexusClient).GetModFiles(ns.Game, ns.ModID, false);
+                    }
+                    catch
+                    {
+                        files = new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>()};
+                    }
+
+                    try
+                    {
+                        await SQL.AddNexusModFiles(ns.Game, ns.ModID, mod.updated_time, files);
+                    }
+                    catch (Exception _)
+                    {
+                        // Could be a PK constraint failure
+                    }
                 }
             }
-
-            if (files == null)
+            catch (Exception ex)
             {
-                Utils.Log($"Found missing Nexus mod file infos {ns.Game} {ns.ModID}");
-                files = await (await _nexusClient).GetModFiles(ns.Game, ns.ModID, false);
-
-                try
-                {
-                    await SQL.AddNexusModFiles(ns.Game, ns.ModID, mod.updated_time, files);
-                }
-                catch (Exception _)
-                {
-                    // Could be a PK constraint failure
-                }
+                return ArchiveStatus.InValid;
             }
 
             if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID))

Wabbajack.BuildServer/Controllers/UploadedFiles.cs

@@ -214,20 +214,24 @@ namespace Wabbajack.BuildServer.Controllers
         {
             var user = User.FindFirstValue(ClaimTypes.Name);
             Utils.Log($"Delete Uploaded File {user} {name}");
-            var files = await SQL.AllUploadedFilesForUser(name);
+            var files = await SQL.AllUploadedFilesForUser(user);
             
             var to_delete = files.First(f => f.MungedName == name);
             
             if (AlphaFile.Exists(Path.Combine("public", "files", to_delete.MungedName)))
                 AlphaFile.Delete(Path.Combine("public", "files", to_delete.MungedName));
 
-            using (var client = new FtpClient("storage.bunnycdn.com"))
-            {
-                client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password);
-                await client.ConnectAsync();
-                if (await client.FileExistsAsync(to_delete.MungedName))
-                    await client.DeleteFileAsync(to_delete.MungedName);
 
+            if (_settings.BunnyCDN_User != "TEST" || _settings.BunnyCDN_Password != "TEST")
+            {
+                using (var client = new FtpClient("storage.bunnycdn.com"))
+                {
+                    client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password);
+                    await client.ConnectAsync();
+                    if (await client.FileExistsAsync(to_delete.MungedName))
+                        await client.DeleteFileAsync(to_delete.MungedName);
+
+                }
             }
 
             await SQL.DeleteUploadedFile(to_delete.Id);
@@ -263,12 +267,7 @@ namespace Wabbajack.BuildServer.Controllers
                 files.Add(uf);
                 await SQL.AddUploadedFile(uf);
             }
-            
-
             return Ok(files.Count);
         }
-
-
-
     }
 }

Wabbajack.BuildServer/JobManager.cs

@@ -86,7 +86,7 @@ namespace Wabbajack.BuildServer
             while (true)
             {
                 await KillOrphanedJobs();
-                //await ScheduledJob<GetNexusUpdatesJob>(TimeSpan.FromHours(1), Job.JobPriority.High);
+                await ScheduledJob<GetNexusUpdatesJob>(TimeSpan.FromHours(1), Job.JobPriority.High);
                 //await ScheduledJob<UpdateModLists>(TimeSpan.FromMinutes(30), Job.JobPriority.High);
                 //await ScheduledJob<EnqueueAllArchives>(TimeSpan.FromHours(2), Job.JobPriority.Low);
                 //await ScheduledJob<EnqueueAllGameFiles>(TimeSpan.FromHours(24), Job.JobPriority.High);

Wabbajack.BuildServer/Models/Sql/SqlService.cs

@@ -138,6 +138,17 @@ namespace Wabbajack.BuildServer.Model.Models
             return Build(0).FirstOrDefault();
         }
 
+        public async Task<IEnumerable<(RelativePath, Hash)>> GameFiles(Game game, Version version)
+        {
+            await using var conn = await Open();
+            var files = await conn.QueryAsync<(RelativePath, Hash)>(
+                @"SELECT Path, Hash FROM dbo.GameFiles where Game = @Game AND GameVersion = @GameVersion",
+                new {Game = game.ToString(), GameVersion = version});
+
+            return files;
+
+        }
+
         public async Task IngestAllMetrics(IEnumerable<Metric> allMetrics)
         {
             await using var conn = await Open();
@@ -287,6 +298,8 @@ namespace Wabbajack.BuildServer.Model.Models
             SqlMapper.AddTypeHandler(new JsonMapper<AJobPayload>());
             SqlMapper.AddTypeHandler(new JsonMapper<JobResult>());
             SqlMapper.AddTypeHandler(new JsonMapper<Job>());
+            SqlMapper.AddTypeHandler(new VersionMapper());
+            SqlMapper.AddTypeHandler(new GameMapper());
         }
 
         public class JsonMapper<T> : SqlMapper.TypeHandler<T>
@@ -328,6 +341,32 @@ namespace Wabbajack.BuildServer.Model.Models
             }
         }
         
+        class VersionMapper : SqlMapper.TypeHandler<Version>
+        {
+            public override void SetValue(IDbDataParameter parameter, Version value)
+            {
+                parameter.Value = value.ToString();
+            }
+
+            public override Version Parse(object value)
+            {
+                return Version.Parse((string)value);
+            }
+        }
+        
+        class GameMapper : SqlMapper.TypeHandler<Game>
+        {
+            public override void SetValue(IDbDataParameter parameter, Game value)
+            {
+                parameter.Value = value.ToString();
+            }
+
+            public override Game Parse(object value)
+            {
+                return GameRegistry.GetByFuzzyName((string)value).Game;
+            }
+        }
+        
 
         #endregion
 

Wabbajack.Lib/ACompiler.cs

@@ -229,7 +229,6 @@ namespace Wabbajack.Lib
 
             result.Name = archive.Name;
             result.Hash = archive.File.Hash;
-            result.Meta = archive.Meta;
             result.Size = archive.File.Size;
 
             await result.State!.GetDownloader().Prepare();
@@ -238,6 +237,9 @@ namespace Wabbajack.Lib
                 Error(
                     $"Unable to resolve link for {archive.Name}. If this is hosted on the Nexus the file may have been removed.");
 
+            result.Meta = string.Join("\n", result.State!.GetMetaIni());
+
+            
             return result;
         }
 

Wabbajack.Lib/ClientAPI.cs

@@ -1,4 +1,6 @@
-using System.Threading.Tasks;
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Lib.Exceptions;
 
@@ -59,5 +61,11 @@ namespace Wabbajack.Lib
             return await GetClient()
                 .GetJsonAsync<NexusCacheStats>($"{Consts.WabbajackBuildServerUri}nexus_cache/stats");
         }
+
+        public static async Task<Dictionary<RelativePath, Hash>> GetGameFiles(Game game, Version version)
+        {
+            return await GetClient()
+                .GetJsonAsync<Dictionary<RelativePath, Hash>>($"{Consts.WabbajackBuildServerUri}game_files/{game}/{version}");
+        }
     }
 }

Wabbajack.Lib/FileUploader/AuthorAPI.cs

@@ -184,18 +184,18 @@ namespace Wabbajack.Lib.FileUploader
 
         public static async Task<string> GetServerLog()
         {
-            return await (await GetAuthorizedClient()).GetStringAsync($"https://{Consts.WabbajackCacheHostname}/heartbeat/logs");
+            return await (await GetAuthorizedClient()).GetStringAsync($"{Consts.WabbajackBuildServerUri}heartbeat/logs");
         }
 
         public static async Task<IEnumerable<string>> GetMyFiles()
         {
-            return (await (await GetAuthorizedClient()).GetStringAsync($"https://{Consts.WabbajackCacheHostname}/uploaded_files/list")).FromJsonString<string[]>();
+            return (await (await GetAuthorizedClient()).GetStringAsync($"{Consts.WabbajackBuildServerUri}uploaded_files/list")).FromJsonString<string[]>();
         }
 
         public static async Task<string> DeleteFile(string name)
         {
             var result = await (await GetAuthorizedClient())
-                .DeleteStringAsync($"https://{Consts.WabbajackCacheHostname}/uploaded_files/{name}");
+                .DeleteStringAsync($"{Consts.WabbajackBuildServerUri}uploaded_files/{name}");
             return result;
         }
     }