Merge pull request #751 from wabbajack-tools/2.0-release-day-bug-fixes

Bug fixes encountered during the 2.0 release
2024-08-30 18:42:17 +00:00 · 2020-04-27 06:16:46 -06:00 · 2020-04-27 06:16:46 -06:00 · d9612609e8
commit d9612609e8
parent e9586c38ac b475334d4b
17 changed files with 155 additions and 49 deletions
--- a/Compression.BSA/BSAReader.cs
+++ b/Compression.BSA/BSAReader.cs
@ -66,16 +66,16 @@ namespace Compression.BSA
        internal uint _totalFolderNameLength;
        internal uint _version;

-        public BSAReader(AbsolutePath filename) : this(filename.OpenRead())
+        public BSAReader(AbsolutePath filename)
        {
            _fileName = filename;
-        }
-
-        public BSAReader(Stream stream)
-        {
+            using var stream = filename.OpenRead();
+            using var br = new BinaryReader(stream);
+            _rdr = br;
            _stream = stream;
-            _rdr = new BinaryReader(_stream);
            LoadHeaders();
+            _rdr = null;
+            _stream = null;
        }

        public IEnumerable<IFile> Files
@ -116,7 +116,6 @@ namespace Compression.BSA

        public async ValueTask DisposeAsync()
        {
-            _stream.Close();
        }

        private void LoadHeaders()
--- a/Wabbajack.BuildServer.Test/MetricsTests.cs
+++ b/Wabbajack.BuildServer.Test/MetricsTests.cs
@ -30,5 +30,12 @@ namespace Wabbajack.BuildServer.Test

            Assert.Equal(subject, result);
        }
+
+        [Fact]
+        public async Task CanLoadMetricsFromSQL()
+        {
+            var sql = Fixture.GetService<SqlService>();
+            var results = await sql.MetricsReport("finish_install");
+        }
    }
 }
--- a/Wabbajack.BuildServer.Test/ModListValidationTests.cs
+++ b/Wabbajack.BuildServer.Test/ModListValidationTests.cs
@ -211,7 +211,7 @@ namespace Wabbajack.BuildServer.Test
            await using (var fs = modListPath.Create())
            {
                using var za = new ZipArchive(fs, ZipArchiveMode.Create);
-                var entry = za.CreateEntry("modlist.json");
+                var entry = za.CreateEntry("modlist");
                await using var es = entry.Open();
                ModListData.ToJson(es);
            }
--- a/Wabbajack.BuildServer/BackendServices/ListIngest.cs
+++ b/Wabbajack.BuildServer/BackendServices/ListIngest.cs
@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO.Compression;
 using System.Threading.Tasks;
+using Newtonsoft.Json;
 using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.Common;
 using Wabbajack.Lib;
@ -42,14 +43,23 @@ namespace Wabbajack.BuildServer.BackendServices
                ModList modlist;
                await using (var fs = modlistPath.OpenRead())
                using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
-                await using (var entry = zip.GetEntry("modlist.json")?.Open())
+                await using (var entry = zip.GetEntry("modlist")?.Open())
                {
                    if (entry == null)
                    {
                        Utils.Log($"Bad Modlist {list.Links.MachineURL}");
                        continue;
                    }
-                    modlist = entry.FromJson<ModList>();
+
+                    try
+                    {
+                        modlist = entry.FromJson<ModList>();
+                    }
+                    catch (JsonReaderException ex)
+                    {
+                        Utils.Log($"Bad JSON format for {list.Links.MachineURL}");
+                        continue;
+                    }
                }

                newData = true;
--- a/Wabbajack.BuildServer/Controllers/GraphQL.cs
+++ b/Wabbajack.BuildServer/Controllers/GraphQL.cs
@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging;
 using Wabbajack.BuildServer.GraphQL;
 using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.BuildServer.Models;
+using Wabbajack.Common;

 namespace Wabbajack.BuildServer.Controllers
 {
@ -35,7 +36,7 @@ namespace Wabbajack.BuildServer.Controllers

            if (result.Errors?.Count > 0)
            {
-                return BadRequest();
+                return BadRequest(result.Errors);
            }

            return Ok(result);
--- a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs
+++ b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs
@ -52,19 +52,25 @@ namespace Wabbajack.BuildServer.Controllers
            var fullPath = folder.RelativeTo((AbsolutePath)_settings.TempFolder);
            Utils.Log($"Ingesting Inis from {fullPath}");
            int loadCount = 0;
-            foreach (var file in fullPath.EnumerateFiles().Where(f => f.Extension == Consts.IniExtension))
-            {
-                var loaded = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile(), true));
-                if (loaded == null)
+            using var queue = new WorkQueue();
+            await fullPath.EnumerateFiles().Where(f => f.Extension == Consts.IniExtension)
+                    .PMap(queue, async file => {
+            
+                try
                {
-                    Utils.Log($"Unsupported Ini {file}");
-                    continue;
+                    var loaded =
+                        (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile(), true));
+
+                    var hash = Hash.FromHex(((string)file.FileNameWithoutExtension).Split("_").First());
+                    await SQL.AddDownloadState(hash, loaded);
+                }
+                catch (Exception ex)
+                {
+                    Utils.Log($"Failure for {file}");
                }

-                var hash = Hash.FromHex(((string)file.FileNameWithoutExtension).Split("_").First()); 
-                await SQL.AddDownloadState(hash, loaded);
                loadCount += 1;
-            }
+            });

            return Ok(loadCount);
        }
@ -82,7 +88,6 @@ namespace Wabbajack.BuildServer.Controllers
            {
                await using var ins = entry.Open();
                var iniString = Encoding.UTF8.GetString(await ins.ReadAllAsync());
-                Utils.Log(iniString);
                var data = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(iniString.LoadIniString(), true));
                
                if (data == null)
--- a/Wabbajack.BuildServer/Controllers/ListValidation.cs
+++ b/Wabbajack.BuildServer/Controllers/ListValidation.cs
@ -68,7 +68,10 @@ namespace Wabbajack.BuildServer.Controllers
                    var (_, result) = ValidateArchive(data, archive);
                    if (result == ArchiveStatus.InValid)
                    {
-                        return await TryToFix(data, archive);
+                        var fixResult = await TryToFix(data, archive);
+                        
+                        return fixResult;
+
                    }

                    return (archive, result);
@ -105,7 +108,7 @@ namespace Wabbajack.BuildServer.Controllers
            });

            
-            var cacheOptions = new MemoryCacheEntryOptions().SetSlidingExpiration(TimeSpan.FromMinutes(1));
+            var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1));
            Cache.Set(ModListSummariesKey, results, cacheOptions);
            return results;
        }
@ -142,6 +145,7 @@ namespace Wabbajack.BuildServer.Controllers
            var result = await _updater.GetAlternative(archive.Hash.ToHex());
            return result switch
            {
+                OkObjectResult ok => (archive, ArchiveStatus.Updated),
                OkResult ok => (archive, ArchiveStatus.Updated),
                AcceptedResult accept => (archive, ArchiveStatus.Updating),
                _ => (archive, ArchiveStatus.InValid)
--- a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs
+++ b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs
@ -216,7 +216,7 @@ namespace Wabbajack.BuildServer.Controllers
        private async Task<Archive> FindNexusAlternative(NexusDownloader.State state, Hash srcHash)
        {
            var origSize = _settings.PathForArchive(srcHash).Size;
-            var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
+            var api = await NexusApiClient.Get(Request?.Headers["apikey"].FirstOrDefault());
            var allMods = await api.GetModFiles(state.Game, state.ModID);
            var archive = allMods.files.Where(m => !string.IsNullOrEmpty(m.category_name))
                .OrderBy(s => Math.Abs((long)s.size - origSize))
--- a/Wabbajack.BuildServer/Controllers/NexusCache.cs
+++ b/Wabbajack.BuildServer/Controllers/NexusCache.cs
@ -41,7 +41,6 @@ namespace Wabbajack.BuildServer.Controllers
        [Route("{GameName}/mods/{ModId}.json")]
        public async Task<ModInfo> GetModInfo(string GameName, long ModId)
        {
-            Utils.Log($"Nexus Mod Info {GameName} {ModId}");
            var game = GameRegistry.GetByFuzzyName(GameName).Game;
            var result = await SQL.GetNexusModInfoString(game, ModId);
            
@ -69,8 +68,6 @@ namespace Wabbajack.BuildServer.Controllers
        [Route("{GameName}/mods/{ModId}/files.json")]
        public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, long ModId)
        {
-            Utils.Log($"Nexus Mod Files {GameName} {ModId}");
-
            var game = GameRegistry.GetByFuzzyName(GameName).Game;
            var result = await SQL.GetModFiles(game, ModId);

--- a/Wabbajack.BuildServer/GraphQL/Query.cs
+++ b/Wabbajack.BuildServer/GraphQL/Query.cs
@ -8,6 +8,7 @@ namespace Wabbajack.BuildServer.GraphQL
    {
        public Query(SqlService sql)
        {
+            /*
            FieldAsync<ListGraphType<ModListStatusType>>("modLists",
                arguments: new QueryArguments(new QueryArgument<ArchiveEnumFilterType>
                {
@ -28,7 +29,7 @@ namespace Wabbajack.BuildServer.GraphQL
                    }

                });
-
+*/
            FieldAsync<ListGraphType<MetricResultType>>("dailyUniqueMetrics",
                arguments: new QueryArguments(
                    new QueryArgument<MetricEnum> {Name = "metric_type", Description = "The grouping of metric data to query"}
--- a/Wabbajack.BuildServer/JobManager.cs
+++ b/Wabbajack.BuildServer/JobManager.cs
@ -1,9 +1,11 @@
 using System;
 using System.Linq;
 using System.Reactive.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Nettle;
+using Wabbajack.BuildServer.BackendServices;
 using Wabbajack.BuildServer.Controllers;
 using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.BuildServer.Models;
@ -77,6 +79,7 @@ namespace Wabbajack.BuildServer
            if (!Settings.JobScheduler) return;

            var task = RunNexusCacheLoop();
+            var listIngest = (new ListIngest(Sql, Settings)).RunLoop(CancellationToken.None);
            
            while (true)
            {
@ -99,6 +102,7 @@ namespace Wabbajack.BuildServer
            }
        }

+
        private async Task KillOrphanedJobs()
        {
            try
@ -126,7 +130,7 @@ namespace Wabbajack.BuildServer
            if (!Settings.RunFrontEndJobs && typeof(T).ImplementsInterface(typeof(IFrontEndJob))) return;
            try
            {
-                var jobs = (await Sql.GetUnfinishedJobs())
+                var jobs = (await Sql.GetAllJobs(span))
                    .Where(j => j.Payload is T)
                    .OrderByDescending(j => j.Created)
                    .Take(10);
@ -134,7 +138,7 @@ namespace Wabbajack.BuildServer
                foreach (var job in jobs)
                {
                    if (job.Started == null || job.Ended == null) return;
-                    if (DateTime.Now - job.Ended < span) return;
+                    if (DateTime.UtcNow - job.Ended < span) return;
                }
                await Sql.EnqueueJob(new Job
                {
--- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs
+++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs
@ -167,7 +167,7 @@ namespace Wabbajack.BuildServer.Model.Models
                        AND DATEADD(DAY, number+1, dbo.MinMetricDate()) < dbo.MaxMetricDate()) as d
                        ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action
                        WHERE d.Action = @action
-                        AND d.Date >= DATEADD(month, -1, GETDATE())
+                        AND d.Date >= DATEADD(month, -1, GETUTCDATE())
                        group by d.Date, d.GroupingSubject, d.Action
                        ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action}))
                .ToList();
@ -184,7 +184,7 @@ namespace Wabbajack.BuildServer.Model.Models
        {
            await using var conn = await Open();
            await conn.ExecuteAsync(
-                @"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)",
+                @"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETUTCDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)",
                new {
                    job.Priority,
                    PrimaryKeyString = job.Payload.PrimaryKeyString,
@ -201,7 +201,7 @@ namespace Wabbajack.BuildServer.Model.Models
        {
            await using var conn = await Open();
            await conn.ExecuteAsync(
-                @"UPDATE dbo.Jobs SET Ended = GETDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id",
+                @"UPDATE dbo.Jobs SET Ended = GETUTCDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id",
                new {
                    job.Id,
                    Success = job.Result.ResultType == JobResultType.Success,
@ -221,7 +221,7 @@ namespace Wabbajack.BuildServer.Model.Models
        {
            await using var conn = await Open();
            var result = await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>(
-                @"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy 
+                @"UPDATE jobs SET Started = GETUTCDATE(), RunBy = @RunBy 
                        WHERE ID in (SELECT TOP(1) ID FROM Jobs 
                                       WHERE Started is NULL
                                       AND PrimaryKeyString NOT IN (SELECT PrimaryKeyString from jobs WHERE Started IS NOT NULL and Ended IS NULL)
@ -229,14 +229,14 @@ namespace Wabbajack.BuildServer.Model.Models
                      SELECT TOP(1) Id, Started, Ended, Created, Payload, Priority FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC",
                new {RunBy = Guid.NewGuid().ToString()});
            return result.Select(k =>
-            new Job {
-                Id = k.Item1,
-                Started = k.Item2,
-                Ended = k.Item3,
-                Created = k.Item4,
-                Payload = k.Item5,
-                Priority = (Job.JobPriority)k.Item6
-            }).FirstOrDefault();
+                new Job {
+                    Id = k.Item1,
+                    Started = k.Item2,
+                    Ended = k.Item3,
+                    Created = k.Item4,
+                    Payload = k.Item5,
+                    Priority = (Job.JobPriority)k.Item6
+                }).FirstOrDefault();
    }
        
        
@ -244,8 +244,16 @@ namespace Wabbajack.BuildServer.Model.Models
        {
            await using var conn = await Open();
            var results =
-                await conn.QueryAsync<Job>("SELECT * from dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL ");
-            return results;
+                await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority FROM dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL ");
+            return results.Select(k =>
+                new Job {
+                    Id = k.Item1,
+                    Started = k.Item2,
+                    Ended = k.Item3,
+                    Created = k.Item4,
+                    Payload = k.Item5,
+                    Priority = (Job.JobPriority)k.Item6
+                });
        }


@ -253,8 +261,16 @@ namespace Wabbajack.BuildServer.Model.Models
        {
            await using var conn = await Open();
            var results =
-                await conn.QueryAsync<Job>("SELECT * from dbo.Jobs WHERE Ended IS NULL ");
-            return results;
+                await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority from dbo.Jobs WHERE Ended IS NULL ");
+            return results.Select(k =>
+                new Job {
+                    Id = k.Item1,
+                    Started = k.Item2,
+                    Ended = k.Item3,
+                    Created = k.Item4,
+                    Payload = k.Item5,
+                    Priority = (Job.JobPriority)k.Item6
+                });
        }

        
@ -353,7 +369,7 @@ namespace Wabbajack.BuildServer.Model.Models
        public async Task<IEnumerable<UploadedFile>> AllUploadedFiles()
        {
            await using var conn = await Open();
-            return await conn.QueryAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles ORDER BY UploadDate DESC");
+            return await conn.QueryAsync<UploadedFile>("SELECT Id, Name, Size, UploadedBy as Uploader, Hash, UploadDate, CDNName FROM dbo.UploadedFiles ORDER BY UploadDate DESC");
        }
        
        public async Task DeleteUploadedFile(Guid dupId)
@ -853,5 +869,22 @@ namespace Wabbajack.BuildServer.Model.Models
        

        #endregion
+
+        public async Task<IEnumerable<Job>> GetAllJobs(TimeSpan from)
+        {
+            await using var conn = await Open();
+            var results =
+                await conn.QueryAsync<(long, DateTime, DateTime, DateTime, AJobPayload, int)>("SELECT Id, Started, Ended, Created, Payload, Priority from dbo.Jobs WHERE Created >= @FromTime ",
+                    new {FromTime = DateTime.UtcNow - from});
+            return results.Select(k =>
+                new Job {
+                    Id = k.Item1,
+                    Started = k.Item2,
+                    Ended = k.Item3,
+                    Created = k.Item4,
+                    Payload = k.Item5,
+                    Priority = (Job.JobPriority)k.Item6
+                });
+        }
    }
 }
--- a/Wabbajack.Common/Hash.cs
+++ b/Wabbajack.Common/Hash.cs
@ -5,6 +5,7 @@ using System.IO;
 using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Tasks;
+using Newtonsoft.Json;
 using File = Alphaleonis.Win32.Filesystem.File;
 using Path = Alphaleonis.Win32.Filesystem.Path;

@ -15,6 +16,7 @@ namespace Wabbajack.Common
    /// Struct representing a xxHash64 value. It's a struct with a ulong in it, but wrapped so we don't confuse
    /// it with other longs in the system.
    /// </summary>
+    [JsonConverter(typeof(Utils.HashJsonConverter))]
    public struct Hash
    {
        private readonly ulong _code;
--- a/Wabbajack.Common/Json.cs
+++ b/Wabbajack.Common/Json.cs
@ -84,7 +84,7 @@ namespace Wabbajack.Common
            return result;
        }

-        private class HashJsonConverter : JsonConverter<Hash>
+        public class HashJsonConverter : JsonConverter<Hash>
        {
            public override void WriteJson(JsonWriter writer, Hash value, JsonSerializer serializer)
            {
@ -98,7 +98,7 @@ namespace Wabbajack.Common
            }
        }

-        private class RelativePathConverter : JsonConverter<RelativePath>
+        public class RelativePathConverter : JsonConverter<RelativePath>
        {
            public override void WriteJson(JsonWriter writer, RelativePath value, JsonSerializer serializer)
            {
--- a/Wabbajack.Common/Paths.cs
+++ b/Wabbajack.Common/Paths.cs
@ -9,6 +9,7 @@ using System.Reflection;
 using System.Text;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
+using Newtonsoft.Json;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
 using File = Alphaleonis.Win32.Filesystem.File;
 using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
@ -405,6 +406,7 @@ namespace Wabbajack.Common
        }
    }

+    [JsonConverter(typeof(Utils.RelativePathConverter))]
    public struct RelativePath : IPath, IEquatable<RelativePath>, IComparable<RelativePath>
    {
        private readonly string? _nullable_path;
--- a/Wabbajack.Test/SanityTests.cs
+++ b/Wabbajack.Test/SanityTests.cs
@ -295,6 +295,41 @@ namespace Wabbajack.Test
            
        }
        
+        [Fact]
+        public async Task CanInstallFilesFromBSAAndBSA()
+        {
+            var profile = utils.AddProfile();
+            var mod = utils.AddMod();
+            var file = utils.AddModFile(mod, @"baz.bin", 128);
+            
+            await utils.Configure();
+
+            
+            using var tempFile = new TempFile();
+            var bsaState = new BSAStateObject
+            {
+                Magic = "BSA\0", Version = 0x69, ArchiveFlags = 0x107, FileFlags = 0x0,
+            };
+
+            await using (var bsa = bsaState.MakeBuilder(1024 * 1024))
+            {
+                await bsa.AddFile(new BSAFileStateObject
+                {
+                    Path = (RelativePath)@"foo\bar\baz.bin", Index = 0, FlipCompression = false
+                }, new MemoryStream(await file.ReadAllBytesAsync()));
+                await bsa.Build(tempFile.Path);
+            }
+            tempFile.Path.CopyTo(file.Parent.Combine("bsa_data.bsa"));
+            
+            var archive = utils.AddManualDownload(
+                new Dictionary<string, byte[]> { { "/stuff/files.bsa", await tempFile.Path.ReadAllBytesAsync() } });
+            
+            await CompileAndInstall(profile);
+            utils.VerifyInstalledFile(mod, @"baz.bin");
+            utils.VerifyInstalledFile(mod, @"bsa_data.bsa");
+            
+        }
+        
        [Fact]
        public async Task CanRecreateBSAsFromFilesSourcedInOtherBSAs()
        {
--- a/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs
+++ b/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs
@ -40,6 +40,12 @@ namespace Wabbajack.VirtualFileSystem

        public async Task MoveTo(AbsolutePath path)
        {
+            if (FileExtractor.MightBeArchive(_path.Extension))
+            {
+                path.Parent.CreateDirectory();
+                await _path.CopyToAsync(path);
+                return;
+            }
            await _path.MoveToAsync(path, true);
            _path = path;
        }