diff --git a/CHANGELOG.md b/CHANGELOG.md index 4240f05e..0484c090 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ ### Changelog +#### Version - 2.1.0.2 - 7/7/2020 +* Don't scan the game folder during compilation. If you are compiling and want `Game Folder Files` create it in your MO2 folder and manually place files into it +* Don't throw a hard error on a post-patch hash failure. +* Don't save the VFS cache to disk or load it during compilation. We have other caches that make this mostly worthless + #### Version - 2.1.0.1 - 7/6/2020 * Don't include saves in .wabbajack files * Don't delete saves from any MO2 profile during installation diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index cd4b5554..52bcf6ec 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -6,8 +6,8 @@ wabbajack-cli Wabbajack x64 - 2.1.0.1 - 2.1.0.1 + 2.1.0.2 + 2.1.0.2 Copyright © 2019-2020 An automated ModList installer true diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj index 662f1e8f..424030f9 100644 --- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj +++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj @@ -4,8 +4,8 @@ WinExe netcoreapp3.1 true - 2.1.0.1 - 2.1.0.1 + 2.1.0.2 + 2.1.0.2 Copyright © 2019-2020 Wabbajack Application Launcher true diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index d7107480..9c596c87 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -221,7 +221,9 @@ namespace Wabbajack.Lib Status($"Verifying Patch {toPatch.To.FileName}"); hash = await toFile.FileHashAsync(); if (hash != toPatch.Hash) - throw new InvalidDataException($"Invalid Hash for {toPatch.To} after patching"); + { + Utils.Log($"NOTE: Invalid Hash for {toPatch.To} after patching {hash} vs {toPatch.Hash}"); + } }); } diff --git a/Wabbajack.Lib/Data.cs b/Wabbajack.Lib/Data.cs index b3a26413..c5552621 100644 --- a/Wabbajack.Lib/Data.cs +++ b/Wabbajack.Lib/Data.cs @@ -39,7 +39,7 @@ namespace Wabbajack.Lib { var v = new T(); v.To = Path; - v.Hash = Hash; + v.Hash = File.Hash; v.Size = File.Size; return v; } diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index b18f16ba..f54ce672 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -104,9 +104,6 @@ namespace Wabbajack.Lib if (cancel.IsCancellationRequested) return false; - if (VFSCacheName.Exists) - await VFS.IntegrateFromFile(VFSCacheName); - List roots; if (UseGamePaths) { @@ -138,7 +135,6 @@ namespace Wabbajack.Lib if (cancel.IsCancellationRequested) return false; await VFS.AddRoots(roots); - await VFS.WriteToFile(VFSCacheName); if (lootPath.Exists) { @@ -183,8 +179,7 @@ namespace Wabbajack.Lib if (cancel.IsCancellationRequested) return false; UpdateTracker.NextStep("Reindexing downloads after meta inferring"); await VFS.AddRoot(MO2DownloadsFolder); - await VFS.WriteToFile(VFSCacheName); - + if (cancel.IsCancellationRequested) return false; UpdateTracker.NextStep("Pre-validating Archives"); @@ -246,27 +241,12 @@ namespace Wabbajack.Lib }); // If Game Folder Files exists, ignore the game folder - IEnumerable gameFiles; - if (!MO2Folder.Combine(Consts.GameFolderFilesDir).Exists) - { - gameFiles = GamePath.EnumerateFiles() - .Where(p => p.IsFile) - .Where(p => p.Extension!= Consts.HashFileExtension) - .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], - Consts.GameFolderFilesDir.Combine(p.RelativeTo(GamePath)))); - } - else - { - gameFiles = new List(); - } - - IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren) .OrderBy(f => f.NestingFactor) .GroupBy(f => f.Hash) .ToDictionary(f => f.Key, f => f.AsEnumerable()); - AllFiles.SetTo(mo2Files.Concat(gameFiles) + AllFiles.SetTo(mo2Files .Concat(lootFiles) .DistinctBy(f => f.Path)); diff --git a/Wabbajack.Server/Controllers/Metrics.cs b/Wabbajack.Server/Controllers/Metrics.cs index f7f03b2d..e371e078 100644 --- a/Wabbajack.Server/Controllers/Metrics.cs +++ b/Wabbajack.Server/Controllers/Metrics.cs @@ -54,27 +54,30 @@ namespace Wabbajack.BuildServer.Controllers } [HttpGet] - [Route("badge/{name}/badge.json")] - public async Task MetricsGitHubBadge(string name) + [Route("badge/{name}/total_installs_badge.json")] + public async Task TotalInstallsBadge(string name) { - var results = (await _sql.MetricsReport("finish_install")) - .GroupBy(m => m.Subject) - .Select(g => new MetricResult - { - SeriesName = g.Key, - Labels = g.Select(m => m.Date.ToString(CultureInfo.InvariantCulture)).ToList(), - Values = g.Select(m => m.Count).ToList() - }).ToList(); + var results = await _sql.TotalInstalls(name); + + Response.ContentType = "application/json"; + + return Ok(results == 0 + ? new Badge($"Modlist {name} not found!", "Error") {color = "red"} + : new Badge("Installations: ", $"{results}") {color = "green"}); + } + + [HttpGet] + [Route("badge/{name}/unique_installs_badge.json")] + public async Task UniqueInstallsBadge(string name) + { + var results = await _sql.UniqueInstalls(name); Response.ContentType = "application/json"; - var modlist = - results.FirstOrDefault(x => x.SeriesName.Equals(name, StringComparison.InvariantCultureIgnoreCase)); - - return Ok(modlist == null - ? new Badge($"Modlist {name} not found!", "Error") {color = "red"}.ToJson() - : new Badge("Installations: ", $"{modlist.Values.Aggregate((x, y) => x + y)}").ToJson()); - } + return Ok(results == 0 + ? new Badge($"Modlist {name} not found!", "Error") {color = "red"} + : new Badge("Installations: ", $"{results}"){color = "green"}) ; + } private static readonly Func ReportTemplate = NettleEngine.GetCompiler().Compile(@" diff --git a/Wabbajack.Server/DataLayer/Metrics.cs b/Wabbajack.Server/DataLayer/Metrics.cs index 57d63698..1d6d5e7a 100644 --- a/Wabbajack.Server/DataLayer/Metrics.cs +++ b/Wabbajack.Server/DataLayer/Metrics.cs @@ -72,5 +72,27 @@ namespace Wabbajack.Server.DataLayer return (await conn.QueryAsync("SELECT TOP(1) MetricsKey from Metrics Where MetricsKey = @MetricsKey", new {MetricsKey = metricsKey})).FirstOrDefault() != null; } + + + public async Task UniqueInstalls(string machineUrl) + { + await using var conn = await Open(); + return await conn.QueryFirstAsync( + @"SELECT COUNT(*) FROM ( + SELECT DISTINCT MetricsKey from dbo.Metrics where Action = 'finish_install' and GroupingSubject in ( + SELECT JSON_VALUE(Metadata, '$.title') FROM dbo.ModLists + WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)) s", + new {MachineURL = machineUrl}); + } + + public async Task TotalInstalls(string machineUrl) + { + await using var conn = await Open(); + return await conn.QueryFirstAsync( + @"SELECT COUNT(*) from dbo.Metrics where Action = 'finish_install' and GroupingSubject in ( + SELECT JSON_VALUE(Metadata, '$.title') FROM dbo.ModLists + WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)", + new {MachineURL = machineUrl}); + } } } diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index 3267c0a8..fb97b132 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -128,14 +128,14 @@ namespace Wabbajack.Server.Services foreach (var patch in patches) { _logger.LogInformation($"Cleaning patch {patch.Src.Archive.Hash} -> {patch.Dest.Archive.Hash}"); - /* + await _discordWebHook.Send(Channel.Ham, new DiscordMessage { Content = $"Removing patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString} due it no longer being required by curated lists" }); -*/ + if (!await DeleteFromCDN(client, PatchName(patch))) { _logger.LogWarning($"Patch file didn't exist {PatchName(patch)}"); diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index a7959d35..ae610d28 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -3,8 +3,8 @@ Exe netcoreapp3.1 - 2.1.0.1 - 2.1.0.1 + 2.1.0.2 + 2.1.0.2 Copyright © 2019-2020 Wabbajack Server win-x64 diff --git a/Wabbajack.Test/SanityTests.cs b/Wabbajack.Test/SanityTests.cs index af5c0f13..a3495c9e 100644 --- a/Wabbajack.Test/SanityTests.cs +++ b/Wabbajack.Test/SanityTests.cs @@ -9,6 +9,7 @@ using Wabbajack.Lib; using Wabbajack.Lib.CompilationSteps.CompilationErrors; using Xunit; using Xunit.Abstractions; +using Xunit.Sdk; using File = Alphaleonis.Win32.Filesystem.File; using Path = Alphaleonis.Win32.Filesystem.Path; @@ -63,7 +64,7 @@ namespace Wabbajack.Test [Fact] public async Task TestDirectMatchFromGameFolder() { - + // This code is disabled, but we'll still test for it in case it somehow gets re-enabled in the future var profile = utils.AddProfile(); var mod = await utils.AddMod(); var testPex = await utils.AddGameFile(@"enbstuff\test.pex", 10); @@ -75,7 +76,8 @@ namespace Wabbajack.Test await CompileAndInstall(profile, useGameFiles: true); - await utils.VerifyInstalledGameFile(@"enbstuff\test.pex"); + // This should fail + await Assert.ThrowsAsync(async () => await utils.VerifyInstalledGameFile(@"enbstuff\test.pex")); } [Fact] @@ -533,6 +535,8 @@ namespace Wabbajack.Test await utils.VerifyInstalledFile(mod, @"Data\SkyrimSE\Update.esm.old"); await utils.VerifyInstalledFile(mod, @"Data\SkyrimSE\Update.esm"); + Assert.False(utils.InstallFolder.Combine(Consts.GameFolderFilesDir).IsDirectory); + } diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index f6839657..f79e7cb1 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -356,7 +356,10 @@ namespace Wabbajack.VirtualFileSystem public async Task Integrate(ICollection files) { Utils.Log($"Integrating {files.Count} files"); - var allFiles = AllFiles.Concat(files).GroupBy(f => f.FullPath).Select(g => g.Last()).ToImmutableList(); + var allFiles = AllFiles.Concat(files) + .OrderByDescending(f => f.LastModified) + .GroupBy(f => f.FullPath).Select(g => g.Last()) + .ToImmutableList(); var byFullPath = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) .ToDictionary(f => f.FullPath)); diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 54eb1d87..f9376227 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -6,8 +6,8 @@ true x64 win10-x64 - 2.1.0.1 - 2.1.0.1 + 2.1.0.2 + 2.1.0.2 Copyright © 2019-2020 An automated ModList installer true