Code for healing MEGA links

CHANGELOG.md

@@ -1,5 +1,11 @@
 ### Changelog
 
+#### Version - 2.1.2.0 - 7/13/2020
+* Can heal hand selected MEGA files
+* Several backend fixes
+* Reworked the ChangeDownload CLI command
+* Fix for a VFS cache error when compiling lists that extract BSAs.
+
 #### Version - 2.1.1.0 - 7/10/2020
 * New CLI option for clearing nexus cache entries (authors only)
 * Retry failed Move commands

Wabbajack.CLI/OptionsDefinition.cs

@@ -25,7 +25,8 @@ namespace Wabbajack.CLI
             typeof(HashFile),
             typeof(InlinedFileReport),
             typeof(ExtractBSA),
-            typeof(PurgeNexusCache)
+            typeof(PurgeNexusCache),
+            typeof(ForceHealing)
         };
     }
 }

Wabbajack.CLI/Verbs/ForceHealing.cs

@@ -0,0 +1,40 @@
+using System.Threading.Tasks;
+using CommandLine;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+
+namespace Wabbajack.CLI.Verbs
+{
+    [Verb("force-healing", HelpText = "Forces a given source download to be healed by a given new-er download. The new download must be valid.")]
+    public class ForceHealing : AVerb
+    {
+        [Option('o', "old", Required = true, HelpText = "Old Archive (must have an attached .meta)")]
+        public string _old { get; set; } = "";
+
+        public AbsolutePath Old => (AbsolutePath)_old;
+        [Option('n', "new", Required = true, HelpText = "New Archive (must have an attached .meta)")]
+        public string _new { get; set; } = "";
+        public AbsolutePath New => (AbsolutePath)_new;
+        
+        
+        protected override async Task<ExitCode> Run()
+        {
+            Utils.Log("Loading Meta files");
+            var oldState = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(Old.WithExtension(Consts.MetaFileExtension).LoadIniFile());
+            var newState = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(New.WithExtension(Consts.MetaFileExtension).LoadIniFile());
+            Utils.Log("Hashing archives");
+            
+            var oldHash = await Old.FileHashCachedAsync();
+            var newHash = await New.FileHashCachedAsync();
+
+            var oldArchive = new Archive(oldState) {Hash = oldHash, Size = Old.Size};
+            var newArchive = new Archive(newState) {Hash = newHash, Size = New.Size};
+
+            Utils.Log($"Contacting Server to request patch ({oldHash} -> {newHash}");
+            Utils.Log($"Response: {await ClientAPI.GetModUpgrade(oldArchive, newArchive, useAuthor: true)}");
+
+            return ExitCode.Ok;
+        }
+    }
+}

Wabbajack.CLI/Wabbajack.CLI.csproj

@@ -6,8 +6,8 @@
         <AssemblyName>wabbajack-cli</AssemblyName>
         <Company>Wabbajack</Company>
         <Platforms>x64</Platforms>
-        <AssemblyVersion>2.1.1.0</AssemblyVersion>
+        <AssemblyVersion>2.1.2.0</AssemblyVersion>
-        <FileVersion>2.1.1.0</FileVersion>
+        <FileVersion>2.1.2.0</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>An automated ModList installer</Description>
         <PublishReadyToRun>true</PublishReadyToRun>        

Wabbajack.Launcher/Wabbajack.Launcher.csproj

@@ -4,8 +4,8 @@
         <OutputType>WinExe</OutputType>
         <TargetFramework>netcoreapp3.1</TargetFramework>
         <UseWPF>true</UseWPF>
-        <AssemblyVersion>2.1.1.0</AssemblyVersion>
+        <AssemblyVersion>2.1.2.0</AssemblyVersion>
-        <FileVersion>2.1.1.0</FileVersion>
+        <FileVersion>2.1.2.0</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Application Launcher</Description>
         <PublishReadyToRun>true</PublishReadyToRun>

Wabbajack.Lib/ClientAPI.cs

@@ -25,21 +25,18 @@ using Wabbajack.Lib.Downloaders;
             NewArchive = newArchive;
         }
 
-        public bool IsValid
+        public async Task<bool> IsValid()
         {
-            get
+            if (OldArchive.Size > 2_500_000_000 || NewArchive.Size > 2_500_000_000) return false;
-            {
+            if (OldArchive.Hash == NewArchive.Hash && OldArchive.State.PrimaryKeyString == NewArchive.State.PrimaryKeyString) return false;
-                if (OldArchive.Size > 2_500_000_000 || NewArchive.Size > 2_500_000_000) return false;
+            if (OldArchive.State.GetType() != NewArchive.State.GetType())
-                if (OldArchive.Hash == NewArchive.Hash && OldArchive.State.PrimaryKeyString == NewArchive.State.PrimaryKeyString) return false;
-                if (OldArchive.State.GetType() != NewArchive.State.GetType())
-                    return false;
-                if (OldArchive.State is IUpgradingState u)
-                {
-                    return u.ValidateUpgrade(NewArchive.State);
-                }
-
                 return false;
+            if (OldArchive.State is IUpgradingState u)
+            {
+                return await u.ValidateUpgrade(OldArchive.Hash, NewArchive.State);
             }
+
+            return false;
         }
     }
     
@@ -52,7 +49,7 @@ using Wabbajack.Lib.Downloaders;
             return client;
         }
 
-        public static async Task<Uri> GetModUpgrade(Archive oldArchive, Archive newArchive, TimeSpan? maxWait = null, TimeSpan? waitBetweenTries = null)
+        public static async Task<Uri> GetModUpgrade(Archive oldArchive, Archive newArchive, TimeSpan? maxWait = null, TimeSpan? waitBetweenTries = null, bool useAuthor = false)
         {
             maxWait ??= TimeSpan.FromMinutes(10);
             waitBetweenTries ??= TimeSpan.FromSeconds(15);
@@ -62,7 +59,7 @@ using Wabbajack.Lib.Downloaders;
             
             RETRY:
             
-            var response = await (await GetClient())
+            var response = await (useAuthor ? await AuthorApi.Client.GetAuthorizedClient() : await GetClient())
                 .PostAsync($"{Consts.WabbajackBuildServerUri}mod_upgrade", new StringContent(request.ToJson(), Encoding.UTF8, "application/json"));
 
             if (response.IsSuccessStatusCode)
@@ -141,5 +138,13 @@ using Wabbajack.Lib.Downloaders;
             }
             return null;
         }
+
+        public static async Task<Archive[]> GetModUpgrades(Hash src)
+        {
+            var client = await GetClient();
+            Utils.Log($"Looking for generic upgrade for {src} ({(long)src})");
+            var results = await client.GetJsonAsync<Archive[]>($"{Consts.WabbajackBuildServerUri}mod_upgrade/find/{src.ToHex()}");
+            return results;
+        }
     }
 }

Wabbajack.Lib/Downloaders/AbstractDownloadState.cs

@@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders
         Task<bool> LoadMetaData();
     }
 
-    public abstract class AbstractDownloadState
+    public abstract class AbstractDownloadState : IUpgradingState
     {
         public static List<Type> KnownSubTypes = new List<Type>
         {
@@ -102,5 +102,61 @@ namespace Wabbajack.Lib.Downloaders
         {
             return string.Join("\n", GetMetaIni());
         }
+
+        public async Task<(Archive? Archive, TempFile NewFile)> ServerFindUpgrade(Archive a)
+        {
+            var alternatives = await ClientAPI.GetModUpgrades(a.Hash);
+            if (alternatives == default)
+                return default;
+
+
+            await DownloadDispatcher.PrepareAll(alternatives.Select(r => r.State));
+            Archive? selected = null;
+            foreach (var result in alternatives)
+            {
+                try
+                {
+                    if (!await result.State.Verify(result)) continue;
+
+                    selected = result;
+                    break;
+                }
+                catch (Exception ex)
+                {
+                    Utils.Log($"Verification error for failed for possible upgrade {result.State.PrimaryKeyString}");
+                    Utils.Log(ex.ToString());
+                }
+            }
+
+            if (selected == null) return default;
+
+            var tmpFile = new TempFile();
+            if (await selected.State.Download(selected, tmpFile.Path))
+            {
+                return (selected, tmpFile);
+            }
+
+            await tmpFile.DisposeAsync();
+            return default;
+
+        }
+
+        public virtual async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
+        {
+            return await ServerFindUpgrade(a);
+        }
+
+        public virtual async Task<bool> ServerValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState)
+        {
+            var alternatives = await ClientAPI.GetModUpgrades(srcHash);
+            return alternatives?.Any(a => a.State.PrimaryKeyString == newArchiveState.PrimaryKeyString) ?? default;
+        }
+
+        public virtual async Task<bool> ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState)
+        {
+            return await ServerValidateUpgrade(srcHash, newArchiveState);
+        }
+        
+
     }
 }

Wabbajack.Lib/Downloaders/HTTPDownloader.cs

@@ -240,7 +240,7 @@ TOP:
 
             }
 
-            public bool ValidateUpgrade(AbstractDownloadState newArchiveState)
+            public override async Task<bool> ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState)
             {
                 var httpState = (State)newArchiveState;
 

Wabbajack.Lib/Downloaders/IUpgradingState.cs

@@ -13,6 +13,6 @@ namespace Wabbajack.Lib.Downloaders
         /// <returns></returns>
         public Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a);
 
-        bool ValidateUpgrade(AbstractDownloadState newArchiveState);
+        Task<bool> ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState);
     }
 }

Wabbajack.Lib/Downloaders/MEGADownloader.cs

@@ -194,10 +194,15 @@ namespace Wabbajack.Lib.Downloaders
                 }
             }
 
-            public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
+            public override Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
             {
-                return default;
+                return ServerFindUpgrade(a);
             }
+            
+            public override async Task<bool> ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState)
+            {
+                return await ServerValidateUpgrade(srcHash, newArchiveState);
+            }                
         }
 
         public ReactiveCommand<Unit, Unit> TriggerLogin { get; }

Wabbajack.Lib/Downloaders/NexusDownloader.cs

@@ -277,7 +277,7 @@ namespace Wabbajack.Lib.Downloaders
                 return (newArchive, tempFile);
             }
 
-            public bool ValidateUpgrade(AbstractDownloadState newArchiveState)
+            public async Task<bool> ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState)
             {
                 var state = (State)newArchiveState;
                 return Game == state.Game && ModID == state.ModID;

Wabbajack.Lib/Metrics.cs

@@ -21,6 +21,7 @@ namespace Wabbajack.Lib
 
         public static async ValueTask<string> GetMetricsKey()
         {
+            TOP:
             using var _ = await _creationLock.WaitAsync();
             if (!Utils.HaveEncryptedJson(Consts.MetricsKeyHeader))
             {
@@ -61,9 +62,19 @@ namespace Wabbajack.Lib
                     // If there's a regkey and a file, return regkey
                     using (RegistryKey regKey = Registry.CurrentUser.CreateSubKey(@"Software\Wabbajack", RegistryKeyPermissionCheck.Default)!)
                     {
-                        string key = await Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)!;
+                        try
-                        regKey.SetValue("x-metrics-key", key);
+                        {
-                        return key;
+                            string key = await Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)!;
+                            regKey.SetValue("x-metrics-key", key);
+                            return key;
+                        }
+                        catch (Exception)
+                        {
+                            // Probably an encryption error
+                            await Utils.DeleteEncryptedJson(Consts.MetricsKeyHeader);
+                            goto TOP;
+                        }
+
                     }
                 }
             }

Wabbajack.Server/Controllers/ModUpgrade.cs

@@ -1,4 +1,7 @@
-using System.Threading.Tasks;
+using System;
+using System.Linq;
+using System.Security.Claims;
+using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
@@ -32,18 +35,50 @@ namespace Wabbajack.BuildServer.Controllers
         [Route("/mod_upgrade")]
         public async Task<IActionResult> PostModUpgrade()
         {
+            var isAuthor = User.Claims.Any(c => c.Type == ClaimTypes.Role && c.Value == "Author");
             var request = (await Request.Body.ReadAllTextAsync()).FromJsonString<ModUpgradeRequest>();
-            if (!request.IsValid)
+            if (!isAuthor)
             {
-                _logger.Log(LogLevel.Information, $"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as upgrade is invalid");
+                var srcDownload = await _sql.GetArchiveDownload(request.OldArchive.State.PrimaryKeyString,
-                return BadRequest("Invalid mod upgrade");
+                    request.OldArchive.Hash, request.OldArchive.Size);
+                var destDownload = await _sql.GetArchiveDownload(request.NewArchive.State.PrimaryKeyString,
+                    request.NewArchive.Hash, request.NewArchive.Size);
+
+                if (srcDownload == default || destDownload == default ||
+                    await _sql.FindPatch(srcDownload.Id, destDownload.Id) == default)
+                {
+                    if (!await request.IsValid())
+                    {
+                        _logger.Log(LogLevel.Information,
+                            $"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as upgrade is invalid");
+                        return BadRequest("Invalid mod upgrade");
+                    }
+
+                    if (_settings.ValidateModUpgrades && !await _sql.HashIsInAModlist(request.OldArchive.Hash))
+                    {
+                        _logger.Log(LogLevel.Information,
+                            $"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as src hash is not in a curated modlist");
+                        return BadRequest("Hash is not in a recent modlist");
+                    }
+
+                }
+
             }
 
-            if (_settings.ValidateModUpgrades && !await _sql.HashIsInAModlist(request.OldArchive.Hash))
+            try
             {
-                _logger.Log(LogLevel.Information, $"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as src hash is not in a curated modlist");
+                if (await request.OldArchive.State.Verify(request.OldArchive))
-                return BadRequest("Hash is not in a recent modlist");
+                {
+                    _logger.LogInformation(
+                        $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), old archive is valid");
+                    return NotFound("File is Valid");
+                }
             }
+            catch (Exception ex)
+            {
+                // ignore
+            }
+
             var oldDownload = await _sql.GetOrEnqueueArchive(request.OldArchive);
             var newDownload = await _sql.GetOrEnqueueArchive(request.NewArchive);
 
@@ -77,5 +112,17 @@ namespace Wabbajack.BuildServer.Controllers
             return Accepted();
         }
 
+        [HttpGet]
+        [Authorize(Roles = "User")]
+        [Route("/mod_upgrade/find/{hashAsHex}")]
+        public async Task<IActionResult> FindUpgrade(string hashAsHex)
+        {
+            var hash = Hash.FromHex(hashAsHex);
+
+            var patches = await _sql.PatchesForSource(hash);
+            return Ok(patches.Select(p => p.Dest).ToList());
+        }
+      
+
     }
 }

Wabbajack.Server/DataLayer/Patches.cs

@@ -47,7 +47,7 @@ namespace Wabbajack.Server.DataLayer
         {
             await using var conn = await Open();
             var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool?, string)>(
-                @"SELECT p.PatchHash, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage 
+                @"SELECT p.PatchSize, p.Finished, p.IsFailed, p.FailMessage 
                       FROM dbo.Patches p
                       LEFT JOIN dbo.ArchiveDownloads src ON p.SrcId = src.Id
                       LEFT JOIN dbo.ArchiveDownloads dest ON p.SrcId = dest.Id
@@ -133,19 +133,19 @@ namespace Wabbajack.Server.DataLayer
             var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
                 "SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId", new {SrcId = sourceDownload});
 
-            List<Patch> results = new List<Patch>();
+            return await AsPatches(patches);
-            foreach (var (srcId, destId, patchSize, finished, isFailed, failMessage) in patches)
+        }
-            {
+        public async Task<List<Patch>> PatchesForSource(Hash sourceHash)
-                results.Add( new Patch {
+        {
-                    Src = await GetArchiveDownload(srcId), 
+            await using var conn = await Open();
-                    Dest = await GetArchiveDownload(destId),
+            var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
-                    PatchSize = patchSize,
+                @"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage 
-                    Finished = finished,
+                     FROM dbo.Patches p
-                    IsFailed = isFailed,
+                     LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id 
-                    FailMessage = failMessage
+                     
-                });
+                     WHERE a.Hash = @Hash AND p.Finished IS NOT NULL AND p.IsFailed = 0", new {Hash = sourceHash});
-            } 
+
-            return results;
+            return await AsPatches(patches);
         }
 
         public async Task MarkPatchUsage(Guid srcId, Guid destId)
@@ -174,21 +174,29 @@ namespace Wabbajack.Server.DataLayer
                         WHERE m.PrimaryKeyString is not null
                         AND (p.LastUsed < DATEADD(d, -7, getutcdate()) OR p.LastUsed is null and p.Finished < DATEADD(d, -7, getutcdate()))");
             
+            return await AsPatches(patches);
+        }
+
+        private async Task<List<Patch>> AsPatches(IEnumerable<(Guid, Guid, long, DateTime?, bool?, string)> patches)
+        {
             List<Patch> results = new List<Patch>();
             foreach (var (srcId, destId, patchSize, finished, isFailed, failMessage) in patches)
             {
-                results.Add( new Patch {
+                results.Add(new Patch
-                    Src = await GetArchiveDownload(srcId), 
+                {
+                    Src = await GetArchiveDownload(srcId),
                     Dest = await GetArchiveDownload(destId),
                     PatchSize = patchSize,
                     Finished = finished,
                     IsFailed = isFailed,
                     FailMessage = failMessage
                 });
-            } 
+            }
+
             return results;
         }
 
+
         public async Task DeletePatch(Patch patch)
         {
             await using var conn = await Open();
@@ -225,5 +233,6 @@ namespace Wabbajack.Server.DataLayer
             });
             
         }
+
     }
 }

Wabbajack.Server/Startup.cs

@@ -136,6 +136,7 @@ namespace Wabbajack.Server
                         headers.Add("Access-Control-Allow-Methods", "POST, GET");
                         headers.Add("Access-Control-Allow-Headers", "Accept, Origin, Content-type");
                         headers.Add("X-ResponseTime-Ms", stopWatch.ElapsedMilliseconds.ToString());
+                        headers.Add("Cache-Control", "no-cache");
                         return Task.CompletedTask;
                     });
                     await next(context);

Wabbajack.Server/Wabbajack.Server.csproj

@@ -3,8 +3,8 @@
     <PropertyGroup>
         <OutputType>Exe</OutputType>
         <TargetFramework>netcoreapp3.1</TargetFramework>
-        <AssemblyVersion>2.1.1.0</AssemblyVersion>
+        <AssemblyVersion>2.1.2.0</AssemblyVersion>
-        <FileVersion>2.1.1.0</FileVersion>
+        <FileVersion>2.1.2.0</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Server</Description>
         <RuntimeIdentifier>win-x64</RuntimeIdentifier>

Wabbajack/Wabbajack.csproj

@@ -6,8 +6,8 @@
     <UseWPF>true</UseWPF>
     <Platforms>x64</Platforms>
     <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
-      <AssemblyVersion>2.1.1.0</AssemblyVersion>
+      <AssemblyVersion>2.1.2.0</AssemblyVersion>
-      <FileVersion>2.1.1.0</FileVersion>
+      <FileVersion>2.1.2.0</FileVersion>
     <Copyright>Copyright © 2019-2020</Copyright>
     <Description>An automated ModList installer</Description>
     <PublishReadyToRun>true</PublishReadyToRun>