2.4.2.2

CHANGELOG.md

@@ -1,5 +1,9 @@
 ### Changelog
 
+#### Version - 2.4.2.2 - 2/6/2020
+* Better Origin game detection
+* Don't check the download whitelist for files that are already downloaded
+
 #### Version - 2.4.2.1 - 2/4/2020
 * HOTFIX - fix for the download path sometimes being empty
 * HOTFIX - fix for some drive types not being detected (e.g. RAID drives)

Wabbajack.CLI/Wabbajack.CLI.csproj

@@ -6,8 +6,8 @@
         <AssemblyName>wabbajack-cli</AssemblyName>
         <Company>Wabbajack</Company>
         <Platforms>x64</Platforms>
-        <AssemblyVersion>2.4.2.1</AssemblyVersion>
+        <AssemblyVersion>2.4.2.2</AssemblyVersion>
-        <FileVersion>2.4.2.1</FileVersion>
+        <FileVersion>2.4.2.2</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>An automated ModList installer</Description>
         <PublishReadyToRun>true</PublishReadyToRun>        

Wabbajack.Common/Consts.cs

@@ -132,6 +132,7 @@ namespace Wabbajack.Common
         public static RelativePath SettingsIni = (RelativePath)"settings.ini";
         public static byte SettingsVersion => 2;
         public static TimeSpan MaxVerifyTime => TimeSpan.FromMinutes(10);
+        public static readonly string WabbajackAuthoredFilesPrefix = "https://wabbajack.b-cdn.net/";
 
         public static RelativePath NativeSettingsJson = (RelativePath)"native_compiler_settings.json";
 

Wabbajack.Launcher/Wabbajack.Launcher.csproj

@@ -4,8 +4,8 @@
         <OutputType>Exe</OutputType>
         <TargetFramework>net5.0-windows</TargetFramework>
         <UseWPF>true</UseWPF>
-        <AssemblyVersion>2.4.2.1</AssemblyVersion>
+        <AssemblyVersion>2.4.2.2</AssemblyVersion>
-        <FileVersion>2.4.2.1</FileVersion>
+        <FileVersion>2.4.2.2</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Application Launcher</Description>
         <PublishReadyToRun>true</PublishReadyToRun>

Wabbajack.Lib/AInstaller.cs

@@ -6,6 +6,7 @@ using System.Linq;
 using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.Validation;
 using Wabbajack.VirtualFileSystem;
 using Path = Alphaleonis.Win32.Filesystem.Path;
 
@@ -194,7 +195,9 @@ namespace Wabbajack.Lib
 
             Info("Getting Nexus API Key, if a browser appears, please accept");
 
-            var dispatchers = missing.Select(m => m.State.GetDownloader()).Distinct();
+            var dispatchers = missing.Select(m => m.State.GetDownloader())
+                .Distinct()
+                .ToList();
 
             await Task.WhenAll(dispatchers.Select(d => d.Prepare()));
 
@@ -204,6 +207,14 @@ namespace Wabbajack.Lib
                 throw new Exception($"Not enough Nexus API calls to download this list, please try again after midnight GMT when your API limits reset");
             }
 
+            var validationData = new ValidateModlist();
+            await validationData.LoadListsFromGithub();
+
+            foreach (var archive in missing.Where(archive => !archive.State.IsWhitelisted(validationData.ServerWhitelist)))
+            {
+                throw new Exception($"File {archive.State.PrimaryKeyString} failed validation");
+            }
+
             await DownloadMissingArchives(missing);
         }
 
@@ -264,6 +275,7 @@ namespace Wabbajack.Lib
             }
             catch (Exception ex)
             {
+                var tsk = Metrics.Send("failed_download", archive.State.PrimaryKeyString);
                 Utils.Log($"Download error for file {archive.Name}");
                 Utils.Log(ex.ToString());
                 return false;

Wabbajack.Lib/MO2Installer.cs

@@ -103,8 +103,7 @@ namespace Wabbajack.Lib
             await ValidateGameESMs();
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Validating Modlist");
+            UpdateTracker.NextStep("Creating Output Folders");
-            await ValidateModlist.RunValidation(ModList);
 
             OutputFolder.CreateDirectory();
             DownloadFolder.CreateDirectory();

Wabbajack.Lib/Validation/ValidateModlist.cs

@@ -57,7 +57,7 @@ namespace Wabbajack.Lib.Validation
 
         public async Task<IEnumerable<string>> Validate(ModList modlist)
         {
-            ConcurrentStack<string> ValidationErrors = new ConcurrentStack<string>();
+            ConcurrentStack<string> ValidationErrors = new();
             modlist.Archives
                 .Where(m => !m.State.IsWhitelisted(ServerWhitelist))
                 .Do(m =>

Wabbajack.Server.Test/ABuildServerSystemTest.cs

@@ -84,7 +84,11 @@ namespace Wabbajack.BuildServer.Test
 
         public T GetService<T>()
         {
-            return (T)_host.Services.GetService(typeof(T));
+            var result = (T)_host.Services.GetService(typeof(T));
+
+            if (result == null)
+                throw new Exception($"Service {typeof(T)} not found in configuration");
+            return result;
         }
 
 

Wabbajack.Server.Test/AuthoredFilesTests.cs

@@ -22,6 +22,11 @@ namespace Wabbajack.BuildServer.Test
         [Fact]
         public async Task CanUploadDownloadAndDeleteAuthoredFiles()
         {
+            var cleanup = Fixture.GetService<AuthoredFilesCleanup>();
+            var sql = Fixture.GetService<SqlService>();
+            
+            var toDelete = await cleanup.FindFilesToDelete();
+            
             await using var file = new TempFile();
             await file.Path.WriteAllBytesAsync(RandomData(Consts.UPLOADED_FILE_BLOCK_SIZE * 4 + Consts.UPLOADED_FILE_BLOCK_SIZE / 3));
             var originalHash = await file.Path.FileHashAsync();
@@ -30,9 +35,23 @@ namespace Wabbajack.BuildServer.Test
             using var queue = new WorkQueue(2);
             var uri = await client.UploadFile(queue, file.Path, (s, percent) => Utils.Log($"({percent}) {s}"));
 
-            var data = await Fixture.GetService<SqlService>().AllAuthoredFiles();
+            var data = (await Fixture.GetService<SqlService>().AllAuthoredFiles()).ToArray();
             Assert.Contains((string)file.Path.FileName, data.Select(f => f.OriginalFileName));
 
+            var listing = await cleanup.GetCDNMungedNames();
+            foreach (var d in data)
+            {
+                Assert.Contains(d.MungedName, listing);
+            }
+
+            // Just uploaded it, so it shouldn't be marked for deletion
+            toDelete = await cleanup.FindFilesToDelete();
+            foreach (var d in data)
+            {
+                Assert.DoesNotContain(d.MungedName, toDelete.CDNDelete);                
+                Assert.DoesNotContain(d.ServerAssignedUniqueId, toDelete.SQLDelete);
+            }
+
             var result = await _client.GetStringAsync(MakeURL("authored_files"));
             Assert.Contains((string)file.Path.FileName, result);
 
@@ -41,6 +60,27 @@ namespace Wabbajack.BuildServer.Test
 
             await state.Download(new Archive(state) {Name = (string)file.Path.FileName}, file.Path);
             Assert.Equal(originalHash, await file.Path.FileHashAsync());
+            
+            // Mark it as old
+            foreach (var d in data)
+            {
+                await sql.TouchAuthoredFile(await sql.GetCDNFileDefinition(d.ServerAssignedUniqueId), DateTime.Now - TimeSpan.FromDays(8));
+            }
+
+            // Now it should be marked for deletion
+            toDelete = await cleanup.FindFilesToDelete();
+            foreach (var d in data)
+            {
+                Assert.Contains(d.MungedName, toDelete.CDNDelete);                
+                Assert.Contains(d.ServerAssignedUniqueId, toDelete.SQLDelete);
+            }
+
+            await cleanup.Execute();
+            
+            toDelete = await cleanup.FindFilesToDelete();
+            
+            Assert.Empty(toDelete.CDNDelete);
+            Assert.Empty(toDelete.SQLDelete);
 
         }
 

Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs

@@ -1,5 +1,7 @@
 using System.Collections.Generic;
+using System.Net;
 using System.Threading.Tasks;
+using FluentFTP;
 using Wabbajack.Common;
 
 namespace Wabbajack.Server.DTOs
@@ -21,5 +23,12 @@ namespace Wabbajack.Server.DTOs
         {
             return (await Utils.FromEncryptedJson<Dictionary<string, BunnyCdnFtpInfo>>("bunnycdn"))[space.ToString()];
         }
+        
+        public async Task<FtpClient> GetClient()
+        {
+            var ftpClient = new FtpClient(Hostname, new NetworkCredential(Username, Password));
+            await ftpClient.ConnectAsync();
+            return ftpClient;
+        }
     }
 }

Wabbajack.Server/DataLayer/AuthoredFiles.cs

@@ -15,13 +15,21 @@ namespace Wabbajack.Server.DataLayer
 {
     public partial class SqlService
     {
-        public async Task TouchAuthoredFile(CDNFileDefinition definition)
+        public async Task TouchAuthoredFile(CDNFileDefinition definition, DateTime? date = null)
         {
             await using var conn = await Open();
-            await conn.ExecuteAsync("UPDATE AuthoredFiles SET LastTouched = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid",
+            if (date == null)
-                new {
+            {
-                    Uid = definition.ServerAssignedUniqueId
+                await conn.ExecuteAsync(
-                });
+                    "UPDATE AuthoredFiles SET LastTouched = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid",
+                    new {Uid = definition.ServerAssignedUniqueId});
+            }
+            else
+            {
+                await conn.ExecuteAsync(
+                    "UPDATE AuthoredFiles SET LastTouched = @Date WHERE ServerAssignedUniqueId = @Uid",
+                    new {Uid = definition.ServerAssignedUniqueId, Date = date});
+            }
         }
 
         public async Task<CDNFileDefinition> CreateAuthoredFile(CDNFileDefinition definition, string login)
@@ -55,12 +63,13 @@ namespace Wabbajack.Server.DataLayer
                 new {Uid = serverAssignedUniqueId})).First();
         }
         
-        public async Task<CDNFileDefinition> DeleteFileDefinition(CDNFileDefinition definition)
+        public async Task DeleteFileDefinition(CDNFileDefinition definition)
         {
             await using var conn = await Open();
-            return (await conn.QueryAsync<CDNFileDefinition>(
+            await conn.ExecuteAsync(
                 "DELETE FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid",
-                new {Uid = definition.ServerAssignedUniqueId})).First();
+                new {Uid = definition.ServerAssignedUniqueId});
+            return;
         }
 
         public async Task<IEnumerable<AuthoredFilesSummary>> AllAuthoredFiles()
@@ -68,8 +77,7 @@ namespace Wabbajack.Server.DataLayer
             await using var conn = await Open();
             var results = await conn.QueryAsync<AuthoredFilesSummary>("SELECT CONVERT(NVARCHAR(50), ServerAssignedUniqueId) as ServerAssignedUniqueId, Size, OriginalFileName, Author, LastTouched, Finalized, MungedName from dbo.AuthoredFilesSummaries ORDER BY LastTouched DESC");
             return results;
-
         }
-        
+
     }
 }

Wabbajack.Server/Services/AuthoredFilesCleanup.cs

@@ -0,0 +1,139 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Web;
+using FluentFTP;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer;
+using Wabbajack.Common;
+using Wabbajack.Lib.AuthorApi;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.ModListRegistry;
+using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
+using WebSocketSharp;
+
+namespace Wabbajack.Server.Services
+{
+    public class AuthoredFilesCleanup : AbstractService<AuthoredFilesCleanup, int>
+    {
+        private SqlService _sql;
+        private DiscordWebHook _discord;
+
+        public AuthoredFilesCleanup(ILogger<AuthoredFilesCleanup> logger, AppSettings settings, QuickSync quickSync, SqlService sql, DiscordWebHook discord) : base(logger, settings, quickSync, TimeSpan.FromHours(6))
+        {
+            _sql = sql;
+            _discord = discord;
+        }
+
+        public override async Task<int> Execute()
+        {
+
+            var toDelete = await FindFilesToDelete();
+
+            var log = new[] {$"CDNDelete ({toDelete.CDNDelete.Length}):\n\n"}
+                .Concat(toDelete.CDNDelete)
+                .Concat(new[] {$"SQLDelete ({toDelete.SQLDelete.Length}"})
+                .Concat(toDelete.SQLDelete)
+                .Concat(new[] {$"CDNRemain ({toDelete.CDNNotDeleted.Length}"})
+                .Concat(toDelete.CDNNotDeleted)
+                .Concat(new[] {$"SQLRemain ({toDelete.SQLNotDeleted.Length}"})
+                .Concat(toDelete.SQLNotDeleted)
+                .ToArray();
+
+            //await AbsolutePath.EntryPoint.Combine("cdn_delete_log.txt").WriteAllLinesAsync(log);
+            
+            
+            foreach (var sqlFile in toDelete.SQLDelete)
+            {
+                Utils.Log($"Deleting {sqlFile} from SQL");
+                await _sql.DeleteFileDefinition(await _sql.GetCDNFileDefinition(sqlFile));
+            }
+
+
+            using var queue = new WorkQueue(6);
+            await toDelete.CDNDelete.Select((d, idx) => (d, idx)).PMap(queue, async cdnFile =>
+            {
+                using var conn = await (await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles)).GetClient();
+                Utils.Log($"Deleting {cdnFile} from CDN");
+                await _discord.Send(Channel.Ham,
+                    new DiscordMessage
+                    {
+                        Content =
+                            $"({cdnFile.idx}/{toDelete.CDNDelete.Length}) {cdnFile.d} is no longer referenced by any modlist and will be removed from the CDN"
+                    });
+                if (await conn.DirectoryExistsAsync(cdnFile.d))
+                    await conn.DeleteDirectoryAsync(cdnFile.d);
+
+                if (await conn.FileExistsAsync(cdnFile.d))
+                    await conn.DeleteFileAsync(cdnFile.d);
+            });
+            return toDelete.CDNDelete.Length + toDelete.SQLDelete.Length;
+            
+        }
+
+        public async Task<(string[] CDNDelete, string[] SQLDelete, string[] CDNNotDeleted, string[] SQLNotDeleted)> FindFilesToDelete()
+        {
+            var cdnNames = (await GetCDNMungedNames()).ToHashSet();
+            var usedNames = (await GetUsedCDNFiles()).ToHashSet();
+            var sqlFiles = (await _sql.AllAuthoredFiles()).ToDictionary(f => f.MungedName);
+            var keep = GetKeepList(cdnNames, usedNames, sqlFiles).ToHashSet();
+
+            var cdnDelete = cdnNames.Where(h => !keep.Contains(h)).ToArray();
+            var sqlDelete = sqlFiles.Where(s => !keep.Contains(s.Value.MungedName))
+                .Select(s => s.Value.ServerAssignedUniqueId)
+                .ToArray();
+
+            var cdnhs = cdnDelete.ToHashSet();
+            var notDeletedCDN = cdnNames.Where(f => !cdnhs.Contains(f)).ToArray();
+            var sqlhs = sqlDelete.ToHashSet();
+            var sqlNotDeleted = sqlFiles.Where(f => !sqlDelete.Contains(f.Value.ServerAssignedUniqueId))
+                .Select(f => f.Value.MungedName)
+                .ToArray();
+            return (cdnDelete, sqlDelete, notDeletedCDN, sqlNotDeleted);
+        }
+
+        private IEnumerable<string> GetKeepList(HashSet<string> cdnNames, HashSet<string> usedNames, Dictionary<string, AuthoredFilesSummary> sqlFiles)
+        {
+            var cutOff = DateTime.UtcNow - TimeSpan.FromDays(7);
+            foreach (var file in sqlFiles.Where(f => f.Value.LastTouched > cutOff))
+                yield return file.Value.MungedName;
+
+            foreach (var file in usedNames)
+                yield return file;
+        }
+
+        public async Task<string[]> GetCDNMungedNames()
+        {
+            using var client = await (await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles)).GetClient();
+            var lst = await client.GetListingAsync(@"\");
+            return lst.Select(l => l.Name).ToArray();
+        }
+
+        public async Task<string[]> GetUsedCDNFiles()
+        {
+            var modlists = (await ModlistMetadata.LoadFromGithub())
+                .Concat((await ModlistMetadata.LoadUnlistedFromGithub()))
+                .Select(f => f.Links.Download)
+                .Where(f => f.StartsWith(Consts.WabbajackAuthoredFilesPrefix))
+                .Select(f => f.Substring(Consts.WabbajackAuthoredFilesPrefix.Length));
+
+            var files = (await _sql.ModListArchives())
+                .Select(a => a.State)
+                .OfType<WabbajackCDNDownloader.State>()
+                .Select(s => s.Url.ToString().Substring(Consts.WabbajackAuthoredFilesPrefix.Length));
+            
+            
+
+            var names = modlists.Concat(files).Distinct().ToArray();
+            var namesBoth = names.Concat(names.Select(HttpUtility.UrlDecode))
+                .Concat(names.Select(HttpUtility.UrlEncode))
+                .Distinct()
+                .ToArray();
+            return namesBoth;
+        }
+    }
+}

Wabbajack.Server/Startup.cs

@@ -75,6 +75,7 @@ namespace Wabbajack.Server
             services.AddSingleton<MirrorQueueService>();
             services.AddSingleton<Watchdog>();
             services.AddSingleton<DiscordFrontend>();
+            services.AddSingleton<AuthoredFilesCleanup>();
             
             services.AddMvc();
             services.AddControllers()
@@ -135,6 +136,7 @@ namespace Wabbajack.Server
             app.UseService<MirrorQueueService>();
             app.UseService<Watchdog>();
             app.UseService<DiscordFrontend>();
+            app.UseService<AuthoredFilesCleanup>();
 
             app.Use(next =>
             {

Wabbajack.Server/Wabbajack.Server.csproj

@@ -3,8 +3,8 @@
     <PropertyGroup>
         <OutputType>Exe</OutputType>
         <TargetFramework>net5.0-windows</TargetFramework>
-        <AssemblyVersion>2.4.2.1</AssemblyVersion>
+        <AssemblyVersion>2.4.2.2</AssemblyVersion>
-        <FileVersion>2.4.2.1</FileVersion>
+        <FileVersion>2.4.2.2</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Server</Description>
         <RuntimeIdentifier>win-x64</RuntimeIdentifier>

Wabbajack.Server/public/metrics.html

@@ -33,40 +33,45 @@
 <canvas id="started_wabbajack_chart" width="800" height="600"></canvas>
 <hr/>
 
+<h2>Exceptions</h2>
+<canvas id="exceptions_chart" width="800" height="600"></canvas>
+<hr/>
+
+
 
 
 
 <script>
-    
+
     var getReport = function(subject, callback) {
         $.getJSON("/metrics/report/"+subject, callback)
     }
-    
+
     var makeChart = function(ele, group) {
 
         var result_fn = function (data) {
-                    var data = _.filter(data, series => _.some(series.values, v => v > 1));
+            var data = _.filter(data, series => _.some(series.values, v => v > 1));
-                    var labels = _.uniq(_.flatten(_.map(data, series => series.labels)));
+            var labels = _.uniq(_.flatten(_.map(data, series => series.labels)));
-                    var datasets = _.map(data, series => {
+            var datasets = _.map(data, series => {
-                        return {
+                return {
-                            label: series.seriesName,
+                    label: series.seriesName,
-                            fill: false,
+                    fill: false,
-                            data: _.last(series.values, 90)
+                    data: _.last(series.values, 90)
-                        }});
+                }});
-                    var ctx = document.getElementById(ele).getContext('2d');
+            var ctx = document.getElementById(ele).getContext('2d');
-                    var chart = new Chart(ctx, {
+            var chart = new Chart(ctx, {
-                        // The type of chart we want to create
+                // The type of chart we want to create
-                        type: 'bar',
+                type: 'bar',
 
-                        // The data for our dataset
+                // The data for our dataset
-                        data: {
+                data: {
-                            labels: _.last(labels, 90),
+                    labels: _.last(labels, 90),
-                            datasets: datasets},
+                    datasets: datasets},
 
-                        // Configuration options go here
+                // Configuration options go here
-                        options: {scales: {xAxes: [{stacked:true}], yAxes: [{stacked:true}]}}
+                options: {scales: {xAxes: [{stacked:true}], yAxes: [{stacked:true}]}}
-                    });
+            });
-                };
+        };
         getReport(group, result_fn);
     };
 
@@ -74,26 +79,26 @@
     var makePieChart = function(ele, group) {
 
         var result_fn = function (data) {
-                    var data = _.filter(data, series => _.some(series.values, v => v > 2));
+            var data = _.filter(data, series => _.some(series.values, v => v > 2));
-                    var labels = _.map(data, series => series.seriesName);
+            var labels = _.map(data, series => series.seriesName);
-                    var datasets = {data : _.map(data, series => {
+            var datasets = {data : _.map(data, series => {
-                            return _.reduce(series.values, (x, y) => x + y, 0)})};
+                    return _.reduce(series.values, (x, y) => x + y, 0)})};
-                    console.log(datasets);
+            console.log(datasets);
-                    console.log(labels);
+            console.log(labels);
-                    var ctx = document.getElementById(ele).getContext('2d');
+            var ctx = document.getElementById(ele).getContext('2d');
-                    var chart = new Chart(ctx, {
+            var chart = new Chart(ctx, {
-                        // The type of chart we want to create
+                // The type of chart we want to create
-                        type: 'pie',
+                type: 'pie',
 
-                        // The data for our dataset
+                // The data for our dataset
-                        data: {
+                data: {
-                            labels: labels,
+                    labels: labels,
-                            datasets: [datasets]},
+                    datasets: [datasets]},
 
-                        // Configuration options go here
+                // Configuration options go here
-                        options: {}
+                options: {}
-                    });
+            });
-                };
+        };
         getReport(group, result_fn)
     };
 
@@ -101,6 +106,7 @@
     makeChart("begin_install_chart", "begin_install");
     makeChart("finished_install_chart", "finish_install");
     makeChart("started_wabbajack_chart", "started_wabbajack");
+    makeChart("exceptions_chart", "Exception");
     makePieChart("finished_install_count", "finish_install");
 </script>
 </body>

Wabbajack/Wabbajack.csproj

@@ -6,8 +6,8 @@
     <UseWPF>true</UseWPF>
     <Platforms>x64</Platforms>
     <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
-      <AssemblyVersion>2.4.2.1</AssemblyVersion>
+      <AssemblyVersion>2.4.2.2</AssemblyVersion>
-      <FileVersion>2.4.2.1</FileVersion>
+      <FileVersion>2.4.2.2</FileVersion>
     <Copyright>Copyright © 2019-2020</Copyright>
     <Description>An automated ModList installer</Description>
     <PublishReadyToRun>true</PublishReadyToRun>