diff --git a/CHANGELOG.md b/CHANGELOG.md
index 75ae5a33..85acfcf7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
### Changelog
+#### Version - 2.4.2.2 - 2/6/2020
+* Better Origin game detection
+* Don't check the download whitelist for files that are already downloaded
+
#### Version - 2.4.2.1 - 2/4/2020
* HOTFIX - fix for the download path sometimes being empty
* HOTFIX - fix for some drive types not being detected (e.g. RAID drives)
diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj
index 884b9f22..a477c70f 100644
--- a/Wabbajack.CLI/Wabbajack.CLI.csproj
+++ b/Wabbajack.CLI/Wabbajack.CLI.csproj
@@ -6,8 +6,8 @@
wabbajack-cli
Wabbajack
x64
- 2.4.2.1
- 2.4.2.1
+ 2.4.2.2
+ 2.4.2.2
Copyright © 2019-2020
An automated ModList installer
true
diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs
index 803cbe6a..659df9d4 100644
--- a/Wabbajack.Common/Consts.cs
+++ b/Wabbajack.Common/Consts.cs
@@ -132,6 +132,7 @@ namespace Wabbajack.Common
public static RelativePath SettingsIni = (RelativePath)"settings.ini";
public static byte SettingsVersion => 2;
public static TimeSpan MaxVerifyTime => TimeSpan.FromMinutes(10);
+ public static readonly string WabbajackAuthoredFilesPrefix = "https://wabbajack.b-cdn.net/";
public static RelativePath NativeSettingsJson = (RelativePath)"native_compiler_settings.json";
diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj
index f2fa3380..f975a804 100644
--- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj
+++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj
@@ -4,8 +4,8 @@
Exe
net5.0-windows
true
- 2.4.2.1
- 2.4.2.1
+ 2.4.2.2
+ 2.4.2.2
Copyright © 2019-2020
Wabbajack Application Launcher
true
diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs
index 8930f44b..5df31d7d 100644
--- a/Wabbajack.Lib/AInstaller.cs
+++ b/Wabbajack.Lib/AInstaller.cs
@@ -6,6 +6,7 @@ using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.Validation;
using Wabbajack.VirtualFileSystem;
using Path = Alphaleonis.Win32.Filesystem.Path;
@@ -194,7 +195,9 @@ namespace Wabbajack.Lib
Info("Getting Nexus API Key, if a browser appears, please accept");
- var dispatchers = missing.Select(m => m.State.GetDownloader()).Distinct();
+ var dispatchers = missing.Select(m => m.State.GetDownloader())
+ .Distinct()
+ .ToList();
await Task.WhenAll(dispatchers.Select(d => d.Prepare()));
@@ -204,6 +207,14 @@ namespace Wabbajack.Lib
throw new Exception($"Not enough Nexus API calls to download this list, please try again after midnight GMT when your API limits reset");
}
+ var validationData = new ValidateModlist();
+ await validationData.LoadListsFromGithub();
+
+ foreach (var archive in missing.Where(archive => !archive.State.IsWhitelisted(validationData.ServerWhitelist)))
+ {
+ throw new Exception($"File {archive.State.PrimaryKeyString} failed validation");
+ }
+
await DownloadMissingArchives(missing);
}
@@ -264,6 +275,7 @@ namespace Wabbajack.Lib
}
catch (Exception ex)
{
+ var tsk = Metrics.Send("failed_download", archive.State.PrimaryKeyString);
Utils.Log($"Download error for file {archive.Name}");
Utils.Log(ex.ToString());
return false;
diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs
index f82a9cd9..33f93b59 100644
--- a/Wabbajack.Lib/MO2Installer.cs
+++ b/Wabbajack.Lib/MO2Installer.cs
@@ -103,8 +103,7 @@ namespace Wabbajack.Lib
await ValidateGameESMs();
if (cancel.IsCancellationRequested) return false;
- UpdateTracker.NextStep("Validating Modlist");
- await ValidateModlist.RunValidation(ModList);
+ UpdateTracker.NextStep("Creating Output Folders");
OutputFolder.CreateDirectory();
DownloadFolder.CreateDirectory();
diff --git a/Wabbajack.Lib/Validation/ValidateModlist.cs b/Wabbajack.Lib/Validation/ValidateModlist.cs
index 43ac949b..360f2389 100644
--- a/Wabbajack.Lib/Validation/ValidateModlist.cs
+++ b/Wabbajack.Lib/Validation/ValidateModlist.cs
@@ -57,7 +57,7 @@ namespace Wabbajack.Lib.Validation
public async Task> Validate(ModList modlist)
{
- ConcurrentStack ValidationErrors = new ConcurrentStack();
+ ConcurrentStack ValidationErrors = new();
modlist.Archives
.Where(m => !m.State.IsWhitelisted(ServerWhitelist))
.Do(m =>
diff --git a/Wabbajack.Server.Test/ABuildServerSystemTest.cs b/Wabbajack.Server.Test/ABuildServerSystemTest.cs
index c3ae1621..62bfb879 100644
--- a/Wabbajack.Server.Test/ABuildServerSystemTest.cs
+++ b/Wabbajack.Server.Test/ABuildServerSystemTest.cs
@@ -84,7 +84,11 @@ namespace Wabbajack.BuildServer.Test
public T GetService()
{
- return (T)_host.Services.GetService(typeof(T));
+ var result = (T)_host.Services.GetService(typeof(T));
+
+ if (result == null)
+ throw new Exception($"Service {typeof(T)} not found in configuration");
+ return result;
}
diff --git a/Wabbajack.Server.Test/AuthoredFilesTests.cs b/Wabbajack.Server.Test/AuthoredFilesTests.cs
index 7ddce64e..3f4eb15f 100644
--- a/Wabbajack.Server.Test/AuthoredFilesTests.cs
+++ b/Wabbajack.Server.Test/AuthoredFilesTests.cs
@@ -22,6 +22,11 @@ namespace Wabbajack.BuildServer.Test
[Fact]
public async Task CanUploadDownloadAndDeleteAuthoredFiles()
{
+ var cleanup = Fixture.GetService();
+ var sql = Fixture.GetService();
+
+ var toDelete = await cleanup.FindFilesToDelete();
+
await using var file = new TempFile();
await file.Path.WriteAllBytesAsync(RandomData(Consts.UPLOADED_FILE_BLOCK_SIZE * 4 + Consts.UPLOADED_FILE_BLOCK_SIZE / 3));
var originalHash = await file.Path.FileHashAsync();
@@ -30,9 +35,23 @@ namespace Wabbajack.BuildServer.Test
using var queue = new WorkQueue(2);
var uri = await client.UploadFile(queue, file.Path, (s, percent) => Utils.Log($"({percent}) {s}"));
- var data = await Fixture.GetService().AllAuthoredFiles();
+ var data = (await Fixture.GetService().AllAuthoredFiles()).ToArray();
Assert.Contains((string)file.Path.FileName, data.Select(f => f.OriginalFileName));
+ var listing = await cleanup.GetCDNMungedNames();
+ foreach (var d in data)
+ {
+ Assert.Contains(d.MungedName, listing);
+ }
+
+ // Just uploaded it, so it shouldn't be marked for deletion
+ toDelete = await cleanup.FindFilesToDelete();
+ foreach (var d in data)
+ {
+ Assert.DoesNotContain(d.MungedName, toDelete.CDNDelete);
+ Assert.DoesNotContain(d.ServerAssignedUniqueId, toDelete.SQLDelete);
+ }
+
var result = await _client.GetStringAsync(MakeURL("authored_files"));
Assert.Contains((string)file.Path.FileName, result);
@@ -41,6 +60,27 @@ namespace Wabbajack.BuildServer.Test
await state.Download(new Archive(state) {Name = (string)file.Path.FileName}, file.Path);
Assert.Equal(originalHash, await file.Path.FileHashAsync());
+
+ // Mark it as old
+ foreach (var d in data)
+ {
+ await sql.TouchAuthoredFile(await sql.GetCDNFileDefinition(d.ServerAssignedUniqueId), DateTime.Now - TimeSpan.FromDays(8));
+ }
+
+ // Now it should be marked for deletion
+ toDelete = await cleanup.FindFilesToDelete();
+ foreach (var d in data)
+ {
+ Assert.Contains(d.MungedName, toDelete.CDNDelete);
+ Assert.Contains(d.ServerAssignedUniqueId, toDelete.SQLDelete);
+ }
+
+ await cleanup.Execute();
+
+ toDelete = await cleanup.FindFilesToDelete();
+
+ Assert.Empty(toDelete.CDNDelete);
+ Assert.Empty(toDelete.SQLDelete);
}
diff --git a/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs b/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs
index 652e9612..34470cd9 100644
--- a/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs
+++ b/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs
@@ -1,5 +1,7 @@
using System.Collections.Generic;
+using System.Net;
using System.Threading.Tasks;
+using FluentFTP;
using Wabbajack.Common;
namespace Wabbajack.Server.DTOs
@@ -21,5 +23,12 @@ namespace Wabbajack.Server.DTOs
{
return (await Utils.FromEncryptedJson>("bunnycdn"))[space.ToString()];
}
+
+ public async Task GetClient()
+ {
+ var ftpClient = new FtpClient(Hostname, new NetworkCredential(Username, Password));
+ await ftpClient.ConnectAsync();
+ return ftpClient;
+ }
}
}
diff --git a/Wabbajack.Server/DataLayer/AuthoredFiles.cs b/Wabbajack.Server/DataLayer/AuthoredFiles.cs
index 0dda7647..87f3b0c6 100644
--- a/Wabbajack.Server/DataLayer/AuthoredFiles.cs
+++ b/Wabbajack.Server/DataLayer/AuthoredFiles.cs
@@ -15,13 +15,21 @@ namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
- public async Task TouchAuthoredFile(CDNFileDefinition definition)
+ public async Task TouchAuthoredFile(CDNFileDefinition definition, DateTime? date = null)
{
await using var conn = await Open();
- await conn.ExecuteAsync("UPDATE AuthoredFiles SET LastTouched = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid",
- new {
- Uid = definition.ServerAssignedUniqueId
- });
+ if (date == null)
+ {
+ await conn.ExecuteAsync(
+ "UPDATE AuthoredFiles SET LastTouched = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid",
+ new {Uid = definition.ServerAssignedUniqueId});
+ }
+ else
+ {
+ await conn.ExecuteAsync(
+ "UPDATE AuthoredFiles SET LastTouched = @Date WHERE ServerAssignedUniqueId = @Uid",
+ new {Uid = definition.ServerAssignedUniqueId, Date = date});
+ }
}
public async Task CreateAuthoredFile(CDNFileDefinition definition, string login)
@@ -55,12 +63,13 @@ namespace Wabbajack.Server.DataLayer
new {Uid = serverAssignedUniqueId})).First();
}
- public async Task DeleteFileDefinition(CDNFileDefinition definition)
+ public async Task DeleteFileDefinition(CDNFileDefinition definition)
{
await using var conn = await Open();
- return (await conn.QueryAsync(
+ await conn.ExecuteAsync(
"DELETE FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid",
- new {Uid = definition.ServerAssignedUniqueId})).First();
+ new {Uid = definition.ServerAssignedUniqueId});
+ return;
}
public async Task> AllAuthoredFiles()
@@ -68,8 +77,7 @@ namespace Wabbajack.Server.DataLayer
await using var conn = await Open();
var results = await conn.QueryAsync("SELECT CONVERT(NVARCHAR(50), ServerAssignedUniqueId) as ServerAssignedUniqueId, Size, OriginalFileName, Author, LastTouched, Finalized, MungedName from dbo.AuthoredFilesSummaries ORDER BY LastTouched DESC");
return results;
-
}
-
+
}
}
diff --git a/Wabbajack.Server/Services/AuthoredFilesCleanup.cs b/Wabbajack.Server/Services/AuthoredFilesCleanup.cs
new file mode 100644
index 00000000..bfa992ed
--- /dev/null
+++ b/Wabbajack.Server/Services/AuthoredFilesCleanup.cs
@@ -0,0 +1,139 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Web;
+using FluentFTP;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer;
+using Wabbajack.Common;
+using Wabbajack.Lib.AuthorApi;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.ModListRegistry;
+using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
+using WebSocketSharp;
+
+namespace Wabbajack.Server.Services
+{
+ public class AuthoredFilesCleanup : AbstractService
+ {
+ private SqlService _sql;
+ private DiscordWebHook _discord;
+
+ public AuthoredFilesCleanup(ILogger logger, AppSettings settings, QuickSync quickSync, SqlService sql, DiscordWebHook discord) : base(logger, settings, quickSync, TimeSpan.FromHours(6))
+ {
+ _sql = sql;
+ _discord = discord;
+ }
+
+ public override async Task Execute()
+ {
+
+ var toDelete = await FindFilesToDelete();
+
+ var log = new[] {$"CDNDelete ({toDelete.CDNDelete.Length}):\n\n"}
+ .Concat(toDelete.CDNDelete)
+ .Concat(new[] {$"SQLDelete ({toDelete.SQLDelete.Length}"})
+ .Concat(toDelete.SQLDelete)
+ .Concat(new[] {$"CDNRemain ({toDelete.CDNNotDeleted.Length}"})
+ .Concat(toDelete.CDNNotDeleted)
+ .Concat(new[] {$"SQLRemain ({toDelete.SQLNotDeleted.Length}"})
+ .Concat(toDelete.SQLNotDeleted)
+ .ToArray();
+
+ //await AbsolutePath.EntryPoint.Combine("cdn_delete_log.txt").WriteAllLinesAsync(log);
+
+
+ foreach (var sqlFile in toDelete.SQLDelete)
+ {
+ Utils.Log($"Deleting {sqlFile} from SQL");
+ await _sql.DeleteFileDefinition(await _sql.GetCDNFileDefinition(sqlFile));
+ }
+
+
+ using var queue = new WorkQueue(6);
+ await toDelete.CDNDelete.Select((d, idx) => (d, idx)).PMap(queue, async cdnFile =>
+ {
+ using var conn = await (await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles)).GetClient();
+ Utils.Log($"Deleting {cdnFile} from CDN");
+ await _discord.Send(Channel.Ham,
+ new DiscordMessage
+ {
+ Content =
+ $"({cdnFile.idx}/{toDelete.CDNDelete.Length}) {cdnFile.d} is no longer referenced by any modlist and will be removed from the CDN"
+ });
+ if (await conn.DirectoryExistsAsync(cdnFile.d))
+ await conn.DeleteDirectoryAsync(cdnFile.d);
+
+ if (await conn.FileExistsAsync(cdnFile.d))
+ await conn.DeleteFileAsync(cdnFile.d);
+ });
+ return toDelete.CDNDelete.Length + toDelete.SQLDelete.Length;
+
+ }
+
+ public async Task<(string[] CDNDelete, string[] SQLDelete, string[] CDNNotDeleted, string[] SQLNotDeleted)> FindFilesToDelete()
+ {
+ var cdnNames = (await GetCDNMungedNames()).ToHashSet();
+ var usedNames = (await GetUsedCDNFiles()).ToHashSet();
+ var sqlFiles = (await _sql.AllAuthoredFiles()).ToDictionary(f => f.MungedName);
+ var keep = GetKeepList(cdnNames, usedNames, sqlFiles).ToHashSet();
+
+ var cdnDelete = cdnNames.Where(h => !keep.Contains(h)).ToArray();
+ var sqlDelete = sqlFiles.Where(s => !keep.Contains(s.Value.MungedName))
+ .Select(s => s.Value.ServerAssignedUniqueId)
+ .ToArray();
+
+ var cdnhs = cdnDelete.ToHashSet();
+ var notDeletedCDN = cdnNames.Where(f => !cdnhs.Contains(f)).ToArray();
+ var sqlhs = sqlDelete.ToHashSet();
+ var sqlNotDeleted = sqlFiles.Where(f => !sqlDelete.Contains(f.Value.ServerAssignedUniqueId))
+ .Select(f => f.Value.MungedName)
+ .ToArray();
+ return (cdnDelete, sqlDelete, notDeletedCDN, sqlNotDeleted);
+ }
+
+ private IEnumerable GetKeepList(HashSet cdnNames, HashSet usedNames, Dictionary sqlFiles)
+ {
+ var cutOff = DateTime.UtcNow - TimeSpan.FromDays(7);
+ foreach (var file in sqlFiles.Where(f => f.Value.LastTouched > cutOff))
+ yield return file.Value.MungedName;
+
+ foreach (var file in usedNames)
+ yield return file;
+ }
+
+ public async Task GetCDNMungedNames()
+ {
+ using var client = await (await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles)).GetClient();
+ var lst = await client.GetListingAsync(@"\");
+ return lst.Select(l => l.Name).ToArray();
+ }
+
+ public async Task GetUsedCDNFiles()
+ {
+ var modlists = (await ModlistMetadata.LoadFromGithub())
+ .Concat((await ModlistMetadata.LoadUnlistedFromGithub()))
+ .Select(f => f.Links.Download)
+ .Where(f => f.StartsWith(Consts.WabbajackAuthoredFilesPrefix))
+ .Select(f => f.Substring(Consts.WabbajackAuthoredFilesPrefix.Length));
+
+ var files = (await _sql.ModListArchives())
+ .Select(a => a.State)
+ .OfType()
+ .Select(s => s.Url.ToString().Substring(Consts.WabbajackAuthoredFilesPrefix.Length));
+
+
+
+ var names = modlists.Concat(files).Distinct().ToArray();
+ var namesBoth = names.Concat(names.Select(HttpUtility.UrlDecode))
+ .Concat(names.Select(HttpUtility.UrlEncode))
+ .Distinct()
+ .ToArray();
+ return namesBoth;
+ }
+ }
+}
diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs
index 6dfd3cf7..2d1c4e2c 100644
--- a/Wabbajack.Server/Startup.cs
+++ b/Wabbajack.Server/Startup.cs
@@ -75,6 +75,7 @@ namespace Wabbajack.Server
services.AddSingleton();
services.AddSingleton();
services.AddSingleton();
+ services.AddSingleton();
services.AddMvc();
services.AddControllers()
@@ -135,6 +136,7 @@ namespace Wabbajack.Server
app.UseService();
app.UseService();
app.UseService();
+ app.UseService();
app.Use(next =>
{
diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj
index 2b1e8f44..d6a3e28e 100644
--- a/Wabbajack.Server/Wabbajack.Server.csproj
+++ b/Wabbajack.Server/Wabbajack.Server.csproj
@@ -3,8 +3,8 @@
Exe
net5.0-windows
- 2.4.2.1
- 2.4.2.1
+ 2.4.2.2
+ 2.4.2.2
Copyright © 2019-2020
Wabbajack Server
win-x64
diff --git a/Wabbajack.Server/public/metrics.html b/Wabbajack.Server/public/metrics.html
index d81e4c0b..e9a22c0a 100644
--- a/Wabbajack.Server/public/metrics.html
+++ b/Wabbajack.Server/public/metrics.html
@@ -33,40 +33,45 @@
+Exceptions
+
+
+
+