Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #853 from wabbajack-tools/auto-heal-redux

Browse Source 
Auto heal redux
This commit is contained in:
Timothy Baldridge 2020-05-20 22:10:22 -07:00 committed by GitHub
commit 7f33c18e2c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 1100 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.Common/Consts.cs
View File

@ -144,6 +144,8 @@ namespace Wabbajack.Common
public static Uri WabbajackOrg = new Uri("https://www.wabbajack.org/");
public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2;
public static string ArchiveUpdatesCDNFolder = "archive_updates";
}
}

3
Wabbajack.Common/Hash.cs
View File

@ -225,7 +225,8 @@ namespace Wabbajack.Common
{
await using var fs = file.OpenRead();
var config = new xxHashConfig {HashSizeInBits = 64};
var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(fs);
await using var hs = new StatusFileStream(fs, $"Hashing {file}");
var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(hs);
return new Hash(BitConverter.ToUInt64(value.Hash));
}
catch (IOException)

11
Wabbajack.Common/Http/Client.cs
View File

@ -19,11 +19,17 @@ namespace Wabbajack.Common.Http
return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions);
}
public async Task<HttpResponseMessage> GetAsync(Uri url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions);
}
public async Task<HttpResponseMessage> PostAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead)
public async Task<HttpResponseMessage> PostAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true)
{
var request = new HttpRequestMessage(HttpMethod.Post, url) {Content = content};
return await SendAsync(request, responseHeadersRead);
return await SendAsync(request, responseHeadersRead, errorsAsExceptions);
}
public async Task<HttpResponseMessage> PutAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead)
@ -79,7 +85,6 @@ namespace Wabbajack.Common.Http
if (errorsAsExceptions)
throw new HttpRequestException(
$"Http Exception {response.StatusCode} - {response.ReasonPhrase} - {msg.RequestUri}");
;
return response;
}
catch (Exception ex)

4
Wabbajack.Common/OctoDiff.cs
View File

@ -28,7 +28,7 @@ namespace Wabbajack.Common
return sigStream;
}
private static void CreateSignature(Stream oldData, FileStream sigStream)
private static void CreateSignature(Stream oldData, Stream sigStream)
{
Utils.Status("Creating Patch Signature");
var signatureBuilder = new SignatureBuilder();
@ -36,7 +36,7 @@ namespace Wabbajack.Common
sigStream.Position = 0;
}
public static void Create(Stream oldData, FileStream newData, FileStream signature, FileStream output)
public static void Create(Stream oldData, Stream newData, Stream signature, Stream output)
{
CreateSignature(oldData, signature);
var db = new DeltaBuilder {ProgressReporter = reporter};

7
Wabbajack.Common/Paths.cs
View File

@ -337,6 +337,13 @@ namespace Wabbajack.Common
await fs.WriteAsync(data);
}
public async Task WriteAllAsync(Stream data, bool disposeAfter = true)
{
await using var fs = Create();
await data.CopyToAsync(fs);
if (disposeAfter) await data.DisposeAsync();
}
public void AppendAllText(string text)
{
File.AppendAllText(_path, text);

2
Wabbajack.Lib/AInstaller.cs
View File

@ -298,7 +298,7 @@ namespace Wabbajack.Lib
.Where(e => e.Extension != Consts.HashFileExtension)
.ToList();
Utils.Log($"Found {toHash} files to hash");
Utils.Log($"Found {toHash.Count} files to hash");
var hashResults = await
toHash

72
Wabbajack.Lib/ClientAPI.cs
View File

@ -1,11 +1,46 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.Exceptions;
namespace Wabbajack.Lib
{
[JsonName("ModUpgradeRequest")]
public class ModUpgradeRequest
{
public Archive OldArchive { get; set; }
public Archive NewArchive { get; set; }
public ModUpgradeRequest(Archive oldArchive, Archive newArchive)
{
OldArchive = oldArchive;
NewArchive = newArchive;
}
public bool IsValid
{
get
{
if (OldArchive.Hash == NewArchive.Hash && OldArchive.State.PrimaryKeyString == NewArchive.State.PrimaryKeyString) return false;
if (OldArchive.State.GetType() != NewArchive.State.GetType())
return false;
if (OldArchive.State is IUpgradingState u)
{
return u.ValidateUpgrade(NewArchive.State);
}
return false;
}
}
}
public class ClientAPI
{
public static Common.Http.Client GetClient()
@ -16,18 +51,39 @@ namespace Wabbajack.Lib
return client;
}
public static async Task<Archive?> GetModUpgrade(Hash hash)
public static async Task<Uri> GetModUpgrade(Archive oldArchive, Archive newArchive, TimeSpan? maxWait = null, TimeSpan? waitBetweenTries = null)
{
using var response = await GetClient()
.GetAsync($"{Consts.WabbajackBuildServerUri}alternative/{hash.ToHex()}");
maxWait ??= TimeSpan.FromMinutes(10);
waitBetweenTries ??= TimeSpan.FromSeconds(15);
var request = new ModUpgradeRequest( oldArchive, newArchive);
var start = DateTime.UtcNow;
RETRY:
var response = await GetClient()
.PostAsync($"{Consts.WabbajackBuildServerUri}mod_upgrade", new StringContent(request.ToJson(), Encoding.UTF8, "application/json"));
if (response.IsSuccessStatusCode)
{
return (await response.Content.ReadAsStringAsync()).FromJsonString<Archive>();
switch (response.StatusCode)
{
case HttpStatusCode.OK:
return new Uri(await response.Content.ReadAsStringAsync());
case HttpStatusCode.Accepted:
Utils.Log($"Waiting for patch processing on the server for {oldArchive.Name}, sleeping for another 15 seconds");
await Task.Delay(TimeSpan.FromSeconds(15));
response.Dispose();
if (DateTime.UtcNow - start > maxWait)
throw new HttpException(response);
goto RETRY;
}
}
Utils.Log($"No Upgrade for {hash}");
Utils.Log(await response.Content.ReadAsStringAsync());
return null;
var ex = new HttpException(response);
response.Dispose();
throw ex;
}
/// <summary>

63
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
@ -95,40 +97,49 @@ namespace Wabbajack.Lib.Downloaders
return true;
}
Utils.Log($"Download failed, looking for upgrade");
var upgrade = await ClientAPI.GetModUpgrade(archive.Hash);
if (upgrade == null)
if (!(archive.State is IUpgradingState))
{
Utils.Log($"No upgrade found for {archive.Hash}");
Utils.Log($"Download failed for {archive.Name} and no upgrade from this download source is possible");
return false;
}
Utils.Log($"Upgrading via {upgrade.State.PrimaryKeyString}");
var upgrade = (IUpgradingState)archive.State;
Utils.Log($"Trying to find solution to broken download for {archive.Name}");
Utils.Log($"Upgrading {archive.Hash}");
var upgradePath = destination.Parent.Combine("_Upgrade_" + archive.Name);
var upgradeResult = await Download(upgrade, upgradePath);
if (!upgradeResult) return false;
var patchName = $"{archive.Hash.ToHex()}_{upgrade.Hash.ToHex()}";
var patchPath = destination.Parent.Combine("_Patch_" + patchName);
var patchState = new Archive(new HTTPDownloader.State($"https://wabbajackcdn.b-cdn.net/updates/{patchName}"))
var result = await upgrade.FindUpgrade(archive);
if (result == default)
{
Name = patchName,
};
Utils.Log(
$"No solution for broken download {archive.Name} {archive.State.PrimaryKeyString} could be found");
return false;
var patchResult = await Download(patchState, patchPath);
if (!patchResult) return false;
Utils.Status($"Applying Upgrade to {archive.Hash}");
await using (var patchStream = patchPath.OpenRead())
await using (var srcStream = upgradePath.OpenRead())
await using (var destStream = destination.Create())
{
OctoDiff.Apply(srcStream, patchStream, destStream);
}
await destination.FileHashCachedAsync();
Utils.Log($"Looking for patch for {archive.Name}");
var patchResult = await ClientAPI.GetModUpgrade(archive, result.Archive!);
Utils.Log($"Downloading patch for {archive.Name}");
var tempFile = new TempFile();
using var response = await (new Common.Http.Client()).GetAsync(patchResult);
await tempFile.Path.WriteAllAsync(await response.Content.ReadAsStreamAsync());
response.Dispose();
Utils.Log($"Applying patch to {archive.Name}");
await using(var src = result.NewFile.Path.OpenShared())
await using (var final = destination.Create())
{
Utils.ApplyPatch(src, () => tempFile.Path.OpenShared(), final);
}
var hash = await destination.FileHashCachedAsync();
if (hash != archive.Hash && archive.Hash != default)
{
Utils.Log("Archive hash didn't match after patching");
return false;
}
return true;
}

32
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -5,6 +5,7 @@ using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading.Tasks;
using AngleSharp.Css;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
@ -46,7 +47,7 @@ namespace Wabbajack.Lib.Downloaders
}
[JsonName("HttpDownloader")]
public class State : AbstractDownloadState
public class State : AbstractDownloadState, IUpgradingState
{
public string Url { get; }
@ -210,6 +211,35 @@ TOP:
return new [] {"[General]", $"directURL={Url}"};
}
public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
{
var tmpFile = new TempFile();
var newArchive = new Archive(this) {Name = a.Name};
try
{
if (!await Download(newArchive, tmpFile.Path))
return default;
}
catch (HttpRequestException)
{
return default;
}
newArchive.Hash = await tmpFile.Path.FileHashAsync();
newArchive.Size = tmpFile.Path.Size;
return (newArchive, tmpFile);
}
public bool ValidateUpgrade(AbstractDownloadState newArchiveState)
{
var httpState = (State)newArchiveState;
return httpState.Url == Url;
}
}
}
}

18
Wabbajack.Lib/Downloaders/IUpgradingState.cs Normal file
View File

@ -0,0 +1,18 @@
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.Lib.Downloaders
{
public interface IUpgradingState
{
/// <summary>
/// Find a possible archive that can be combined with a server generated patch to get the input archive
/// state;
/// </summary>
/// <param name="a"></param>
/// <returns></returns>
public Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a);
bool ValidateUpgrade(AbstractDownloadState newArchiveState);
}
}

37
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -140,7 +140,7 @@ namespace Wabbajack.Lib.Downloaders
}
[JsonName("NexusDownloader")]
public class State : AbstractDownloadState, IMetaState
public class State : AbstractDownloadState, IMetaState, IUpgradingState
{
[JsonIgnore]
public Uri URL => new Uri($"http://nexusmods.com/{Game.MetaData().NexusName}/mods/{ModID}");
@ -240,6 +240,41 @@ namespace Wabbajack.Lib.Downloaders
{
return new[] {"[General]", $"gameName={Game.MetaData().MO2ArchiveName}", $"modID={ModID}", $"fileID={FileID}"};
}
public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
{
var client = await NexusApiClient.Get();
var mod = await client.GetModInfo(Game, ModID);
var files = await client.GetModFiles(Game, ModID);
var oldFile = files.files.FirstOrDefault(f => f.file_id == FileID);
var newFile = files.files.OrderByDescending(f => f.uploaded_timestamp).FirstOrDefault();
if (!mod.available || oldFile == default || newFile == default)
{
return default;
}
var tempFile = new TempFile();
var newArchive = new Archive(new State {Game = Game, ModID = ModID, FileID = FileID})
{
Name = newFile.file_name,
};
await newArchive.State.Download(newArchive, tempFile.Path);
newArchive.Size = tempFile.Path.Size;
newArchive.Hash = await tempFile.Path.FileHashAsync();
return (newArchive, tempFile);
}
public bool ValidateUpgrade(AbstractDownloadState newArchiveState)
{
var state = (State)newArchiveState;
return Game == state.Game && ModID == state.ModID;
}
}
}
}

8
Wabbajack.Lib/Exceptions/HttpException.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Net.Http;
namespace Wabbajack.Lib.Exceptions
{
@ -12,5 +13,12 @@ namespace Wabbajack.Lib.Exceptions
Code = code;
Reason = reason;
}
public HttpException(HttpResponseMessage response) : base(
$"Http Error {response.StatusCode} - {response.ReasonPhrase}")
{
Code = (int)response.StatusCode;
Reason = response.ReasonPhrase;
}
}
}

80
Wabbajack.Server.Test/ABuildServerSystemTest.cs
View File

@ -1,12 +1,18 @@
using System;
using System.Collections.Generic;
using System.IO.Compression;
using System.Reactive.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Wabbajack.Common;
using Wabbajack.Common.Http;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.FileUploader;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server;
using Wabbajack.Server.DataLayer;
using Xunit;
@ -182,5 +188,79 @@ namespace Wabbajack.BuildServer.Test
_unsubErr.Dispose();
}
protected async Task<Uri> MakeModList(string modFileName)
{
var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!");
var test_archive_path = modFileName.RelativeTo(Fixture.ServerPublicFolder);
await test_archive_path.WriteAllBytesAsync(archive_data);
ModListData = new ModList();
ModListData.Archives.Add(
new Archive(new HTTPDownloader.State(MakeURL(modFileName.ToString())))
{
Hash = await test_archive_path.FileHashAsync(),
Name = "test_archive",
Size = test_archive_path.Size,
});
var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder);
await using (var fs = modListPath.Create())
{
using var za = new ZipArchive(fs, ZipArchiveMode.Create);
var entry = za.CreateEntry("modlist");
await using var es = entry.Open();
ModListData.ToJson(es);
}
ModListMetaData = new List<ModlistMetadata>
{
new ModlistMetadata
{
Official = false,
Author = "Test Suite",
Description = "A test",
DownloadMetadata = new DownloadMetadata
{
Hash = await modListPath.FileHashAsync(),
Size = modListPath.Size
},
Links = new ModlistMetadata.LinksObject
{
MachineURL = "test_list",
Download = MakeURL("test_modlist.wabbajack")
}
},
new ModlistMetadata
{
Official = true,
Author = "Test Suite",
Description = "A list with a broken hash",
DownloadMetadata = new DownloadMetadata()
{
Hash = Hash.FromLong(42),
Size = 42
},
Links = new ModlistMetadata.LinksObject
{
MachineURL = "broken_list",
Download = MakeURL("test_modlist.wabbajack")
}
}
};
var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder);
ModListMetaData.ToJson(metadataPath);
return new Uri(MakeURL("test_mod_list_metadata.json"));
}
public ModList ModListData { get; set; }
public List<ModlistMetadata> ModListMetaData { get; set; }
}
}

137
Wabbajack.Server.Test/ModListValidationTests.cs
View File

@ -25,7 +25,7 @@ namespace Wabbajack.BuildServer.Test
[Fact]
public async Task CanLoadMetadataFromTestServer()
{
var modlist = await MakeModList();
var modlist = await MakeModList("CanLoadMetadataFromTestServer.txt");
Consts.ModlistMetadataURL = modlist.ToString();
var data = await ModlistMetadata.LoadFromGithub();
Assert.Equal(2, data.Count);
@ -35,11 +35,11 @@ namespace Wabbajack.BuildServer.Test
[Fact]
public async Task CanIngestModLists()
{
var modlist = await MakeModList();
var modlist = await MakeModList("CanIngestModLists.txt");
Consts.ModlistMetadataURL = modlist.ToString();
var sql = Fixture.GetService<SqlService>();
var downloader = Fixture.GetService<ModListDownloader>();
Assert.Equal(2, await downloader.CheckForNewLists());
await downloader.CheckForNewLists();
foreach (var list in ModListMetaData)
{
@ -54,7 +54,7 @@ namespace Wabbajack.BuildServer.Test
[Fact]
public async Task CanValidateModLists()
{
var modlists = await MakeModList();
var modlists = await MakeModList("can_validate_file.txt");
Consts.ModlistMetadataURL = modlists.ToString();
Utils.Log("Updating modlists");
await RevalidateLists(true);
@ -68,7 +68,7 @@ namespace Wabbajack.BuildServer.Test
await CheckListFeeds(0, 1);
Utils.Log("Break List");
var archive = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
var archive = "can_validate_file.txt".RelativeTo(Fixture.ServerPublicFolder);
await archive.MoveToAsync(archive.WithExtension(new Extension(".moved")), true);
// We can revalidate but the non-nexus archives won't be checked yet since the list didn't change
@ -101,6 +101,58 @@ namespace Wabbajack.BuildServer.Test
await CheckListFeeds(0, 1);
}
[Fact]
public async Task CanHealLists()
{
var modlists = await MakeModList("CanHealLists.txt");
Consts.ModlistMetadataURL = modlists.ToString();
Utils.Log("Updating modlists");
await RevalidateLists(true);
Utils.Log("Checking validated results");
var data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
await CheckListFeeds(0, 1);
Utils.Log("Break List by changing the file");
var archive = "CanHealLists.txt".RelativeTo(Fixture.ServerPublicFolder);
await archive.WriteAllTextAsync("broken");
// We can revalidate but the non-nexus archives won't be checked yet since the list didn't change
await RevalidateLists(false);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
// Run the non-nexus validator
await RevalidateLists(true);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(0, data.ValidationSummary.Passed);
Assert.Equal(1, data.ValidationSummary.Updating);
var patcher = Fixture.GetService<PatchBuilder>();
Assert.Equal(1, await patcher.Execute());
await RevalidateLists(false);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
Assert.Equal(0, data.ValidationSummary.Updating);
}
private async Task RevalidateLists(bool runNonNexus)
@ -117,6 +169,9 @@ namespace Wabbajack.BuildServer.Test
var validator = Fixture.GetService<ListValidator>();
await validator.Execute();
var archiver = Fixture.GetService<ArchiveDownloader>();
await archiver.Execute();
}
private async Task CheckListFeeds(int failed, int passed)
@ -135,79 +190,7 @@ namespace Wabbajack.BuildServer.Test
}
private async Task<Uri> MakeModList()
{
var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!");
var test_archive_path = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
await test_archive_path.WriteAllBytesAsync(archive_data);
ModListData = new ModList();
ModListData.Archives.Add(
new Archive(new HTTPDownloader.State(MakeURL("test_archive.txt")))
{
Hash = await test_archive_path.FileHashAsync(),
Name = "test_archive",
Size = test_archive_path.Size,
});
var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder);
await using (var fs = modListPath.Create())
{
using var za = new ZipArchive(fs, ZipArchiveMode.Create);
var entry = za.CreateEntry("modlist");
await using var es = entry.Open();
ModListData.ToJson(es);
}
ModListMetaData = new List<ModlistMetadata>
{
new ModlistMetadata
{
Official = false,
Author = "Test Suite",
Description = "A test",
DownloadMetadata = new DownloadMetadata
{
Hash = await modListPath.FileHashAsync(),
Size = modListPath.Size
},
Links = new ModlistMetadata.LinksObject
{
MachineURL = "test_list",
Download = MakeURL("test_modlist.wabbajack")
}
},
new ModlistMetadata
{
Official = true,
Author = "Test Suite",
Description = "A list with a broken hash",
DownloadMetadata = new DownloadMetadata()
{
Hash = Hash.FromLong(42),
Size = 42
},
Links = new ModlistMetadata.LinksObject
{
MachineURL = "broken_list",
Download = MakeURL("test_modlist.wabbajack")
}
}
};
var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder);
ModListMetaData.ToJson(metadataPath);
return new Uri(MakeURL("test_mod_list_metadata.json"));
}
public ModList ModListData { get; set; }
public List<ModlistMetadata> ModListMetaData { get; set; }
}
}

139
Wabbajack.Server.Test/ModlistUpdater.cs Normal file
View File

@ -0,0 +1,139 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.BuildServer;
using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.Exceptions;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Server.Test
{
public class ModlistUpdater : ABuildServerSystemTest
{
public ModlistUpdater(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output,
fixture)
{
}
[Fact]
public async Task CanIndexAndUpdateFiles()
{
var validator = Fixture.GetService<ListValidator>();
var nonNexus = Fixture.GetService<NonNexusDownloadValidator>();
var modLists = await MakeModList("CanIndexAndUpdateFiles.txt");
Consts.ModlistMetadataURL = modLists.ToString();
var listDownloader = Fixture.GetService<ModListDownloader>();
var downloader = Fixture.GetService<ArchiveDownloader>();
var archiver = Fixture.GetService<ArchiveMaintainer>();
var patcher = Fixture.GetService<PatchBuilder>();
var sql = Fixture.GetService<SqlService>();
var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!");
var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!");
var oldDataHash = oldFileData.xxHash();
var newDataHash = newFileData.xxHash();
var oldArchive = new Archive(new NexusDownloader.State {Game = Game.Enderal, ModID = 42, FileID = 10})
{
Size = oldFileData.Length,
Hash = oldDataHash
};
var newArchive = new Archive(new NexusDownloader.State {Game = Game.Enderal, ModID = 42, FileID = 11})
{
Size = newFileData.Length,
Hash = newDataHash
};
await IngestData(archiver, oldFileData);
await IngestData(archiver, newFileData);
await sql.EnqueueDownload(oldArchive);
var oldDownload = await sql.GetNextPendingDownload();
await oldDownload.Finish(sql);
await sql.EnqueueDownload(newArchive);
var newDownload = await sql.GetNextPendingDownload();
await newDownload.Finish(sql);
await Assert.ThrowsAsync<HttpException>(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
Assert.Equal(1, await patcher.Execute());
Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_updates/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
}
[Fact]
public async Task TestEndToEndArchiveUpdating()
{
var modLists = await MakeModList("TestEndToEndArchiveUpdating.txt");
Consts.ModlistMetadataURL = modLists.ToString();
var downloader = Fixture.GetService<ArchiveDownloader>();
var archiver = Fixture.GetService<ArchiveMaintainer>();
var patcher = Fixture.GetService<PatchBuilder>();
var sql = Fixture.GetService<SqlService>();
var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!");
var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!");
var oldDataHash = oldFileData.xxHash();
var newDataHash = newFileData.xxHash();
await "TestEndToEndArchiveUpdating.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(oldFileData);
var oldArchive = new Archive(new HTTPDownloader.State(MakeURL("TestEndToEndArchiveUpdating.txt")))
{
Size = oldFileData.Length,
Hash = oldDataHash
};
await IngestData(archiver, oldFileData);
await sql.EnqueueDownload(oldArchive);
var oldDownload = await sql.GetNextPendingDownload();
await oldDownload.Finish(sql);
// Now update the file
await"TestEndToEndArchiveUpdating.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(newFileData);
using var tempFile = new TempFile();
var pendingRequest = DownloadDispatcher.DownloadWithPossibleUpgrade(oldArchive, tempFile.Path);
for (var times = 0; await downloader.Execute() == 0 && times < 40; times ++)
{
await Task.Delay(TimeSpan.FromMilliseconds(200));
}
for (var times = 0; await patcher.Execute() == 0 && times < 40; times ++)
{
await Task.Delay(TimeSpan.FromMilliseconds(200));
}
Assert.True(await pendingRequest);
Assert.Equal(oldDataHash, await tempFile.Path.FileHashAsync());
}
private async Task IngestData(ArchiveMaintainer am, byte[] data)
{
using var f = new TempFile();
await f.Path.WriteAllBytesAsync(data);
await am.Ingest(f.Path);
}
}
}

33
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -551,6 +551,39 @@ CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED
) ON [PRIMARY]
GO
/****** Object: Table [dbo].[Patches] Script Date: 5/18/2020 6:26:07 AM ******/
CREATE TABLE [dbo].[Patches](
[SrcId] [uniqueidentifier] NOT NULL,
[DestId] [uniqueidentifier] NOT NULL,
[PatchSize] [bigint] NULL,
[Finished] [datetime] NULL,
[IsFailed] [tinyint] NULL,
[FailMessage] [varchar](MAX) NULL,
CONSTRAINT [PK_Patches] PRIMARY KEY CLUSTERED
(
[SrcId] ASC,
[DestId] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY]
GO
ALTER TABLE [dbo].[Patches] WITH CHECK ADD CONSTRAINT [FK_DestId] FOREIGN KEY([DestId])
REFERENCES [dbo].[ArchiveDownloads] ([Id])
GO
ALTER TABLE [dbo].[Patches] CHECK CONSTRAINT [FK_DestId]
GO
ALTER TABLE [dbo].[Patches] WITH CHECK ADD CONSTRAINT [FK_SrcId] FOREIGN KEY([SrcId])
REFERENCES [dbo].[ArchiveDownloads] ([Id])
GO
ALTER TABLE [dbo].[Patches] CHECK CONSTRAINT [FK_SrcId]
GO
/****** Object: Table [dbo].[NexusKeys] Script Date: 5/15/2020 5:20:02 PM ******/
SET ANSI_NULLS ON
GO

59
Wabbajack.Server/Controllers/ModUpgrade.cs Normal file
View File

@ -0,0 +1,59 @@
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
[ApiController]
public class ModUpgrade : ControllerBase
{
private ILogger<ModUpgrade> _logger;
private SqlService _sql;
private DiscordWebHook _discord;
private AppSettings _settings;
public ModUpgrade(ILogger<ModUpgrade> logger, SqlService sql, DiscordWebHook discord, AppSettings settings)
{
_logger = logger;
_sql = sql;
_discord = discord;
_settings = settings;
}
[HttpPost]
[Route("/mod_upgrade")]
public async Task<IActionResult> PostModUpgrade()
{
var request = (await Request.Body.ReadAllTextAsync()).FromJsonString<ModUpgradeRequest>();
if (!request.IsValid)
{
return BadRequest("Invalid mod upgrade");
}
var oldDownload = await _sql.GetOrEnqueueArchive(request.OldArchive);
var newDownload = await _sql.GetOrEnqueueArchive(request.NewArchive);
_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash}");
var patch = await _sql.FindOrEnqueuePatch(oldDownload.Id, newDownload.Id);
if (patch.Finished.HasValue)
{
if (patch.PatchSize != 0)
{
return
Ok(
$"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{Consts.ArchiveUpdatesCDNFolder}/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
}
return NotFound("Patch creation failed");
}
// Still processing
return Accepted();
}
}
}

2
Wabbajack.Server/DTOs/ArchiveDownload.cs
View File

@ -23,7 +23,7 @@ namespace Wabbajack.Server.DTOs
public async Task Finish(SqlService service)
{
IsFailed = true;
IsFailed = false;
DownloadFinished = DateTime.UtcNow;
await service.UpdatePendingDownload(this);
}

38
Wabbajack.Server/DTOs/Patch.cs Normal file
View File

@ -0,0 +1,38 @@
using System;
using System.Threading.Tasks;
using Microsoft.VisualBasic;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.DTOs
{
public class Patch
{
public ArchiveDownload Src { get; set; }
public ArchiveDownload Dest { get; set; }
public long PatchSize { get; set; }
public DateTime? Finished { get; set; }
public bool? IsFailed { get; set; }
public string FailMessage { get; set; }
public async Task Finish(SqlService sql, long size)
{
IsFailed = false;
Finished = DateTime.UtcNow;
PatchSize = size;
await sql.FinializePatch(this);
}
public async Task Fail(SqlService sql, string msg)
{
IsFailed = true;
Finished = DateTime.UtcNow;
FailMessage = msg;
await sql.FinializePatch(this);
}
}
}

109
Wabbajack.Server/DataLayer/ArchiveDownloads.cs
View File

@ -13,6 +13,26 @@ namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task<Guid> AddKnownDownload(Archive a, DateTime downloadFinished)
{
await using var conn = await Open();
var Id = Guid.NewGuid();
await conn.ExecuteAsync(
"INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader, DownloadFinished, IsFailed) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader, @DownloadFinished, @IsFailed)",
new
{
Id = Id,
PrimaryKeyString = a.State.PrimaryKeyString,
Size = a.Size == 0 ? null : (long?)a.Size,
Hash = a.Hash == default ? null : (Hash?)a.Hash,
DownloadState = a.State,
Downloader = AbstractDownloadState.TypeToName[a.State.GetType()],
DownloadFinished = downloadFinished,
IsFailed = false
});
return Id;
}
public async Task<Guid> EnqueueDownload(Archive a)
{
await using var conn = await Open();
@ -26,7 +46,7 @@ namespace Wabbajack.Server.DataLayer
Size = a.Size == 0 ? null : (long?)a.Size,
Hash = a.Hash == default ? null : (Hash?)a.Hash,
DownloadState = a.State,
Downloader = AbstractDownloadState.TypeToName[a.State.GetType()]
Downloader = AbstractDownloadState.TypeToName[a.State.GetType()],
});
return Id;
}
@ -37,6 +57,93 @@ namespace Wabbajack.Server.DataLayer
return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, PrimaryKeyString FROM ArchiveDownloads")).ToHashSet();
}
public async Task<ArchiveDownload> GetArchiveDownload(Guid id)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE Id = @id",
new {Id = id});
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
public async Task<ArchiveDownload> GetArchiveDownload(string primaryKeyString, Hash hash, long size)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size",
new
{
PrimaryKeyString = primaryKeyString,
Hash = hash,
Size = size
});
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
public async Task<ArchiveDownload> GetOrEnqueueArchive(Archive a)
{
await using var conn = await Open();
using var trans = await conn.BeginTransactionAsync();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size",
new
{
PrimaryKeyString = a.State.PrimaryKeyString,
Hash = a.Hash,
Size = a.Size
}, trans);
if (result != default)
{
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
var id = Guid.NewGuid();
await conn.ExecuteAsync(
"INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)",
new
{
Id = id,
PrimaryKeyString = a.State.PrimaryKeyString,
Size = a.Size == 0 ? null : (long?)a.Size,
Hash = a.Hash == default ? null : (Hash?)a.Hash,
DownloadState = a.State,
Downloader = AbstractDownloadState.TypeToName[a.State.GetType()]
}, trans);
await trans.CommitAsync();
return new ArchiveDownload {Id = id, Archive = a,};
}
public async Task<ArchiveDownload> GetNextPendingDownload(bool ignoreNexus = false)
{
await using var conn = await Open();

148
Wabbajack.Server/DataLayer/Patches.cs Normal file
View File

@ -0,0 +1,148 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
/// <summary>
/// Adds a patch record
/// </summary>
/// <param name="patch"></param>
/// <returns></returns>
public async Task AddPatch(Patch patch)
{
await using var conn = await Open();
await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)",
new {SrcId = patch.Src.Id, DestId = patch.Dest.Id});
}
/// <summary>
/// Adds a patch record
/// </summary>
/// <param name="patch"></param>
/// <returns></returns>
public async Task FinializePatch(Patch patch)
{
await using var conn = await Open();
await conn.ExecuteAsync("UPDATE dbo.Patches SET PatchSize = @PatchSize, Finished = @Finished, IsFailed = @IsFailed, FailMessage = @FailMessage WHERE SrcId = @SrcId AND DestID = @DestId",
new
{
SrcId = patch.Src.Id,
DestId = patch.Dest.Id,
PatchSize = patch.PatchSize,
Finished = patch.Finished,
IsFailed = patch.IsFailed,
FailMessage = patch.FailMessage
});
}
public async Task<Patch> FindPatch(Guid src, Guid dest)
{
await using var conn = await Open();
var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool?, string)>(
@"SELECT p.PatchHash, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads src ON p.SrcId = src.Id
LEFT JOIN dbo.ArchiveDownloads dest ON p.SrcId = dest.Id
WHERE SrcId = @SrcId
AND DestId = @DestId
AND src.DownloadFinished IS NOT NULL
AND dest.DownloadFinished IS NOT NULL",
new
{
SrcId = src,
DestId = dest
});
if (patch == default)
return default;
return new Patch {
Src = await GetArchiveDownload(src),
Dest = await GetArchiveDownload(dest),
PatchSize = patch.Item1,
Finished = patch.Item2,
IsFailed = patch.Item3,
FailMessage = patch.Item4
};
}
public async Task<Patch> FindOrEnqueuePatch(Guid src, Guid dest)
{
await using var conn = await Open();
var trans = await conn.BeginTransactionAsync();
var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
"SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId",
new
{
SrcId = src,
DestId = dest
}, trans);
if (patch == default)
{
await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)",
new {SrcId = src, DestId = dest}, trans);
await trans.CommitAsync();
return new Patch {Src = await GetArchiveDownload(src), Dest = await GetArchiveDownload(dest),};
}
else
{
return new Patch {
Src = await GetArchiveDownload(src),
Dest = await GetArchiveDownload(dest),
PatchSize = patch.Item3,
Finished = patch.Item4,
IsFailed = patch.Item5,
FailMessage = patch.Item6
};
}
}
public async Task<Patch> GetPendingPatch()
{
await using var conn = await Open();
var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
"SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE Finished is NULL");
if (patch == default)
return default(Patch);
return new Patch {
Src = await GetArchiveDownload(patch.Item1),
Dest = await GetArchiveDownload(patch.Item2),
PatchSize = patch.Item3,
Finished = patch.Item4,
IsFailed = patch.Item5,
FailMessage = patch.Item6
};
}
public async Task<List<Patch>> PatchesForSource(Guid sourceDownload)
{
await using var conn = await Open();
var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
"SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId", new {SrcId = sourceDownload});
List<Patch> results = new List<Patch>();
foreach (var (srcId, destId, patchSize, finished, isFinished, failMessage) in patches)
{
results.Add( new Patch {
Src = await GetArchiveDownload(srcId),
Dest = await GetArchiveDownload(destId),
PatchSize = patchSize,
Finished = finished,
IsFailed = isFinished,
FailMessage = failMessage
});
}
return results;
}
}
}

2
Wabbajack.Server/GlobalInformation.cs
View File

@ -5,7 +5,7 @@ namespace Wabbajack.Server
public class GlobalInformation
{
public TimeSpan NexusRSSPollRate = TimeSpan.FromMinutes(1);
public TimeSpan NexusAPIPollRate = TimeSpan.FromHours(24);
public TimeSpan NexusAPIPollRate = TimeSpan.FromMinutes(15);
public DateTime LastNexusSyncUTC { get; set; }
public TimeSpan TimeSinceLastNexusSync => DateTime.UtcNow - LastNexusSyncUTC;
}

1
Wabbajack.Server/Services/ArchiveDownloader.cs
View File

@ -67,6 +67,7 @@ namespace Wabbajack.Server.Services
if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
{
_logger.Log(LogLevel.Warning, $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}");
await nextDownload.Fail(_sql, "Invalid Hash");
continue;
}

108
Wabbajack.Server/Services/ListValidator.cs
View File

@ -24,17 +24,19 @@ namespace Wabbajack.Server.Services
private SqlService _sql;
private DiscordWebHook _discord;
private NexusKeyMaintainance _nexus;
private ArchiveMaintainer _archives;
public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries { get; private set; } =
new (ModListSummary Summary, DetailedStatus Detailed)[0];
public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus)
: base(logger, settings, TimeSpan.FromMinutes(10))
public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives)
: base(logger, settings, TimeSpan.FromMinutes(5))
{
_sql = sql;
_discord = discord;
_nexus = nexus;
_archives = archives;
}
public override async Task<int> Execute()
@ -42,14 +44,19 @@ namespace Wabbajack.Server.Services
var data = await _sql.GetValidationData();
using var queue = new WorkQueue();
var oldSummaries = Summaries;
var results = await data.ModLists.PMap(queue, async list =>
{
var oldSummary =
oldSummaries.FirstOrDefault(s => s.Summary.MachineURL == list.Metadata.Links.MachineURL);
var (metadata, modList) = list;
var archives = await modList.Archives.PMap(queue, async archive =>
{
var (_, result) = await ValidateArchive(data, archive);
// TODO : auto-healing goes here
if (result == ArchiveStatus.InValid)
return await TryToHeal(data, archive);
return (archive, result);
});
@ -80,12 +87,103 @@ namespace Wabbajack.Server.Services
}).ToList()
};
if (oldSummary != default && oldSummary.Summary.Failed != summary.Failed)
{
_logger.Log(LogLevel.Information, $"Number of failures {oldSummary.Summary.Failed} -> {summary.Failed}");
if (summary.HasFailures)
{
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Embeds = new[]
{
new DiscordEmbed
{
Description =
$"Number of failures in {summary.Name} (`{summary.MachineURL}`) was {oldSummary.Summary.Failed} is now {summary.Failed}",
Url = new Uri(
$"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html")
}
}
});
}
if (!summary.HasFailures)
{
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Embeds = new[]
{
new DiscordEmbed
{
Description = $"{summary.Name} (`{summary.MachineURL}`) is now passing.",
}
}
});
}
}
return (summary, detailed);
});
Summaries = results;
return Summaries.Count(s => s.Summary.HasFailures);
}
private AsyncLock _healLock = new AsyncLock();
private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive)
{
using var _ = await _healLock.WaitAsync();
if (!(archive.State is IUpgradingState))
return (archive, ArchiveStatus.InValid);
var srcDownload = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size);
if (srcDownload == null || srcDownload.IsFailed == true)
{
return (archive, ArchiveStatus.InValid);
}
var patches = await _sql.PatchesForSource(srcDownload.Id);
foreach (var patch in patches)
{
if (patch.Finished is null)
return (archive, ArchiveStatus.Updating);
if (patch.IsFailed == true)
continue;
var (_, status) = await ValidateArchive(data, patch.Dest.Archive);
if (status == ArchiveStatus.Valid)
return (archive, ArchiveStatus.Updated);
}
var upgradeTime = DateTime.UtcNow;
var upgrade = await (archive.State as IUpgradingState)?.FindUpgrade(archive);
if (upgrade == default)
{
return (archive, ArchiveStatus.InValid);
}
await _archives.Ingest(upgrade.NewFile.Path);
var id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime);
var destDownload = await _sql.GetArchiveDownload(id);
await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload});
_logger.Log(LogLevel.Information, $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}");
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}" });
upgrade.NewFile.Dispose();
return (archive, ArchiveStatus.Updating);
}
private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(ValidationData data, Archive archive)
{
switch (archive.State)
@ -108,7 +206,7 @@ namespace Wabbajack.Server.Services
return isValid ? (archive, ArchiveStatus.Valid) : (archive, ArchiveStatus.InValid);
}
return (archive, ArchiveStatus.InValid);
return (archive, ArchiveStatus.Valid);
}
}
}

6
Wabbajack.Server/Services/NexusPoll.cs
View File

@ -66,7 +66,7 @@ namespace Wabbajack.Server.Services
public async Task UpdateNexusCacheAPI()
{
using var _ = _logger.BeginScope("Nexus Update via API");
_logger.Log(LogLevel.Information, "Starting");
_logger.Log(LogLevel.Information, "Starting Nexus Update via API");
var api = await NexusApiClient.Get();
var gameTasks = GameRegistry.Games.Values
@ -117,7 +117,7 @@ namespace Wabbajack.Server.Services
public void Start()
{
if (!_settings.RunBackEndJobs) return;
/*
Task.Run(async () =>
{
while (true)
@ -133,7 +133,7 @@ namespace Wabbajack.Server.Services
await Task.Delay(_globalInformation.NexusRSSPollRate);
}
});
*/
Task.Run(async () =>
{
while (true)

108
Wabbajack.Server/Services/PatchBuilder.cs Normal file
View File

@ -0,0 +1,108 @@
using System;
using System.IO;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
using Microsoft.Extensions.Logging;
using Splat;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class PatchBuilder : AbstractService<PatchBuilder, int>
{
private DiscordWebHook _discordWebHook;
private SqlService _sql;
private ArchiveMaintainer _maintainer;
public PatchBuilder(ILogger<PatchBuilder> logger, SqlService sql, AppSettings settings, ArchiveMaintainer maintainer,
DiscordWebHook discordWebHook) : base(logger, settings, TimeSpan.FromMinutes(1))
{
_discordWebHook = discordWebHook;
_sql = sql;
_maintainer = maintainer;
}
public override async Task<int> Execute()
{
int count = 0;
while (true)
{
var patch = await _sql.GetPendingPatch();
if (patch == default) break;
try
{
_logger.LogInformation(
$"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}");
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
_maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath);
_maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath);
var patchName = $"{Consts.ArchiveUpdatesCDNFolder}\\{patch.Src.Archive.Hash.ToHex()}_{patch.Dest.Archive.Hash.ToHex()}";
using var sigFile = new TempFile();
await using var srcStream = srcPath.OpenShared();
await using var destStream = destPath.OpenShared();
await using var sigStream = sigFile.Path.Create();
using var ftpClient = await GetBunnyCdnFtpClient();
if (!await ftpClient.DirectoryExistsAsync(Consts.ArchiveUpdatesCDNFolder))
await ftpClient.CreateDirectoryAsync(Consts.ArchiveUpdatesCDNFolder);
await using var patchOutput = await ftpClient.OpenWriteAsync(patchName);
OctoDiff.Create(destStream, srcStream, sigStream, patchOutput);
await patchOutput.DisposeAsync();
var size = await ftpClient.GetFileSizeAsync(patchName);
await patch.Finish(_sql, size);
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Built {size.ToFileSizeString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
}
catch (Exception ex)
{
_logger.LogError(ex, "Error while building patch");
await patch.Fail(_sql, ex.ToString());
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Failure building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
}
count++;
}
return count;
}
private async Task<FtpClient> GetBunnyCdnFtpClient()
{
var info = Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
await client.ConnectAsync();
return client;
}
}
}

2
Wabbajack.Server/Startup.cs
View File

@ -65,6 +65,7 @@ namespace Wabbajack.Server
services.AddSingleton<ArchiveDownloader>();
services.AddSingleton<DiscordWebHook>();
services.AddSingleton<NexusKeyMaintainance>();
services.AddSingleton<PatchBuilder>();
services.AddMvc();
services.AddControllers()
@ -116,6 +117,7 @@ namespace Wabbajack.Server
app.UseService<ArchiveDownloader>();
app.UseService<DiscordWebHook>();
app.UseService<NexusKeyMaintainance>();
app.UseService<PatchBuilder>();
app.Use(next =>
{