Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Can heal a simple pre-indexed download

Browse Source
This commit is contained in:
Timothy Baldridge 2020-05-19 21:25:41 -06:00
parent 2a8021c6f9
commit 03f5ff6c92
19 changed files with 564 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Wabbajack.Common/Http/Client.cs
View File

@ -19,11 +19,17 @@ namespace Wabbajack.Common.Http
return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions);
}
public async Task<HttpResponseMessage> GetAsync(Uri url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions);
}
public async Task<HttpResponseMessage> PostAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead)
public async Task<HttpResponseMessage> PostAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true)
{
var request = new HttpRequestMessage(HttpMethod.Post, url) {Content = content};
return await SendAsync(request, responseHeadersRead);
return await SendAsync(request, responseHeadersRead, errorsAsExceptions);
}
public async Task<HttpResponseMessage> PutAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead)
@ -79,7 +85,6 @@ namespace Wabbajack.Common.Http
if (errorsAsExceptions)
throw new HttpRequestException(
$"Http Exception {response.StatusCode} - {response.ReasonPhrase} - {msg.RequestUri}");
;
return response;
}
catch (Exception ex)

4
Wabbajack.Common/OctoDiff.cs
View File

@ -28,7 +28,7 @@ namespace Wabbajack.Common
return sigStream;
}
private static void CreateSignature(Stream oldData, FileStream sigStream)
private static void CreateSignature(Stream oldData, Stream sigStream)
{
Utils.Status("Creating Patch Signature");
var signatureBuilder = new SignatureBuilder();
@ -36,7 +36,7 @@ namespace Wabbajack.Common
sigStream.Position = 0;
}
public static void Create(Stream oldData, FileStream newData, FileStream signature, FileStream output)
public static void Create(Stream oldData, Stream newData, Stream signature, Stream output)
{
CreateSignature(oldData, signature);
var db = new DeltaBuilder {ProgressReporter = reporter};

7
Wabbajack.Common/Paths.cs
View File

@ -337,6 +337,13 @@ namespace Wabbajack.Common
await fs.WriteAsync(data);
}
public async Task WriteAllAsync(Stream data, bool disposeAfter = true)
{
await using var fs = Create();
await fs.CopyToAsync(data);
if (disposeAfter) await data.DisposeAsync();
}
public void AppendAllText(string text)
{
File.AppendAllText(_path, text);

72
Wabbajack.Lib/ClientAPI.cs
View File

@ -1,11 +1,46 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.Exceptions;
namespace Wabbajack.Lib
{
[JsonName("ModUpgradeRequest")]
public class ModUpgradeRequest
{
public Archive OldArchive { get; set; }
public Archive NewArchive { get; set; }
public ModUpgradeRequest(Archive oldArchive, Archive newArchive)
{
OldArchive = oldArchive;
NewArchive = newArchive;
}
public bool IsValid
{
get
{
if (OldArchive.Hash == NewArchive.Hash && OldArchive.State.PrimaryKeyString == NewArchive.State.PrimaryKeyString) return false;
if (OldArchive.State.GetType() != NewArchive.State.GetType())
return false;
if (OldArchive.State is IUpgradingState u)
{
return u.ValidateUpgrade(NewArchive.State);
}
return false;
}
}
}
public class ClientAPI
{
public static Common.Http.Client GetClient()
@ -16,18 +51,39 @@ namespace Wabbajack.Lib
return client;
}
public static async Task<Archive?> GetModUpgrade(Hash hash)
public static async Task<Uri> GetModUpgrade(Archive oldArchive, Archive newArchive, TimeSpan? maxWait = null, TimeSpan? waitBetweenTries = null)
{
using var response = await GetClient()
.GetAsync($"{Consts.WabbajackBuildServerUri}alternative/{hash.ToHex()}");
maxWait ??= TimeSpan.FromMinutes(10);
waitBetweenTries ??= TimeSpan.FromSeconds(15);
var request = new ModUpgradeRequest( oldArchive, newArchive);
var start = DateTime.UtcNow;
RETRY:
var response = await GetClient()
.PostAsync($"{Consts.WabbajackBuildServerUri}mod_upgrade", new StringContent(request.ToJson(), Encoding.UTF8, "application/json"));
if (response.IsSuccessStatusCode)
{
return (await response.Content.ReadAsStringAsync()).FromJsonString<Archive>();
switch (response.StatusCode)
{
case HttpStatusCode.OK:
return new Uri(await response.Content.ReadAsStringAsync());
case HttpStatusCode.Accepted:
Utils.Log($"Waiting for patch processing on the server for {oldArchive.Name}, sleeping for another 15 seconds");
await Task.Delay(TimeSpan.FromSeconds(15));
response.Dispose();
if (DateTime.UtcNow - start > maxWait)
throw new HttpException(response);
goto RETRY;
}
}
Utils.Log($"No Upgrade for {hash}");
Utils.Log(await response.Content.ReadAsStringAsync());
return null;
var ex = new HttpException(response);
response.Dispose();
throw ex;
}
/// <summary>

63
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
@ -95,40 +97,49 @@ namespace Wabbajack.Lib.Downloaders
return true;
}
Utils.Log($"Download failed, looking for upgrade");
var upgrade = await ClientAPI.GetModUpgrade(archive.Hash);
if (upgrade == null)
if (!(archive.State is IUpgradingState))
{
Utils.Log($"No upgrade found for {archive.Hash}");
Utils.Log($"Download failed for {archive.Name} and no upgrade from this download source is possible");
return false;
}
Utils.Log($"Upgrading via {upgrade.State.PrimaryKeyString}");
var upgrade = (IUpgradingState)archive.State;
Utils.Log($"Trying to find solution to broken download for {archive.Name}");
Utils.Log($"Upgrading {archive.Hash}");
var upgradePath = destination.Parent.Combine("_Upgrade_" + archive.Name);
var upgradeResult = await Download(upgrade, upgradePath);
if (!upgradeResult) return false;
var patchName = $"{archive.Hash.ToHex()}_{upgrade.Hash.ToHex()}";
var patchPath = destination.Parent.Combine("_Patch_" + patchName);
var patchState = new Archive(new HTTPDownloader.State($"https://wabbajackcdn.b-cdn.net/updates/{patchName}"))
var result = await upgrade.FindUpgrade(archive);
if (result == default)
{
Name = patchName,
};
Utils.Log(
$"No solution for broken download {archive.Name} {archive.State.PrimaryKeyString} could be found");
return false;
var patchResult = await Download(patchState, patchPath);
if (!patchResult) return false;
Utils.Status($"Applying Upgrade to {archive.Hash}");
await using (var patchStream = patchPath.OpenRead())
await using (var srcStream = upgradePath.OpenRead())
await using (var destStream = destination.Create())
{
OctoDiff.Apply(srcStream, patchStream, destStream);
}
await destination.FileHashCachedAsync();
Utils.Log($"Looking for patch for {archive.Name}");
var patchResult = await ClientAPI.GetModUpgrade(archive, result.Archive!);
Utils.Log($"Downloading patch for {archive.Name}");
var tempFile = new TempFile();
using var response = await (new Common.Http.Client()).GetAsync(patchResult);
await tempFile.Path.WriteAllAsync(await response.Content.ReadAsStreamAsync());
response.Dispose();
Utils.Log($"Applying patch to {archive.Name}");
await using(var src = result.NewFile.Path.OpenShared())
await using (var final = destination.Create())
{
Utils.ApplyPatch(src, () => tempFile.Path.OpenShared(), final);
}
var hash = await destination.FileHashCachedAsync();
if (hash != archive.Hash && archive.Hash != default)
{
Utils.Log("Archive hash didn't match after patching");
return false;
}
return true;
}

18
Wabbajack.Lib/Downloaders/IUpgradingState.cs Normal file
View File

@ -0,0 +1,18 @@
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.Lib.Downloaders
{
public interface IUpgradingState
{
/// <summary>
/// Find a possible archive that can be combined with a server generated patch to get the input archive
/// state;
/// </summary>
/// <param name="a"></param>
/// <returns></returns>
public Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a);
bool ValidateUpgrade(AbstractDownloadState newArchiveState);
}
}

37
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -140,7 +140,7 @@ namespace Wabbajack.Lib.Downloaders
}
[JsonName("NexusDownloader")]
public class State : AbstractDownloadState, IMetaState
public class State : AbstractDownloadState, IMetaState, IUpgradingState
{
[JsonIgnore]
public Uri URL => new Uri($"http://nexusmods.com/{Game.MetaData().NexusName}/mods/{ModID}");
@ -240,6 +240,41 @@ namespace Wabbajack.Lib.Downloaders
{
return new[] {"[General]", $"gameName={Game.MetaData().MO2ArchiveName}", $"modID={ModID}", $"fileID={FileID}"};
}
public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
{
var client = await NexusApiClient.Get();
var mod = await client.GetModInfo(Game, ModID);
var files = await client.GetModFiles(Game, ModID);
var oldFile = files.files.FirstOrDefault(f => f.file_id == FileID);
var newFile = files.files.OrderByDescending(f => f.uploaded_timestamp).FirstOrDefault();
if (!mod.available || oldFile == default || newFile == default)
{
return default;
}
var tempFile = new TempFile();
var newArchive = new Archive(new State {Game = Game, ModID = ModID, FileID = FileID})
{
Name = newFile.file_name,
};
await newArchive.State.Download(newArchive, tempFile.Path);
newArchive.Size = tempFile.Path.Size;
newArchive.Hash = await tempFile.Path.FileHashAsync();
return (newArchive, tempFile);
}
public bool ValidateUpgrade(AbstractDownloadState newArchiveState)
{
var state = (State)newArchiveState;
return Game == state.Game && ModID == state.ModID;
}
}
}
}

8
Wabbajack.Lib/Exceptions/HttpException.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Net.Http;
namespace Wabbajack.Lib.Exceptions
{
@ -12,5 +13,12 @@ namespace Wabbajack.Lib.Exceptions
Code = code;
Reason = reason;
}
public HttpException(HttpResponseMessage response) : base(
$"Http Error {response.StatusCode} - {response.ReasonPhrase}")
{
Code = (int)response.StatusCode;
Reason = response.ReasonPhrase;
}
}
}

107
Wabbajack.Server.Test/ModlistUpdater.cs
View File

@ -9,6 +9,7 @@ using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.Exceptions;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
@ -38,97 +39,49 @@ namespace Wabbajack.Server.Test
var listDownloader = Fixture.GetService<ModListDownloader>();
var downloader = Fixture.GetService<ArchiveDownloader>();
var archiver = Fixture.GetService<ArchiveMaintainer>();
var patcher = Fixture.GetService<PatchBuilder>();
var sql = Fixture.GetService<SqlService>();
var modId = long.MaxValue >> 1;
var oldFileId = long.MaxValue >> 2;
var newFileId = (long.MaxValue >> 2) + 1;
var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!");
var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!");
var oldDataHash = oldFileData.xxHash();
var newDataHash = newFileData.xxHash();
Assert.Equal(2, await listDownloader.CheckForNewLists());
Assert.Equal(1, await downloader.Execute());
Assert.Equal(0, await nonNexus.Execute());
Assert.Equal(0, await validator.Execute());
Assert.True(archiver.HaveArchive(oldDataHash));
Assert.False(archiver.HaveArchive(newDataHash));
var status = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.Equal(0, status.ValidationSummary.Failed);
// Update the archive
await "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(newFileData);
// Nothing new to do
Assert.Equal(0, await listDownloader.CheckForNewLists());
Assert.Equal(0, await downloader.Execute());
// List now fails after we check the manual link
Assert.Equal(1, await nonNexus.Execute());
Assert.Equal(1, await validator.Execute());
/*
Assert.True(await sql.HaveIndexdFile(oldDataHash));
Assert.True(await sql.HaveIndexdFile(newDataHash));
var settings = Fixture.GetService<AppSettings>();
Assert.Equal($"Oldfile_{oldDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(oldDataHash));
Assert.Equal($"Newfile_{newDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(newDataHash));
Utils.Log($"Download Updating {oldDataHash} -> {newDataHash}");
await using var conn = await sql.Open();
await conn.ExecuteAsync("DELETE FROM dbo.DownloadStates WHERE Hash in (@OldHash, @NewHash);",
new {OldHash = (long)oldDataHash, NewHash = (long)newDataHash});
await sql.AddDownloadState(oldDataHash, new NexusDownloader.State
var oldArchive = new Archive(new NexusDownloader.State {Game = Game.Enderal, ModID = 42, FileID = 10})
{
Game = Game.Oblivion,
ModID = modId,
FileID = oldFileId
});
await sql.AddDownloadState(newDataHash, new NexusDownloader.State
Size = oldFileData.Length,
Hash = oldDataHash
};
var newArchive = new Archive(new NexusDownloader.State {Game = Game.Enderal, ModID = 42, FileID = 11})
{
Game = Game.Oblivion,
ModID = modId,
FileID = newFileId
});
Size = newFileData.Length,
Hash = newDataHash
};
await IngestData(archiver, oldFileData);
await IngestData(archiver, newFileData);
await sql.EnqueueDownload(oldArchive);
var oldDownload = await sql.GetNextPendingDownload();
await oldDownload.Finish(sql);
await sql.EnqueueDownload(newArchive);
var newDownload = await sql.GetNextPendingDownload();
await newDownload.Finish(sql);
Assert.NotNull(await sql.GetNexusStateByHash(oldDataHash));
Assert.NotNull(await sql.GetNexusStateByHash(newDataHash));
// No nexus info, so no upgrade
var noUpgrade = await ClientAPI.GetModUpgrade(oldDataHash);
Assert.Null(noUpgrade);
await Assert.ThrowsAsync<HttpException>(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
Assert.Equal(1, await patcher.Execute());
// Add Nexus info
await sql.AddNexusModFiles(Game.Oblivion, modId, DateTime.Now,
new NexusApiClient.GetModFilesResponse
{
files = new List<NexusFileInfo>
{
new NexusFileInfo {category_name = "MAIN", file_id = newFileId, file_name = "New File"},
new NexusFileInfo {category_name = null, file_id = oldFileId, file_name = "Old File"}
}
});
Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_upgrades/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
var enqueuedUpgrade = await ClientAPI.GetModUpgrade(oldDataHash);
// Not Null because upgrade was enqueued
Assert.NotNull(enqueuedUpgrade);
}
await RunAllJobs();
Assert.True($"{oldDataHash.ToHex()}_{newDataHash.ToHex()}".RelativeTo(Fixture.ServerUpdatesFolder).IsFile);
*/
private async Task IngestData(ArchiveMaintainer am, byte[] data)
{
using var f = new TempFile();
await f.Path.WriteAllBytesAsync(data);
await am.Ingest(f.Path);
}
}
}

33
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -551,6 +551,39 @@ CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED
) ON [PRIMARY]
GO
/****** Object: Table [dbo].[Patches] Script Date: 5/18/2020 6:26:07 AM ******/
CREATE TABLE [dbo].[Patches](
[SrcId] [uniqueidentifier] NOT NULL,
[DestId] [uniqueidentifier] NOT NULL,
[PatchSize] [bigint] NULL,
[Finished] [datetime] NULL,
[IsFailed] [tinyint] NULL,
[FailMessage] [varchar](MAX) NULL,
CONSTRAINT [PK_Patches] PRIMARY KEY CLUSTERED
(
[SrcId] ASC,
[DestId] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY]
GO
ALTER TABLE [dbo].[Patches] WITH CHECK ADD CONSTRAINT [FK_DestId] FOREIGN KEY([DestId])
REFERENCES [dbo].[ArchiveDownloads] ([Id])
GO
ALTER TABLE [dbo].[Patches] CHECK CONSTRAINT [FK_DestId]
GO
ALTER TABLE [dbo].[Patches] WITH CHECK ADD CONSTRAINT [FK_SrcId] FOREIGN KEY([SrcId])
REFERENCES [dbo].[ArchiveDownloads] ([Id])
GO
ALTER TABLE [dbo].[Patches] CHECK CONSTRAINT [FK_SrcId]
GO
/****** Object: Table [dbo].[NexusKeys] Script Date: 5/15/2020 5:20:02 PM ******/
SET ANSI_NULLS ON
GO

2
Wabbajack.Server/Controllers/AuthoredFiles.cs
View File

@ -99,7 +99,7 @@ namespace Wabbajack.BuildServer.Controllers
private async Task<FtpClient> GetBunnyCdnFtpClient()
{
var info = Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
var info = Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-patch-ftp-info");
var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
await client.ConnectAsync();
return client;

58
Wabbajack.Server/Controllers/ModUpgrade.cs Normal file
View File

@ -0,0 +1,58 @@
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
[ApiController]
public class ModUpgrade : ControllerBase
{
private ILogger<ModUpgrade> _logger;
private SqlService _sql;
private DiscordWebHook _discord;
private AppSettings _settings;
public ModUpgrade(ILogger<ModUpgrade> logger, SqlService sql, DiscordWebHook discord, AppSettings settings)
{
_logger = logger;
_sql = sql;
_discord = discord;
_settings = settings;
}
[HttpPost]
[Route("/mod_upgrade")]
public async Task<IActionResult> PostModUpgrade()
{
var request = (await Request.Body.ReadAllTextAsync()).FromJsonString<ModUpgradeRequest>();
if (!request.IsValid)
{
return BadRequest("Invalid mod upgrade");
}
var oldDownload = await _sql.GetOrEnqueueArchive(request.OldArchive);
var newDownload = await _sql.GetOrEnqueueArchive(request.NewArchive);
var patch = await _sql.FindOrEnqueuePatch(oldDownload.Id, newDownload.Id);
if (patch.Finished.HasValue)
{
if (patch.PatchSize != 0)
{
return
Ok(
$"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/archive_upgrades/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
}
return NotFound("Patch creation failed");
}
// Still processing
return Accepted();
}
}
}

26
Wabbajack.Server/DTOs/Patch.cs
View File

@ -1,5 +1,8 @@
using System;
using System.Threading.Tasks;
using Microsoft.VisualBasic;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.DTOs
{
@ -7,8 +10,29 @@ namespace Wabbajack.Server.DTOs
{
public ArchiveDownload Src { get; set; }
public ArchiveDownload Dest { get; set; }
public Hash PatchHash { get; set; }
public long PatchSize { get; set; }
public DateTime? Finished { get; set; }
public bool? IsFailed { get; set; }
public string FailMessage { get; set; }
public async Task Finish(SqlService sql, long size)
{
IsFailed = false;
Finished = DateTime.UtcNow;
PatchSize = size;
await sql.FinializePatch(this);
}
public async Task Fail(SqlService sql, string msg)
{
IsFailed = true;
Finished = DateTime.UtcNow;
FailMessage = msg;
await sql.FinializePatch(this);
}
}
}

67
Wabbajack.Server/DataLayer/ArchiveDownloads.cs
View File

@ -56,6 +56,73 @@ namespace Wabbajack.Server.DataLayer
};
}
public async Task<ArchiveDownload> GetArchiveDownload(string primaryKeyString, Hash hash, long size)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size",
new
{
PrimaryKeyString = primaryKeyString,
Hash = hash,
Size = size
});
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
public async Task<ArchiveDownload> GetOrEnqueueArchive(Archive a)
{
await using var conn = await Open();
using var trans = await conn.BeginTransactionAsync();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size",
new
{
PrimaryKeyString = a.State.PrimaryKeyString,
Hash = a.Hash,
Size = a.Size
}, trans);
if (result != default)
{
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
var id = Guid.NewGuid();
await conn.ExecuteAsync(
"INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)",
new
{
Id = id,
PrimaryKeyString = a.State.PrimaryKeyString,
Size = a.Size == 0 ? null : (long?)a.Size,
Hash = a.Hash == default ? null : (Hash?)a.Hash,
DownloadState = a.State,
Downloader = AbstractDownloadState.TypeToName[a.State.GetType()]
}, trans);
await trans.CommitAsync();
return new ArchiveDownload {Id = id, Archive = a,};
}
public async Task<ArchiveDownload> GetNextPendingDownload(bool ignoreNexus = false)
{

78
Wabbajack.Server/DataLayer/Patches.cs
View File

@ -28,38 +28,98 @@ namespace Wabbajack.Server.DataLayer
public async Task FinializePatch(Patch patch)
{
await using var conn = await Open();
await conn.ExecuteAsync("UPDATE dbo.Patches SET PatchSize = @Size, PatchHash = @PatchHash, Finished = @Finished WHERE SrcId = @SrcId AND DestID = @DestId",
await conn.ExecuteAsync("UPDATE dbo.Patches SET PatchSize = @PatchSize, Finished = @Finished, IsFailed = @IsFailed, FailMessage = @FailMessage WHERE SrcId = @SrcId AND DestID = @DestId",
new
{
SrcId = patch.Src.Id,
DestId = patch.Dest.Id,
PatchHash = patch.PatchHash,
PatchSize = patch.PatchSize,
Finshed = patch.Finished
Finished = patch.Finished,
IsFailed = patch.IsFailed,
FailMessage = patch.FailMessage
});
}
public async Task<Patch> FindPatch(Guid src, Guid dest)
{
await using var conn = await Open();
var patch = await conn.QueryFirstOrDefaultAsync<(Hash, long, DateTime?)>(
"SELECT PatchHash, PatchSize, Finished FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId",
var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool?, string)>(
@"SELECT p.PatchHash, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads src ON p.SrcId = src.Id
LEFT JOIN dbo.ArchiveDownloads dest ON p.SrcId = dest.Id
WHERE SrcId = @SrcId
AND DestId = @DestId
AND src.DownloadFinished IS NOT NULL
AND dest.DownloadFinished IS NOT NULL",
new
{
SrcId = src,
DestId = dest
});
if (patch == default)
return default(Patch);
return default;
return new Patch {
Src = await GetArchiveDownload(src),
Dest = await GetArchiveDownload(dest),
PatchHash = patch.Item1,
PatchSize = patch.Item2,
Finished = patch.Item3
PatchSize = patch.Item1,
Finished = patch.Item2,
IsFailed = patch.Item3,
FailMessage = patch.Item4
};
}
public async Task<Patch> FindOrEnqueuePatch(Guid src, Guid dest)
{
await using var conn = await Open();
var trans = await conn.BeginTransactionAsync();
var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool, string)>(
"SELECT PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId",
new
{
SrcId = src,
DestId = dest
}, trans);
if (patch == default)
{
await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)",
new {SrcId = src, DestId = dest}, trans);
await trans.CommitAsync();
return new Patch {Src = await GetArchiveDownload(src), Dest = await GetArchiveDownload(dest),};
}
else
{
await trans.CommitAsync();
return new Patch {
Src = await GetArchiveDownload(src),
Dest = await GetArchiveDownload(dest),
PatchSize = patch.Item1,
Finished = patch.Item2,
IsFailed = patch.Item3,
FailMessage = patch.Item4
};
}
}
public async Task<Patch> GetPendingPatch()
{
await using var conn = await Open();
var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
"SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE Finished is NULL");
if (patch == default)
return default(Patch);
return new Patch {
Src = await GetArchiveDownload(patch.Item1),
Dest = await GetArchiveDownload(patch.Item2),
PatchSize = patch.Item3,
Finished = patch.Item4,
IsFailed = patch.Item5,
FailMessage = patch.Item6
};
}
}
}

2
Wabbajack.Server/GlobalInformation.cs
View File

@ -5,7 +5,7 @@ namespace Wabbajack.Server
public class GlobalInformation
{
public TimeSpan NexusRSSPollRate = TimeSpan.FromMinutes(1);
public TimeSpan NexusAPIPollRate = TimeSpan.FromHours(24);
public TimeSpan NexusAPIPollRate = TimeSpan.FromMinutes(15);
public DateTime LastNexusSyncUTC { get; set; }
public TimeSpan TimeSinceLastNexusSync => DateTime.UtcNow - LastNexusSyncUTC;
}

6
Wabbajack.Server/Services/NexusPoll.cs
View File

@ -66,7 +66,7 @@ namespace Wabbajack.Server.Services
public async Task UpdateNexusCacheAPI()
{
using var _ = _logger.BeginScope("Nexus Update via API");
_logger.Log(LogLevel.Information, "Starting");
_logger.Log(LogLevel.Information, "Starting Nexus Update via API");
var api = await NexusApiClient.Get();
var gameTasks = GameRegistry.Games.Values
@ -117,7 +117,7 @@ namespace Wabbajack.Server.Services
public void Start()
{
if (!_settings.RunBackEndJobs) return;
/*
Task.Run(async () =>
{
while (true)
@ -133,7 +133,7 @@ namespace Wabbajack.Server.Services
await Task.Delay(_globalInformation.NexusRSSPollRate);
}
});
*/
Task.Run(async () =>
{
while (true)

95
Wabbajack.Server/Services/PatchBuilder.cs Normal file
View File

@ -0,0 +1,95 @@
using System;
using System.IO;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
using Microsoft.Extensions.Logging;
using Splat;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class PatchBuilder : AbstractService<PatchBuilder, int>
{
private DiscordWebHook _discordWebHook;
private SqlService _sql;
private ArchiveMaintainer _maintainer;
public PatchBuilder(ILogger<PatchBuilder> logger, SqlService sql, AppSettings settings, ArchiveMaintainer maintainer,
DiscordWebHook discordWebHook) : base(logger, settings, TimeSpan.FromMinutes(1))
{
_discordWebHook = discordWebHook;
_sql = sql;
_maintainer = maintainer;
}
public override async Task<int> Execute()
{
int count = 0;
while (true)
{
var patch = await _sql.GetPendingPatch();
if (patch == default) break;
try
{
_logger.LogInformation(
$"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}");
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
_maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath);
_maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath);
var patchName = $"archive_updates\\{patch.Src.Archive.Hash}_{patch.Dest.Archive.Hash}";
using var sigFile = new TempFile();
await using var srcStream = srcPath.OpenShared();
await using var destStream = destPath.OpenShared();
await using var sigStream = sigFile.Path.Create();
using var ftpClient = await GetBunnyCdnFtpClient();
if (!await ftpClient.DirectoryExistsAsync("archive_updates"))
await ftpClient.CreateDirectoryAsync("archive_updates");
await using var patchOutput = await ftpClient.OpenWriteAsync(patchName);
OctoDiff.Create(destStream, srcStream, sigStream, patchOutput);
await patchOutput.DisposeAsync();
var size = await ftpClient.GetFileSizeAsync(patchName);
await patch.Finish(_sql, size);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error while building patch");
await patch.Fail(_sql, ex.ToString());
}
count++;
}
return count;
}
private async Task<FtpClient> GetBunnyCdnFtpClient()
{
var info = Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
await client.ConnectAsync();
return client;
}
}
}

2
Wabbajack.Server/Startup.cs
View File

@ -65,6 +65,7 @@ namespace Wabbajack.Server
services.AddSingleton<ArchiveDownloader>();
services.AddSingleton<DiscordWebHook>();
services.AddSingleton<NexusKeyMaintainance>();
services.AddSingleton<PatchBuilder>();
services.AddMvc();
services.AddControllers()
@ -116,6 +117,7 @@ namespace Wabbajack.Server
app.UseService<ArchiveDownloader>();
app.UseService<DiscordWebHook>();
app.UseService<NexusKeyMaintainance>();
app.UseService<PatchBuilder>();
app.Use(next =>
{