diff --git a/Wabbajack.Common/Exceptions/HttpException.cs b/Wabbajack.Common/Exceptions/HttpException.cs index b34b4f80..aba72e11 100644 --- a/Wabbajack.Common/Exceptions/HttpException.cs +++ b/Wabbajack.Common/Exceptions/HttpException.cs @@ -1,8 +1,10 @@ using System; using System.Net.Http; +using Wabbajack.Common.Serialization.Json; namespace Wabbajack.Common.Exceptions { + [JsonName("HttpException")] public class HttpException : Exception { public string Reason { get; set; } diff --git a/Wabbajack.Common/Http/Client.cs b/Wabbajack.Common/Http/Client.cs index c57b78d2..644a51e5 100644 --- a/Wabbajack.Common/Http/Client.cs +++ b/Wabbajack.Common/Http/Client.cs @@ -77,27 +77,33 @@ namespace Wabbajack.Common.Http if (Cookies.Count > 0) Cookies.ForEach(c => ClientFactory.Cookies.Add(c)); int retries = 0; + HttpResponseMessage response; TOP: try { - var response = await ClientFactory.Client.SendAsync(msg, responseHeadersRead); + response = await ClientFactory.Client.SendAsync(msg, responseHeadersRead); if (response.IsSuccessStatusCode) return response; if (errorsAsExceptions) + { + response.Dispose(); throw new HttpException(response); - + } + return response; } catch (Exception ex) { if (ex is HttpException http) { - if (http.Code != 503) throw; + if (http.Code < 500) throw; + retries++; var ms = Utils.NextRandom(100, 1000); Utils.Log($"Got a 503 from {msg.RequestUri} retrying in {ms}ms"); await Task.Delay(ms); + msg = CloneMessage(msg); goto TOP; } if (retries > Consts.MaxHTTPRetries) throw; diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index 7000115a..fef0d841 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -27,6 +27,7 @@ namespace Wabbajack.Lib { get { + if (OldArchive.Size > 2_500_000_000 || NewArchive.Size > 2_500_000_000) return false; if (OldArchive.Hash == NewArchive.Hash && OldArchive.State.PrimaryKeyString == NewArchive.State.PrimaryKeyString) return false; if (OldArchive.State.GetType() != NewArchive.State.GetType()) return false; diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index f92d3cbd..6eb1d8da 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -231,6 +231,11 @@ TOP: newArchive.Hash = await tmpFile.Path.FileHashAsync(); newArchive.Size = tmpFile.Path.Size; + if (newArchive.Hash == a.Hash || a.Size > 2_500_000_000 || newArchive.Size > 2_500_000_000) + { + return default; + } + return (newArchive, tmpFile); } diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 11d08438..81c20273 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -248,12 +248,17 @@ namespace Wabbajack.Lib.Downloaders var mod = await client.GetModInfo(Game, ModID); var files = await client.GetModFiles(Game, ModID); var oldFile = files.files.FirstOrDefault(f => f.file_id == FileID); - var newFile = files.files.OrderByDescending(f => f.uploaded_timestamp).FirstOrDefault(); + var newFile = files.files.Where(f => f.category_name != null).OrderByDescending(f => f.uploaded_timestamp).FirstOrDefault(); if (!mod.available || oldFile == default || newFile == default) { return default; } + // Size is in KB + if (oldFile.size > 2_500_000 || newFile.size > 2_500_000 || oldFile.file_id == newFile.file_id) + { + return default; + } var tempFile = new TempFile(); diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 1bc68c28..130a7651 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -170,7 +170,7 @@ namespace Wabbajack.Server.Services var upgrade = await (archive.State as IUpgradingState)?.FindUpgrade(archive); if (upgrade == default) { - _logger.Log(LogLevel.Information, $"Cannot heal {archive.State} because an alternative wasn't found"); + _logger.Log(LogLevel.Information, $"Cannot heal {archive.State.PrimaryKeyString} because an alternative wasn't found"); return (archive, ArchiveStatus.InValid); } @@ -178,7 +178,7 @@ namespace Wabbajack.Server.Services var id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime); var destDownload = await _sql.GetArchiveDownload(id); - + await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload}); _logger.Log(LogLevel.Information, $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}"); diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index f7e95817..682296e6 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging; using Splat; using Wabbajack.BuildServer; using Wabbajack.Common; +using Wabbajack.Lib; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Server.DataLayer; using Wabbajack.Server.DTOs; @@ -33,6 +34,8 @@ namespace Wabbajack.Server.Services int count = 0; while (true) { + count++; + var patch = await _sql.GetPendingPatch(); if (patch == default) break; @@ -48,6 +51,18 @@ namespace Wabbajack.Server.Services $"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}" }); + if (patch.Src.Archive.Hash == patch.Dest.Archive.Hash) + { + await patch.Fail(_sql, "Hashes match"); + continue; + } + + if (patch.Src.Archive.Size > 2_500_000_000 || patch.Dest.Archive.Size > 2_500_000_000) + { + await patch.Fail(_sql, "Too large to patch"); + continue; + } + _maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath); _maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath); @@ -86,8 +101,6 @@ namespace Wabbajack.Server.Services }); } - - count++; } return count;