From 8af841f3d6fbf31daa2881a9c7c7f0b875169aff Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 2 Apr 2020 21:57:59 -0600 Subject: [PATCH] Fix several more tests --- Compression.BSA.Test/BSATests.cs | 4 +- Compression.BSA/BA2Builder.cs | 64 ++++----- .../IndexedFilesTests.cs | 23 +-- .../Wabbajack.BuildServer.Test.csproj | 3 + .../sql/wabbajack_db.sql | 12 +- .../Controllers/IndexedFiles.cs | 6 +- .../Controllers/ListValidation.cs | 10 +- .../Controllers/ModlistUpdater.cs | 10 +- .../Controllers/NexusCache.cs | 6 +- .../Models/Jobs/EnqueueRecentFiles.cs | 2 +- .../Models/Jobs/GetNexusUpdatesJob.cs | 2 +- .../Models/Jobs/UpdateModLists.cs | 9 +- .../Models/NexusCacheData.cs | 2 +- .../Models/Sql/SqlService.cs | 134 +++++++++++++++--- Wabbajack.Common/Util/DiskSlabAllocator.cs | 11 +- Wabbajack.Common/Util/TempFile.cs | 9 +- .../Downloaders/AbstractIPS4Downloader.cs | 2 +- .../Downloaders/BethesdaNetDownloader.cs | 2 +- .../Downloaders/DownloadDispatcher.cs | 4 +- .../Downloaders/DropboxDownloader.cs | 2 +- .../Downloaders/GameFileSourceDownloader.cs | 2 +- .../Downloaders/GoogleDriveDownloader.cs | 2 +- Wabbajack.Lib/Downloaders/HTTPDownloader.cs | 2 +- Wabbajack.Lib/Downloaders/IDownloader.cs | 2 +- Wabbajack.Lib/Downloaders/MEGADownloader.cs | 2 +- Wabbajack.Lib/Downloaders/ManualDownloader.cs | 2 +- .../Downloaders/MediaFireDownloader.cs | 2 +- Wabbajack.Lib/Downloaders/ModDBDownloader.cs | 2 +- Wabbajack.Lib/Downloaders/NexusDownloader.cs | 37 +++-- .../Downloaders/SteamWorkshopDownloader.cs | 2 +- .../Downloaders/YouTubeDownloader.cs | 2 +- Wabbajack.Lib/FileUploader/AuthorAPI.cs | 10 +- Wabbajack.Lib/NexusApi/Dtos.cs | 10 +- .../ContentRightsManagementTests.cs | 4 +- Wabbajack.Test/DownloaderTests.cs | 14 +- 35 files changed, 268 insertions(+), 144 deletions(-) diff --git a/Compression.BSA.Test/BSATests.cs b/Compression.BSA.Test/BSATests.cs index 5156e8c2..afc8cbb0 100644 --- a/Compression.BSA.Test/BSATests.cs +++ b/Compression.BSA.Test/BSATests.cs @@ -60,9 +60,9 @@ namespace Compression.BSA.Test var state = new NexusDownloader.State { - ModID = mod.ToString(), + ModID = mod, Game = game, - FileID = file.file_id.ToString() + FileID = file.file_id }; await state.Download(src); return src; diff --git a/Compression.BSA/BA2Builder.cs b/Compression.BSA/BA2Builder.cs index 696fec6d..e596f8df 100644 --- a/Compression.BSA/BA2Builder.cs +++ b/Compression.BSA/BA2Builder.cs @@ -4,6 +4,7 @@ using System.IO; using System.IO.MemoryMappedFiles; using System.Linq; using System.Text; +using ICSharpCode.SharpZipLib.Zip.Compression; using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using Wabbajack.Common; @@ -56,40 +57,38 @@ namespace Compression.BSA public void Build(AbsolutePath filename) { SortEntries(); - using (var fs = filename.Create()) - using (var bw = new BinaryWriter(fs)) + using var fs = filename.Create(); + using var bw = new BinaryWriter(fs); + + bw.Write(Encoding.ASCII.GetBytes(_state.HeaderMagic)); + bw.Write(_state.Version); + bw.Write(Encoding.ASCII.GetBytes(Enum.GetName(typeof(EntryType), _state.Type))); + bw.Write((uint)_entries.Count); + var table_offset_loc = bw.BaseStream.Position; + bw.Write((ulong)0); + + foreach (var entry in _entries) { - bw.Write(Encoding.ASCII.GetBytes(_state.HeaderMagic)); - bw.Write(_state.Version); - bw.Write(Encoding.ASCII.GetBytes(Enum.GetName(typeof(EntryType), _state.Type))); - bw.Write((uint)_entries.Count); - var table_offset_loc = bw.BaseStream.Position; - bw.Write((ulong)0); + entry.WriteHeader(bw); + } - foreach (var entry in _entries) - { - entry.WriteHeader(bw); - } + foreach (var entry in _entries) + { + entry.WriteData(bw); + } - foreach (var entry in _entries) - { - entry.WriteData(bw); - } + if (!_state.HasNameTable) return; - if (_state.HasNameTable) - { - var pos = bw.BaseStream.Position; - bw.BaseStream.Seek(table_offset_loc, SeekOrigin.Begin); - bw.Write((ulong) pos); - bw.BaseStream.Seek(pos, SeekOrigin.Begin); + var pos = bw.BaseStream.Position; + bw.BaseStream.Seek(table_offset_loc, SeekOrigin.Begin); + bw.Write((ulong)pos); + bw.BaseStream.Seek(pos, SeekOrigin.Begin); - foreach (var entry in _entries) - { - var bytes = Encoding.UTF7.GetBytes(entry.FullName); - bw.Write((ushort)bytes.Length); - bw.BaseStream.Write(bytes, 0, bytes.Length); - } - } + foreach (var entry in _entries) + { + var bytes = Encoding.UTF7.GetBytes(entry.FullName); + bw.Write((ushort)bytes.Length); + bw.BaseStream.Write(bytes, 0, bytes.Length); } } @@ -108,8 +107,8 @@ namespace Compression.BSA { var builder = new BA2DX10FileEntryBuilder {_state = state}; - var header_size = DDS.HeaderSizeForFormat((DXGI_FORMAT) state.PixelFormat) + 4; - new BinaryReader(src).ReadBytes((int)header_size); + var headerSize = DDS.HeaderSizeForFormat((DXGI_FORMAT) state.PixelFormat) + 4; + new BinaryReader(src).ReadBytes((int)headerSize); // This can't be parallel because it all runs off the same base IO stream. builder._chunks = new List(); @@ -169,8 +168,9 @@ namespace Compression.BSA } else { + var deflater = new Deflater(Deflater.BEST_COMPRESSION); using var ms = new MemoryStream(); - using (var ds = new DeflaterOutputStream(ms)) + using (var ds = new DeflaterOutputStream(ms, deflater)) { ds.IsStreamOwner = false; src.CopyToLimit(ds, (int)chunk.FullSz); diff --git a/Wabbajack.BuildServer.Test/IndexedFilesTests.cs b/Wabbajack.BuildServer.Test/IndexedFilesTests.cs index d266f98a..ff001810 100644 --- a/Wabbajack.BuildServer.Test/IndexedFilesTests.cs +++ b/Wabbajack.BuildServer.Test/IndexedFilesTests.cs @@ -49,34 +49,35 @@ namespace Wabbajack.BuildServer.Test [Fact] public async Task CanNotifyOfInis() { - var files = await @"sql\NotifyStates".RelativeTo(AbsolutePath.EntryPoint) - .EnumerateFiles() - .Where(f => f.Extension == Consts.IniExtension) - .PMap(Queue, async ini => (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(ini.LoadIniFile()))); - - var archives = files.Select(f => + var archive = new Archive { - State = f, + State = new NexusDownloader.State + { + Game = Game.SkyrimSpecialEdition, + ModID = long.MaxValue >> 3, + FileID = long.MaxValue >> 3, + }, Name = Guid.NewGuid().ToString() - }); - Assert.True(await AuthorAPI.UploadPackagedInis(archives)); + }; + Assert.True(await AuthorAPI.UploadPackagedInis(new[] {archive})); var SQL = Fixture.GetService(); var job = await SQL.GetJob(); + Assert.NotNull(job); Assert.IsType(job.Payload); var payload = (IndexJob)job.Payload; Assert.IsType(payload.Archive.State); var casted = (NexusDownloader.State)payload.Archive.State; - Assert.Equal(Game.Skyrim, casted.Game); + Assert.Equal(Game.SkyrimSpecialEdition, casted.Game); // Insert the record into SQL await SQL.AddDownloadState(Hash.FromHex("00e8bbbf591f61a3"), casted); // Enqueue the same file again - Assert.True(await AuthorAPI.UploadPackagedInis(archives)); + Assert.True(await AuthorAPI.UploadPackagedInis(new[] {archive})); // File is aleady indexed so nothing gets enqueued Assert.Null(await SQL.GetJob()); diff --git a/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj b/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj index 401bc87b..5d137ab6 100644 --- a/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj +++ b/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj @@ -49,6 +49,9 @@ Always + + Always + diff --git a/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql b/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql index d0319faa..72734f00 100644 --- a/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql +++ b/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql @@ -277,9 +277,9 @@ GO CREATE TABLE [dbo].[Metrics]( [Id] [bigint] IDENTITY(1,1) NOT NULL, [Timestamp] [datetime] NOT NULL, - [Action] [varchar](64) NOT NULL, - [Subject] [varchar](max) NOT NULL, - [MetricsKey] [varchar](64) NULL, + [Action] [nvarchar](64) NOT NULL, + [Subject] [nvarchar](max) NOT NULL, + [MetricsKey] [nvarchar](64) NULL, [GroupingSubject] AS (substring([Subject],(0),case when patindex('%[0-9].%',[Subject])=(0) then len([Subject])+(1) else patindex('%[0-9].%',[Subject]) end)), CONSTRAINT [PK_Metrics] PRIMARY KEY CLUSTERED ( @@ -326,9 +326,9 @@ GO CREATE TABLE [dbo].[DownloadStates]( [Id] [binary](32) NOT NULL, [Hash] [bigint] NOT NULL, - [PrimaryKey] [varchar](max) NOT NULL, - [IniState] [varchar](max) NOT NULL, - [JsonState] [varchar](max) NOT NULL, + [PrimaryKey] [nvarchar](max) NOT NULL, + [IniState] [nvarchar](max) NOT NULL, + [JsonState] [nvarchar](max) NOT NULL, CONSTRAINT [PK_DownloadStates] PRIMARY KEY CLUSTERED ( [Id] ASC diff --git a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs index b42cae5c..4f3b465a 100644 --- a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs @@ -60,7 +60,7 @@ namespace Wabbajack.BuildServer.Controllers int loadCount = 0; foreach (var file in fullPath.EnumerateFiles().Where(f => f.Extension == Consts.IniExtension)) { - var loaded = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile())); + var loaded = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile(), true)); if (loaded == null) { Utils.Log($"Unsupported Ini {file}"); @@ -88,7 +88,9 @@ namespace Wabbajack.BuildServer.Controllers { await using var ins = entry.Open(); var iniString = Encoding.UTF8.GetString(await ins.ReadAllAsync()); - var data = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(iniString.LoadIniString())); + Utils.Log(iniString); + var data = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(iniString.LoadIniString(), true)); + if (data == null) { Utils.Log("No valid INI parser for: \n" + iniString); diff --git a/Wabbajack.BuildServer/Controllers/ListValidation.cs b/Wabbajack.BuildServer/Controllers/ListValidation.cs index f0d0a82f..0c4c59f3 100644 --- a/Wabbajack.BuildServer/Controllers/ListValidation.cs +++ b/Wabbajack.BuildServer/Controllers/ListValidation.cs @@ -27,9 +27,9 @@ namespace Wabbajack.BuildServer.Controllers [HttpGet] [Route("status.json")] - public async Task> HandleGetLists() + public async Task> HandleGetLists() { - return await Db.ModListStatus.AsQueryable().Select(m => m.Summary).ToListAsync(); + return await SQL.GetModListSummaries(); } private static readonly Func HandleGetRssFeedTemplate = NettleEngine.GetCompiler().Compile(@" @@ -53,7 +53,7 @@ namespace Wabbajack.BuildServer.Controllers [Route("status/{Name}/broken.rss")] public async Task HandleGetRSSFeed(string Name) { - var lst = (await ModListStatus.ByName(Db, Name)).DetailedStatus; + var lst = await SQL.GetDetailedModlistStatus(Name); var response = HandleGetRssFeedTemplate(new { lst, @@ -91,7 +91,7 @@ namespace Wabbajack.BuildServer.Controllers public async Task HandleGetListHtml(string Name) { - var lst = (await ModListStatus.ByName(Db, Name)).DetailedStatus; + var lst = await SQL.GetDetailedModlistStatus(Name); var response = HandleGetListTemplate(new { lst, @@ -112,7 +112,7 @@ namespace Wabbajack.BuildServer.Controllers public async Task HandleGetListJson(string Name) { - var lst = (await ModListStatus.ByName(Db, Name)).DetailedStatus; + var lst = await SQL.GetDetailedModlistStatus(Name); lst.Archives.Do(a => a.Archive.Meta = null); return new ContentResult { diff --git a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs index 2298e390..a695d386 100644 --- a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs +++ b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs @@ -41,8 +41,8 @@ namespace Wabbajack.BuildServer.Controllers [Route("/delete_updates")] public async Task DeleteUpdates() { - var lists = await Db.ModListStatus.AsQueryable().ToListAsync(); - var archives = lists.SelectMany(list => list.DetailedStatus.Archives) + var lists = await SQL.GetDetailedModlistStatuses(); + var archives = lists.SelectMany(list => list.Archives) .Select(a => a.Archive.Hash.ToHex()) .ToHashSet(); @@ -108,7 +108,7 @@ namespace Wabbajack.BuildServer.Controllers .ToListAsync(); if (mod_files.SelectMany(f => f.Data.files) - .Any(f => f.category_name != null && f.file_id.ToString() == nexusState.FileID)) + .Any(f => f.category_name != null && f.file_id == nexusState.FileID)) { await Metric("not_required_upgrade", startingHash.ToString()); return BadRequest("Upgrade Not Required"); @@ -164,7 +164,7 @@ namespace Wabbajack.BuildServer.Controllers { var origSize = _settings.PathForArchive(srcHash).Size; var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - var allMods = await api.GetModFiles(state.Game, int.Parse(state.ModID)); + var allMods = await api.GetModFiles(state.Game, state.ModID); var archive = allMods.files.Where(m => !string.IsNullOrEmpty(m.category_name)) .OrderBy(s => Math.Abs((long)s.size - origSize)) .Select(s => new Archive { @@ -174,7 +174,7 @@ namespace Wabbajack.BuildServer.Controllers { Game = state.Game, ModID = state.ModID, - FileID = s.file_id.ToString() + FileID = s.file_id }}).FirstOrDefault(); if (archive == null) diff --git a/Wabbajack.BuildServer/Controllers/NexusCache.cs b/Wabbajack.BuildServer/Controllers/NexusCache.cs index 68fe9160..b07e4801 100644 --- a/Wabbajack.BuildServer/Controllers/NexusCache.cs +++ b/Wabbajack.BuildServer/Controllers/NexusCache.cs @@ -116,14 +116,14 @@ namespace Wabbajack.BuildServer.Controllers foreach (var record in data.ModInfos) { - await SQL.AddNexusModInfo(GameRegistry.GetByFuzzyName(record.Game).Game, long.Parse(record.ModId), + await SQL.AddNexusModInfo(GameRegistry.GetByFuzzyName(record.Game).Game, record.ModId, record.LastCheckedUTC, record.Data); totalRows += 1; } foreach (var record in data.FileInfos) { - await SQL.AddNexusFileInfo(GameRegistry.GetByFuzzyName(record.Game).Game, long.Parse(record.ModId), + await SQL.AddNexusFileInfo(GameRegistry.GetByFuzzyName(record.Game).Game, record.ModId, long.Parse(record.FileId), record.LastCheckedUTC, record.Data); totalRows += 1; @@ -131,7 +131,7 @@ namespace Wabbajack.BuildServer.Controllers foreach (var record in data.ModFiles) { - await SQL.AddNexusModFiles(GameRegistry.GetByFuzzyName(record.Game).Game, long.Parse(record.ModId), + await SQL.AddNexusModFiles(GameRegistry.GetByFuzzyName(record.Game).Game, record.ModId, record.LastCheckedUTC, record.Data); totalRows += 1; } diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs index 71205465..8e92f0a8 100644 --- a/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs +++ b/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs @@ -53,7 +53,7 @@ namespace Wabbajack.BuildServer.Models.Jobs { var state = new NexusDownloader.State { - Game = tuple.Game, ModID = tuple.ModId.ToString(), FileID = tuple.File.file_id.ToString() + Game = tuple.Game, ModID = tuple.ModId, FileID = tuple.File.file_id }; return new Archive {State = state, Name = tuple.File.file_name}; }).ToList(); diff --git a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs index c1d2e9f9..4598d82b 100644 --- a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs @@ -56,7 +56,7 @@ namespace Wabbajack.BuildServer.Models.Jobs // Mod activity could hide files var b = d.mod.LastestModActivity.AsUnixTime(); - return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.ModId.ToString()}; + return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.ModId}; }); var purged = await collected.PMap(queue, async t => diff --git a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs index 9f978b60..a3b0a3b9 100644 --- a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs +++ b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs @@ -178,8 +178,6 @@ namespace Wabbajack.BuildServer.Models.Jobs return false; var game = gameMeta.Game; - if (!int.TryParse(state.ModID, out var modID)) - return false; var modFiles = (await db.NexusModFiles.AsQueryable().Where(g => g.Game == gameMeta.NexusName && g.ModId == state.ModID).FirstOrDefaultAsync())?.Data; @@ -187,14 +185,11 @@ namespace Wabbajack.BuildServer.Models.Jobs { Utils.Log($"No Cache for {state.PrimaryKeyString} falling back to HTTP"); var nexusApi = await NexusApiClient.Get(); - modFiles = await nexusApi.GetModFiles(game, modID); + modFiles = await nexusApi.GetModFiles(game, state.ModID); } - if (!ulong.TryParse(state.FileID, out var fileID)) - return false; - var found = modFiles.files - .FirstOrDefault(file => file.file_id == fileID && file.category_name != null); + .FirstOrDefault(file => file.file_id == state.FileID && file.category_name != null); return found != null; } catch (Exception ex) diff --git a/Wabbajack.BuildServer/Models/NexusCacheData.cs b/Wabbajack.BuildServer/Models/NexusCacheData.cs index a28ca5f5..9a25e6bd 100644 --- a/Wabbajack.BuildServer/Models/NexusCacheData.cs +++ b/Wabbajack.BuildServer/Models/NexusCacheData.cs @@ -13,7 +13,7 @@ namespace Wabbajack.BuildServer.Models public string Game { get; set; } [BsonIgnoreIfNull] - public string ModId { get; set; } + public long ModId { get; set; } public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow; diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index 4702a463..6df50608 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections; +using System; using System.Collections.Generic; using System.Data; using System.Data.SqlClient; @@ -7,15 +6,13 @@ using System.Linq; using System.Threading.Tasks; using Dapper; using Microsoft.Extensions.Configuration; -using Microsoft.VisualBasic; using Newtonsoft.Json; -using ReactiveUI; -using Wabbajack.BuildServer.GraphQL; using Wabbajack.BuildServer.Model.Models.Results; using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.NexusApi; using Wabbajack.VirtualFileSystem; @@ -197,7 +194,7 @@ namespace Wabbajack.BuildServer.Model.Models await conn.ExecuteAsync( @"INSERT INTO dbo.Jobs (Created, Priority, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @Payload, @OnSuccess)", new { - Priority = job.Priority, + job.Priority, Payload = job.Payload.ToJSON(), OnSuccess = job.OnSuccess?.ToJSON() ?? null}); } @@ -212,8 +209,8 @@ namespace Wabbajack.BuildServer.Model.Models await using var conn = await Open(); await conn.ExecuteAsync( @"UPDATE dbo.Jobs SET Finshed = GETDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id", - new { - Id = job.Id, + new { + job.Id, Success = job.Result.ResultType == JobResultType.Success, ResultPayload = job.Result.ToJSON() @@ -259,7 +256,7 @@ namespace Wabbajack.BuildServer.Model.Models public override AJobPayload Parse(object value) { - return Utils.FromJSONString((string)value); + return ((string)value).FromJSONString(); } } @@ -288,12 +285,12 @@ namespace Wabbajack.BuildServer.Model.Models new { Id = uf.Id.ToString(), - Name = uf.Name, - Size = uf.Size, + uf.Name, + uf.Size, UploadedBy = uf.Uploader, Hash = (long)uf.Hash, - UploadDate = uf.UploadDate, - CDNName = uf.CDNName + uf.UploadDate, + uf.CDNName }); } @@ -334,10 +331,115 @@ namespace Wabbajack.BuildServer.Model.Models public async Task HaveIndexedArchivePrimaryKey(string key) { await using var conn = await Open(); - var results = await conn.QueryAsync( - "SELECT * FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey", + var results = await conn.QueryFirstOrDefaultAsync( + "SELECT PrimaryKey FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey", new {PrimaryKey = key}); - return results.Any(); + return results != null; } + + public async Task AddNexusFileInfo(Game game, long modId, long fileId, DateTime lastCheckedUtc, NexusFileInfo data) + { + await using var conn = await Open(); + + await conn.ExecuteAsync("INSERT INTO dbo.NexusFileInfos (Game, ModId, FileId, LastChecked, Data) VALUES " + + "(@Game, @ModId, @FileId, @LastChecked, @Data)", + new + { + Game = game.MetaData().NexusGameId, + ModId = modId, + FileId = fileId, + LastChecked = lastCheckedUtc, + Data = JsonConvert.SerializeObject(data) + }); + + } + + public async Task AddNexusModInfo(Game game, long modId, DateTime lastCheckedUtc, ModInfo data) + { + await using var conn = await Open(); + + await conn.ExecuteAsync( + @"MERGE dbo.NexusModInfos AS Target + USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source + ON Target.Game = Source.Game AND Target.ModId = Source.ModId + WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked + WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);", + new + { + Game = game.MetaData().NexusGameId, + ModId = modId, + LastChecked = lastCheckedUtc, + Data = JsonConvert.SerializeObject(data) + }); + + } + + public async Task AddNexusModFiles(Game game, long modId, DateTime lastCheckedUtc, NexusApiClient.GetModFilesResponse data) + { + await using var conn = await Open(); + + await conn.ExecuteAsync( + @"MERGE dbo.NexusModFiles AS Target + USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source + ON Target.Game = Source.Game AND Target.ModId = Source.ModId + WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked + WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);", + new + { + Game = game.MetaData().NexusGameId, + ModId = modId, + LastChecked = lastCheckedUtc, + Data = JsonConvert.SerializeObject(data) + }); + + } + + public async Task GetNexusModInfoString(Game game, long modId) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync( + "SELECT Data FROM dbo.NexusModInfos WHERE Game = @Game AND @ModId = ModId", + new {Game = game.MetaData().NexusGameId, ModId = modId}); + return result == null ? null : JsonConvert.DeserializeObject(result); + } + + public async Task GetModFiles(Game game, long modId) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync( + "SELECT Data FROM dbo.NexusModFiles WHERE Game = @Game AND @ModId = ModId", + new {Game = game.MetaData().NexusGameId, ModId = modId}); + return result == null ? null : JsonConvert.DeserializeObject(result); + } + + #region ModLists + public async Task> GetModListSummaries() + { + await using var conn = await Open(); + var results = await conn.QueryAsync("SELECT Summary from dbo.ModLists"); + return results.Select(s => s.FromJSONString()).ToList(); + } + + public async Task GetDetailedModlistStatus(string machineUrl) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync("SELECT DetailedStatus from dbo.ModLists WHERE MachineURL = @MachineURL", + new + { + machineUrl + }); + return result.FromJSONString(); + } + + public async Task> GetDetailedModlistStatuses() + { + await using var conn = await Open(); + var results = await conn.QueryAsync("SELECT DetailedStatus from dbo.ModLists"); + return results.Select(s => s.FromJSONString()).ToList(); + } + + + + #endregion } } diff --git a/Wabbajack.Common/Util/DiskSlabAllocator.cs b/Wabbajack.Common/Util/DiskSlabAllocator.cs index 067047fc..57f217bf 100644 --- a/Wabbajack.Common/Util/DiskSlabAllocator.cs +++ b/Wabbajack.Common/Util/DiskSlabAllocator.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.IO.MemoryMappedFiles; @@ -14,11 +15,14 @@ namespace Wabbajack.Common private readonly MemoryMappedFile _mmap; private long _head = 0; private readonly FileStream _fileStream; + private List _allocated = new List(); + private long _size; public DiskSlabAllocator(long size) { _file = new TempFile(); _fileStream = _file.File.Open(FileMode.Create, FileAccess.ReadWrite); + _size = size; _mmap = MemoryMappedFile.CreateFromFile(_fileStream, null, size, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, false); } @@ -26,14 +30,19 @@ namespace Wabbajack.Common { lock (this) { + if (_head + size >= _size) + throw new InvalidDataException($"Size out of range. Declared {_size} used {_head + size}"); var startAt = _head; _head += size; - return _mmap.CreateViewStream(startAt, size, MemoryMappedFileAccess.ReadWrite); + var stream = _mmap.CreateViewStream(startAt, size, MemoryMappedFileAccess.ReadWrite); + _allocated.Add(stream); + return stream; } } public void Dispose() { + _allocated.Do(s => s.Dispose()); _mmap?.Dispose(); _fileStream?.Dispose(); _file?.Dispose(); diff --git a/Wabbajack.Common/Util/TempFile.cs b/Wabbajack.Common/Util/TempFile.cs index e5dcab1b..782df0ab 100644 --- a/Wabbajack.Common/Util/TempFile.cs +++ b/Wabbajack.Common/Util/TempFile.cs @@ -16,10 +16,17 @@ namespace Wabbajack.Common public bool DeleteAfter = true; public TempFile(bool deleteAfter = true, bool createFolder = true) - : this(new FileInfo((string)((AbsolutePath)AlphaPath.GetTempPath()).Combine(AlphaPath.GetRandomFileName()))) + : this(new FileInfo((string)GetTempFilePath())) { } + private static AbsolutePath GetTempFilePath() + { + var path = (@"temp\" + Guid.NewGuid()).RelativeTo(AbsolutePath.EntryPoint); + path.Parent.CreateDirectory(); + return path; + } + public TempFile(FileInfo file, bool deleteAfter = true, bool createFolder = true) { this.File = file; diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs index 3f2d7172..cd02c798 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs @@ -24,7 +24,7 @@ namespace Wabbajack.Lib.Downloaders public override string SiteName { get; } public override Uri SiteURL { get; } - public async Task GetDownloaderState(dynamic archiveINI) + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) { Uri url = DownloaderUtils.GetDirectURL(archiveINI); diff --git a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs index 31a1d69c..25d7a4eb 100644 --- a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs +++ b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs @@ -39,7 +39,7 @@ namespace Wabbajack.Lib.Downloaders var result = await Utils.Log(new RequestBethesdaNetLogin()).Task; } - public async Task GetDownloaderState(dynamic archiveINI) + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) { var url = (Uri)DownloaderUtils.GetDirectURL(archiveINI); return StateFromUrl(url); diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index 6123133e..645c1f9f 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -62,9 +62,9 @@ namespace Wabbajack.Lib.Downloaders return inst; } - public static async Task ResolveArchive(dynamic ini) + public static async Task ResolveArchive(dynamic ini, bool quickMode = false) { - var states = await Task.WhenAll(Downloaders.Select(d => (Task)d.GetDownloaderState(ini))); + var states = await Task.WhenAll(Downloaders.Select(d => (Task)d.GetDownloaderState(ini, quickMode))); return states.FirstOrDefault(result => result != null); } diff --git a/Wabbajack.Lib/Downloaders/DropboxDownloader.cs b/Wabbajack.Lib/Downloaders/DropboxDownloader.cs index 0986193a..6a2da893 100644 --- a/Wabbajack.Lib/Downloaders/DropboxDownloader.cs +++ b/Wabbajack.Lib/Downloaders/DropboxDownloader.cs @@ -7,7 +7,7 @@ namespace Wabbajack.Lib.Downloaders { public class DropboxDownloader : IDownloader, IUrlDownloader { - public async Task GetDownloaderState(dynamic archiveINI) + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) { var urlstring = archiveINI?.General?.directURL; return GetDownloaderState(urlstring); diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs index f102d7a2..844e0ce3 100644 --- a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs @@ -14,7 +14,7 @@ namespace Wabbajack.Lib.Downloaders { public class GameFileSourceDownloader : IDownloader { - public async Task GetDownloaderState(dynamic archiveINI) + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) { var gameName = (string)archiveINI?.General?.gameName; var gameFile = (string)archiveINI?.General?.gameFile; diff --git a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs index 21822e05..79df3478 100644 --- a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs @@ -10,7 +10,7 @@ namespace Wabbajack.Lib.Downloaders { public class GoogleDriveDownloader : IDownloader, IUrlDownloader { - public async Task GetDownloaderState(dynamic archiveINI) + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) { var url = archiveINI?.General?.directURL; return GetDownloaderState(url); diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index 9cf9ab53..cf000dca 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -16,7 +16,7 @@ namespace Wabbajack.Lib.Downloaders public class HTTPDownloader : IDownloader, IUrlDownloader { - public async Task GetDownloaderState(dynamic archiveINI) + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) { var url = archiveINI?.General?.directURL; return GetDownloaderState(url, archiveINI); diff --git a/Wabbajack.Lib/Downloaders/IDownloader.cs b/Wabbajack.Lib/Downloaders/IDownloader.cs index eac0bf50..ddac3407 100644 --- a/Wabbajack.Lib/Downloaders/IDownloader.cs +++ b/Wabbajack.Lib/Downloaders/IDownloader.cs @@ -4,7 +4,7 @@ namespace Wabbajack.Lib.Downloaders { public interface IDownloader { - Task GetDownloaderState(dynamic archiveINI); + Task GetDownloaderState(dynamic archiveINI, bool quickMode = false); /// /// Called before any downloads are inacted by the installer; diff --git a/Wabbajack.Lib/Downloaders/MEGADownloader.cs b/Wabbajack.Lib/Downloaders/MEGADownloader.cs index 0780c3d1..d1c92f40 100644 --- a/Wabbajack.Lib/Downloaders/MEGADownloader.cs +++ b/Wabbajack.Lib/Downloaders/MEGADownloader.cs @@ -7,7 +7,7 @@ namespace Wabbajack.Lib.Downloaders { public class MegaDownloader : IDownloader, IUrlDownloader { - public async Task GetDownloaderState(dynamic archiveINI) + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) { var url = archiveINI?.General?.directURL; return GetDownloaderState(url); diff --git a/Wabbajack.Lib/Downloaders/ManualDownloader.cs b/Wabbajack.Lib/Downloaders/ManualDownloader.cs index db33df6a..3721f116 100644 --- a/Wabbajack.Lib/Downloaders/ManualDownloader.cs +++ b/Wabbajack.Lib/Downloaders/ManualDownloader.cs @@ -61,7 +61,7 @@ namespace Wabbajack.Lib.Downloaders } } - public async Task GetDownloaderState(dynamic archiveINI) + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) { var url = archiveINI?.General?.manualURL; return url != null ? new State { Url = url} : null; diff --git a/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs b/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs index 03d920ec..03e09d90 100644 --- a/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs +++ b/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs @@ -10,7 +10,7 @@ namespace Wabbajack.Lib.Downloaders { public class MediaFireDownloader : IUrlDownloader { - public async Task GetDownloaderState(dynamic archiveINI) + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) { Uri url = DownloaderUtils.GetDirectURL(archiveINI); if (url == null || url.Host != "www.mediafire.com") return null; diff --git a/Wabbajack.Lib/Downloaders/ModDBDownloader.cs b/Wabbajack.Lib/Downloaders/ModDBDownloader.cs index 8865b570..fd3d218b 100644 --- a/Wabbajack.Lib/Downloaders/ModDBDownloader.cs +++ b/Wabbajack.Lib/Downloaders/ModDBDownloader.cs @@ -13,7 +13,7 @@ namespace Wabbajack.Lib.Downloaders { public class ModDBDownloader : IDownloader, IUrlDownloader { - public async Task GetDownloaderState(dynamic archiveINI) + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) { var url = archiveINI?.General?.directURL; return GetDownloaderState(url); diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 4f9184ba..f310aacf 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -51,13 +51,23 @@ namespace Wabbajack.Lib.Downloaders canExecute: IsLoggedIn.ObserveOnGuiThread()); } - public async Task GetDownloaderState(dynamic archiveINI) + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) { var general = archiveINI?.General; if (general.modID != null && general.fileID != null && general.gameName != null) { var game = GameRegistry.GetByFuzzyName((string)general.gameName).Game; + if (quickMode) + { + return new State + { + Game = GameRegistry.GetByFuzzyName((string)general.gameName).Game, + ModID = long.Parse(general.modID), + FileID = long.Parse(general.fileID), + }; + } + var client = await NexusApiClient.Get(); ModInfo info; try @@ -79,8 +89,8 @@ namespace Wabbajack.Lib.Downloaders IsNSFW = info.contains_adult_content, Description = NexusApiUtils.FixupSummary(info.summary), Game = GameRegistry.GetByFuzzyName((string)general.gameName).Game, - ModID = general.modID, - FileID = general.fileID + ModID = long.Parse(general.modID), + FileID = long.Parse(general.fileID) }; } @@ -151,9 +161,9 @@ namespace Wabbajack.Lib.Downloaders public Game Game { get; set; } [Key(7)] - public string ModID { get; set; } + public long ModID { get; set; } [Key(8)] - public string FileID { get; set; } + public long FileID { get; set; } public async Task LoadMetaData() { @@ -195,22 +205,11 @@ namespace Wabbajack.Lib.Downloaders { try { - var gameMeta = Game.MetaData(); - if (gameMeta == null) - return false; - - var game = gameMeta.Game; - if (!int.TryParse(ModID, out var modID)) - return false; - var client = await NexusApiClient.Get(); - var modFiles = await client.GetModFiles(game, modID); - - if (!ulong.TryParse(FileID, out var fileID)) - return false; + var modFiles = await client.GetModFiles(Game, ModID); var found = modFiles.files - .FirstOrDefault(file => file.file_id == fileID && file.category_name != null); + .FirstOrDefault(file => file.file_id == FileID && file.category_name != null); return found != null; } catch (Exception ex) @@ -233,7 +232,7 @@ namespace Wabbajack.Lib.Downloaders public override string[] GetMetaIni() { - return new[] {"[General]", $"gameName={Game.MetaData().MO2Name}", $"modID={ModID}", $"fileID={FileID}"}; + return new[] {"[General]", $"gameName={Game.MetaData().MO2ArchiveName}", $"modID={ModID}", $"fileID={FileID}"}; } } } diff --git a/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs b/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs index ff67116f..96cfadde 100644 --- a/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs +++ b/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs @@ -16,7 +16,7 @@ namespace Wabbajack.Lib.Downloaders { private SteamWorkshopItem _item; - public async Task GetDownloaderState(dynamic archiveINI) + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) { var id = archiveINI?.General?.itemID; var steamID = archiveINI?.General?.steamID; diff --git a/Wabbajack.Lib/Downloaders/YouTubeDownloader.cs b/Wabbajack.Lib/Downloaders/YouTubeDownloader.cs index 4538ca28..bd568752 100644 --- a/Wabbajack.Lib/Downloaders/YouTubeDownloader.cs +++ b/Wabbajack.Lib/Downloaders/YouTubeDownloader.cs @@ -20,7 +20,7 @@ namespace Wabbajack.Lib.Downloaders { public class YouTubeDownloader : IDownloader { - public async Task GetDownloaderState(dynamic archiveINI) + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode) { var directURL = (Uri)DownloaderUtils.GetDirectURL(archiveINI); var state = (State)UriToState(directURL); diff --git a/Wabbajack.Lib/FileUploader/AuthorAPI.cs b/Wabbajack.Lib/FileUploader/AuthorAPI.cs index 2e605ef1..442b0135 100644 --- a/Wabbajack.Lib/FileUploader/AuthorAPI.cs +++ b/Wabbajack.Lib/FileUploader/AuthorAPI.cs @@ -158,8 +158,14 @@ namespace Wabbajack.Lib.FileUploader } var client = new Common.Http.Client(); - await client.PostAsync($"{Consts.WabbajackBuildServerUri}indexed_files/notify", new ByteArrayContent(ms.ToArray())); - return true; + var response = await client.PostAsync($"{Consts.WabbajackBuildServerUri}indexed_files/notify", new ByteArrayContent(ms.ToArray())); + + if (response.IsSuccessStatusCode) return true; + + Utils.Log("Error sending Inis"); + Utils.Log(await response.Content.ReadAsStringAsync()); + return false; + } catch (Exception ex) { diff --git a/Wabbajack.Lib/NexusApi/Dtos.cs b/Wabbajack.Lib/NexusApi/Dtos.cs index fab90bcd..3ffe2d4c 100644 --- a/Wabbajack.Lib/NexusApi/Dtos.cs +++ b/Wabbajack.Lib/NexusApi/Dtos.cs @@ -15,20 +15,20 @@ namespace Wabbajack.Lib.NexusApi public class NexusFileInfo { - public ulong category_id { get; set; } + public long category_id { get; set; } public string category_name { get; set; } public string changelog_html { get; set; } public string description { get; set; } public string external_virus_scan_url { get; set; } - public ulong file_id { get; set; } + public long file_id { get; set; } public string file_name { get; set; } public bool is_primary { get; set; } public string mod_version { get; set; } public string name { get; set; } - public ulong size { get; set; } - public ulong size_kb { get; set; } + public long size { get; set; } + public long size_kb { get; set; } public DateTime uploaded_time { get; set; } - public ulong uploaded_timestamp { get; set; } + public long uploaded_timestamp { get; set; } public string version { get; set; } } diff --git a/Wabbajack.Test/ContentRightsManagementTests.cs b/Wabbajack.Test/ContentRightsManagementTests.cs index bb1b101c..4c9b1eb6 100644 --- a/Wabbajack.Test/ContentRightsManagementTests.cs +++ b/Wabbajack.Test/ContentRightsManagementTests.cs @@ -54,8 +54,8 @@ namespace Wabbajack.Test { Game = Game.Skyrim, Author = "bill", - ModID = "42", - FileID = "33", + ModID = 42, + FileID = 33, }, Hash = Hash.FromLong(42) } diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index bb8c0636..42dfef90 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -514,14 +514,14 @@ namespace Wabbajack.Test await inst.DownloadMissingArchives(archivesa, true); await inst.DownloadMissingArchives(archivesb, true); - Assert.Equal(folder.EnumerateFiles().Select(f => f.FileName).OrderBy(a => a).ToArray(), - new RelativePath[] + Assert.Equal(new[] { (RelativePath)@"Download.esm", (RelativePath)@"Download.esm.xxHash", - (RelativePath)@"Download_c4047f2251d8eead22df4b4888cc4b833ae7d9a6766ff29128e083d944f9ec4b_.esm", - (RelativePath)@"Download_c4047f2251d8eead22df4b4888cc4b833ae7d9a6766ff29128e083d944f9ec4b_.esm.xxHash" - }.OrderBy(a => a).ToArray()); + (RelativePath)@"Download_ed33cbb256e5328361da8d9227df9cab1bb43a79a87dca2f223b2e2762ccaad1_.esm", + (RelativePath)@"Download_ed33cbb256e5328361da8d9227df9cab1bb43a79a87dca2f223b2e2762ccaad1_.esm.xxHash" + }.OrderBy(a => a).ToArray(), + folder.EnumerateFiles().Select(f => f.FileName).OrderBy(a => a).ToArray()); Consts.TestMode = true; @@ -549,8 +549,8 @@ namespace Wabbajack.Test State = new NexusDownloader.State { Game = Game.SkyrimSpecialEdition, - ModID = "24808", - FileID = "123501" + ModID = 24808, + FileID = 123501 } }; Assert.True(await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, dest));