using System; using System.Collections; using System.Collections.Generic; using System.Data; using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; using Dapper; using Microsoft.Extensions.Configuration; using Newtonsoft.Json; using Wabbajack.BuildServer.Model.Models.Results; using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.NexusApi; using Wabbajack.VirtualFileSystem; namespace Wabbajack.BuildServer.Model.Models { public class SqlService { private AppSettings _settings; public SqlService(AppSettings settings) { _settings = settings; } public async Task Open() { var conn = new SqlConnection(_settings.SqlConnection); await conn.OpenAsync(); return conn; } public async Task MergeVirtualFile(VirtualFile vfile) { var files = new List(); var contents = new List(); IngestFile(vfile, files, contents); files = files.DistinctBy(f => f.Hash).ToList(); contents = contents.DistinctBy(c => (c.Parent, c.Path)).ToList(); await using var conn = await Open(); await conn.ExecuteAsync("dbo.MergeIndexedFiles", new {Files = files.ToDataTable(), Contents = contents.ToDataTable()}, commandType: CommandType.StoredProcedure); } private static void IngestFile(VirtualFile root, ICollection files, ICollection contents) { files.Add(new IndexedFile { Hash = (long)root.Hash, Sha256 = root.ExtendedHashes.SHA256.FromHex(), Sha1 = root.ExtendedHashes.SHA1.FromHex(), Md5 = root.ExtendedHashes.MD5.FromHex(), Crc32 = BitConverter.ToInt32(root.ExtendedHashes.CRC.FromHex()), Size = root.Size }); if (root.Children == null) return; foreach (var child in root.Children) { IngestFile(child, files, contents); contents.Add(new ArchiveContent { Parent = (long)root.Hash, Child = (long)child.Hash, Path = (RelativePath)child.Name }); } } public async Task HaveIndexdFile(Hash hash) { await using var conn = await Open(); var row = await conn.QueryAsync(@"SELECT * FROM IndexedFile WHERE Hash = @Hash", new {Hash = (long)hash}); return row.Any(); } class ArchiveContentsResult { public long Parent { get; set; } public long Hash { get; set; } public long Size { get; set; } public string Path { get; set; } } /// /// Get the name, path, hash and size of the file with the provided hash, and all files perhaps /// contained inside this file. Note: files themselves do not have paths, so the top level result /// will have a null path /// /// The xxHash64 of the file to look up /// public async Task AllArchiveContents(long hash) { await using var conn = await Open(); var files = await conn.QueryAsync(@" SELECT 0 as Parent, i.Hash, i.Size, null as Path FROM IndexedFile i WHERE Hash = @Hash UNION ALL SELECT a.Parent, i.Hash, i.Size, a.Path FROM AllArchiveContent a LEFT JOIN IndexedFile i ON i.Hash = a.Child WHERE TopParent = @Hash", new {Hash = hash}); var grouped = files.GroupBy(f => f.Parent).ToDictionary(f => f.Key, f=> (IEnumerable)f); List Build(long parent) { if (grouped.TryGetValue(parent, out var children)) { return children.Select(f => new IndexedVirtualFile { Name = (RelativePath)f.Path, Hash = Hash.FromLong(f.Hash), Size = f.Size, Children = Build(f.Hash) }).ToList(); } return new List(); } return Build(0).FirstOrDefault(); } public async Task IngestAllMetrics(IEnumerable allMetrics) { await using var conn = await Open(); await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", allMetrics); } public async Task IngestMetric(Metric metric) { await using var conn = await Open(); await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", metric); } public async Task> MetricsReport(string action) { await using var conn = await Open(); return (await conn.QueryAsync(@" SELECT d.Date, d.GroupingSubject as Subject, Count(*) as Count FROM (select DISTINCT CONVERT(date, Timestamp) as Date, GroupingSubject, Action, MetricsKey from dbo.Metrics) m RIGHT OUTER JOIN (SELECT CONVERT(date, DATEADD(DAY, number + 1, dbo.MinMetricDate())) as Date, GroupingSubject, Action FROM master..spt_values CROSS JOIN ( SELECT DISTINCT GroupingSubject, Action FROM dbo.Metrics WHERE MetricsKey is not null AND Subject != 'Default' AND TRY_CONVERT(uniqueidentifier, Subject) is null) as keys WHERE type = 'P' AND DATEADD(DAY, number+1, dbo.MinMetricDate()) < dbo.MaxMetricDate()) as d ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action WHERE d.Action = @action group by d.Date, d.GroupingSubject, d.Action ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action})) .ToList(); } #region JobRoutines /// /// Enqueue a Job into the Job queue to be run at a later time /// /// /// public async Task EnqueueJob(Job job) { await using var conn = await Open(); await conn.ExecuteAsync( @"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)", new { job.Priority, PrimaryKeyString = job.Payload.PrimaryKeyString, Payload = job.Payload.ToJson(), OnSuccess = job.OnSuccess?.ToJson() ?? null}); } /// /// Enqueue a Job into the Job queue to be run at a later time /// /// /// public async Task FinishJob(Job job) { await using var conn = await Open(); await conn.ExecuteAsync( @"UPDATE dbo.Jobs SET Ended = GETDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id", new { job.Id, Success = job.Result.ResultType == JobResultType.Success, ResultContent = job.Result.ToJson() }); if (job.OnSuccess != null) await EnqueueJob(job.OnSuccess); } /// /// Get a Job from the Job queue to run. /// /// public async Task GetJob() { await using var conn = await Open(); var result = await conn.QueryAsync( @"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy WHERE ID in (SELECT TOP(1) ID FROM Jobs WHERE Started is NULL AND PrimaryKeyString NOT IN (SELECT PrimaryKeyString from jobs WHERE Started IS NOT NULL and Ended IS NULL) ORDER BY Priority DESC, Created); SELECT TOP(1) * FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC", new {RunBy = Guid.NewGuid().ToString()}); return result.FirstOrDefault(); } public async Task> GetRunningJobs() { await using var conn = await Open(); var results = await conn.QueryAsync("SELECT * from dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL "); return results; } public async Task> GetUnfinishedJobs() { await using var conn = await Open(); var results = await conn.QueryAsync("SELECT * from dbo.Jobs WHERE Ended IS NULL "); return results; } #endregion #region TypeMappers static SqlService() { SqlMapper.AddTypeHandler(new PayloadMapper()); SqlMapper.AddTypeHandler(new HashMapper()); } public class PayloadMapper : SqlMapper.TypeHandler { public override void SetValue(IDbDataParameter parameter, AJobPayload value) { parameter.Value = value.ToJson(); } public override AJobPayload Parse(object value) { return ((string)value).FromJsonString(); } } class HashMapper : SqlMapper.TypeHandler { public override void SetValue(IDbDataParameter parameter, Hash value) { parameter.Value = (long)value; } public override Hash Parse(object value) { return Hash.FromLong((long)value); } } #endregion public async Task AddUploadedFile(UploadedFile uf) { await using var conn = await Open(); await conn.ExecuteAsync( "INSERT INTO dbo.UploadedFiles (Id, Name, Size, UploadedBy, Hash, UploadDate, CDNName) VALUES " + "(@Id, @Name, @Size, @UploadedBy, @Hash, @UploadDate, @CDNName)", new { Id = uf.Id.ToString(), uf.Name, uf.Size, UploadedBy = uf.Uploader, Hash = (long)uf.Hash, uf.UploadDate, uf.CDNName }); } public async Task UploadedFileById(Guid fileId) { await using var conn = await Open(); return await conn.QueryFirstAsync("SELECT * FROM dbo.UploadedFiles WHERE Id = @Id", new {Id = fileId.ToString()}); } public async Task> AllUploadedFilesForUser(string user) { await using var conn = await Open(); return await conn.QueryAsync("SELECT * FROM dbo.UploadedFiles WHERE UploadedBy = @uploadedBy", new {UploadedBy = user}); } public async Task> AllUploadedFiles() { await using var conn = await Open(); return await conn.QueryAsync("SELECT * FROM dbo.UploadedFiles ORDER BY UploadDate DESC"); } public async Task DeleteUploadedFile(Guid dupId) { await using var conn = await Open(); await conn.ExecuteAsync("SELECT * FROM dbo.UploadedFiles WHERE Id = @id", new { Id = dupId.ToString() }); } public async Task AddDownloadState(Hash hash, AbstractDownloadState state) { await using var conn = await Open(); await conn.ExecuteAsync("INSERT INTO dbo.DownloadStates (Id, Hash, PrimaryKey, IniState, JsonState) " + "VALUES (@Id, @Hash, @PrimaryKey, @IniState, @JsonState)", new { Id = state.PrimaryKeyString.StringSha256Hex().FromHex(), Hash = hash, PrimaryKey = state.PrimaryKeyString, IniState = string.Join("\n", state.GetMetaIni()), JsonState = state.ToJson() }); } public async Task GetIniForHash(Hash id) { await using var conn = await Open(); var results = await conn.QueryAsync("SELECT IniState FROM dbo.DownloadStates WHERE Hash = @Hash", new { Hash = id }); return results.FirstOrDefault(); } public async Task HaveIndexedArchivePrimaryKey(string key) { await using var conn = await Open(); var results = await conn.QueryFirstOrDefaultAsync( "SELECT PrimaryKey FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey", new {PrimaryKey = key}); return results != null; } public async Task AddNexusFileInfo(Game game, long modId, long fileId, DateTime lastCheckedUtc, NexusFileInfo data) { await using var conn = await Open(); await conn.ExecuteAsync("INSERT INTO dbo.NexusFileInfos (Game, ModId, FileId, LastChecked, Data) VALUES " + "(@Game, @ModId, @FileId, @LastChecked, @Data)", new { Game = game.MetaData().NexusGameId, ModId = modId, FileId = fileId, LastChecked = lastCheckedUtc, Data = JsonConvert.SerializeObject(data) }); } public async Task AddNexusModInfo(Game game, long modId, DateTime lastCheckedUtc, ModInfo data) { await using var conn = await Open(); await conn.ExecuteAsync( @"MERGE dbo.NexusModInfos AS Target USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source ON Target.Game = Source.Game AND Target.ModId = Source.ModId WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);", new { Game = game.MetaData().NexusGameId, ModId = modId, LastChecked = lastCheckedUtc, Data = JsonConvert.SerializeObject(data) }); } public async Task AddNexusModFiles(Game game, long modId, DateTime lastCheckedUtc, NexusApiClient.GetModFilesResponse data) { await using var conn = await Open(); await conn.ExecuteAsync( @"MERGE dbo.NexusModFiles AS Target USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source ON Target.Game = Source.Game AND Target.ModId = Source.ModId WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);", new { Game = game.MetaData().NexusGameId, ModId = modId, LastChecked = lastCheckedUtc, Data = JsonConvert.SerializeObject(data) }); } public async Task GetNexusModInfoString(Game game, long modId) { await using var conn = await Open(); var result = await conn.QueryFirstOrDefaultAsync( "SELECT Data FROM dbo.NexusModInfos WHERE Game = @Game AND @ModId = ModId", new {Game = game.MetaData().NexusGameId, ModId = modId}); return result == null ? null : JsonConvert.DeserializeObject(result); } public async Task GetModFiles(Game game, long modId) { await using var conn = await Open(); var result = await conn.QueryFirstOrDefaultAsync( "SELECT Data FROM dbo.NexusModFiles WHERE Game = @Game AND @ModId = ModId", new {Game = game.MetaData().NexusGameId, ModId = modId}); return result == null ? null : JsonConvert.DeserializeObject(result); } #region ModLists public async Task> GetModListSummaries() { await using var conn = await Open(); var results = await conn.QueryAsync("SELECT Summary from dbo.ModLists"); return results.Select(s => s.FromJsonString()).ToList(); } public async Task GetDetailedModlistStatus(string machineUrl) { await using var conn = await Open(); var result = await conn.QueryFirstOrDefaultAsync("SELECT DetailedStatus from dbo.ModLists WHERE MachineURL = @MachineURL", new { machineUrl }); return result.FromJsonString(); } public async Task> GetDetailedModlistStatuses() { await using var conn = await Open(); var results = await conn.QueryAsync("SELECT DetailedStatus from dbo.ModLists"); return results.Select(s => s.FromJsonString()).ToList(); } #endregion #region Logins public async Task AddLogin(string name) { var key = NewAPIKey(); await using var conn = await Open(); await conn.ExecuteAsync("INSERT INTO dbo.ApiKeys (Owner, ApiKey) VALUES (@Owner, @ApiKey)", new {Owner = name, ApiKey = key}); return key; } public static string NewAPIKey() { var arr = new byte[128]; new Random().NextBytes(arr); return arr.ToHex(); } public async Task LoginByAPIKey(string key) { await using var conn = await Open(); var result = await conn.QueryAsync(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @ApiKey", new {ApiKey = key}); return result.FirstOrDefault(); } public async Task> GetAllUserKeys() { await using var conn = await Open(); var result = await conn.QueryAsync<(string Owner, string Key)>("SELECT Owner, ApiKey FROM dbo.ApiKeys"); return result; } #endregion #region Auto-healing routines public async Task GetNexusStateByHash(Hash startingHash) { await using var conn = await Open(); var result = await conn.QueryFirstOrDefaultAsync(@"SELECT JsonState FROM dbo.DownloadStates WHERE Hash = @hash AND PrimaryKey like 'NexusDownloader+State|%'", new {Hash = (long)startingHash}); return result == null ? null : new Archive(result.FromJsonString()) { Hash = startingHash }; } public async Task DownloadStateByPrimaryKey(string primaryKey) { await using var conn = await Open(); var result = await conn.QueryFirstOrDefaultAsync<(long Hash, string State)>(@"SELECT Hash, JsonState FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey", new {PrimaryKey = primaryKey}); return result == default ? null : new Archive(result.State.FromJsonString()) { Hash = Hash.FromLong(result.Hash) }; } #endregion /// /// Returns a hashset the only contains hashes from the input that do not exist in IndexedArchives /// /// /// /// public async Task> FilterByExistingIndexedArchives(HashSet searching) { await using var conn = await Open(); var found = await conn.QueryAsync("SELECT Hash from dbo.IndexedFile WHERE Hash in @Hashes", new {Hashes = searching.Select(h => (long)h)}); return searching.Except(found.Select(h => Hash.FromLong(h)).ToHashSet()).ToHashSet(); } /// /// Returns a hashset the only contains primary keys from the input that do not exist in IndexedArchives /// /// /// /// public async Task> FilterByExistingPrimaryKeys(HashSet pks) { await using var conn = await Open(); var found = await conn.QueryAsync("SELECT Hash from dbo.IndexedFile WHERE PrimaryKey in @PrimaryKeys", new {PrimaryKeys = pks.ToList()}); return pks.Except(found.ToHashSet()).ToHashSet(); } public async Task DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date) { await using var conn = await Open(); var deleted = await conn.ExecuteScalarAsync( @"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date SELECT @@ROWCOUNT AS Deleted", new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date}); return deleted; } public async Task DeleteNexusModFilesUpdatedBeforeDate(Game game, long modId, DateTime date) { await using var conn = await Open(); var deleted = await conn.ExecuteScalarAsync( @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date SELECT @@ROWCOUNT AS Deleted", new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date}); return deleted; } public async Task UpdateModListStatus(ModListStatus dto) { await using var conn = await Open(); await conn.ExecuteAsync(@"MERGE dbo.ModLists AS Target USING (SELECT @MachineUrl MachineUrl, @Metadata Metadata, @Summary Summary, @DetailedStatus DetailedStatus) AS Source ON Target.MachineUrl = Source.MachineUrl WHEN MATCHED THEN UPDATE SET Target.Summary = Source.Summary, Target.Metadata = Source.Metadata, Target.DetailedStatus = Source.DetailedStatus WHEN NOT MATCHED THEN INSERT (MachineUrl, Summary, Metadata, DetailedStatus) VALUES (@MachineUrl, @Summary, @Metadata, @DetailedStatus);", new { MachineUrl = dto.Metadata.Links.MachineURL, Metadata = dto.Metadata.ToJson(), Summary = dto.Summary.ToJson(), DetailedStatus = dto.DetailedStatus.ToJson() }); } } }