using System; using System.Collections; using System.Collections.Generic; using System.Data; using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; using Dapper; using Microsoft.Extensions.Configuration; using Newtonsoft.Json; using Wabbajack.BuildServer.Model.Models.Results; using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.NexusApi; using Wabbajack.VirtualFileSystem; namespace Wabbajack.BuildServer.Model.Models { public class SqlService { private AppSettings _settings; public SqlService(AppSettings settings) { _settings = settings; } public async Task Open() { var conn = new SqlConnection(_settings.SqlConnection); await conn.OpenAsync(); return conn; } public async Task MergeVirtualFile(VirtualFile vfile) { var files = new List(); var contents = new List(); IngestFile(vfile, files, contents); files = files.DistinctBy(f => f.Hash).ToList(); contents = contents.DistinctBy(c => (c.Parent, c.Path)).ToList(); await using var conn = await Open(); await conn.ExecuteAsync("dbo.MergeIndexedFiles", new {Files = files.ToDataTable(), Contents = contents.ToDataTable()}, commandType: CommandType.StoredProcedure); } private static void IngestFile(VirtualFile root, ICollection files, ICollection contents) { files.Add(new IndexedFile { Hash = (long)root.Hash, Sha256 = root.ExtendedHashes.SHA256.FromHex(), Sha1 = root.ExtendedHashes.SHA1.FromHex(), Md5 = root.ExtendedHashes.MD5.FromHex(), Crc32 = BitConverter.ToInt32(root.ExtendedHashes.CRC.FromHex()), Size = root.Size }); if (root.Children == null) return; foreach (var child in root.Children) { IngestFile(child, files, contents); contents.Add(new ArchiveContent { Parent = (long)root.Hash, Child = (long)child.Hash, Path = (RelativePath)child.Name }); } } public async Task HaveIndexdFile(Hash hash) { await using var conn = await Open(); var row = await conn.QueryAsync(@"SELECT * FROM IndexedFile WHERE Hash = @Hash", new {Hash = (long)hash}); return row.Any(); } class ArchiveContentsResult { public long Parent { get; set; } public long Hash { get; set; } public long Size { get; set; } public string Path { get; set; } } /// /// Get the name, path, hash and size of the file with the provided hash, and all files perhaps /// contained inside this file. Note: files themselves do not have paths, so the top level result /// will have a null path /// /// The xxHash64 of the file to look up /// public async Task AllArchiveContents(long hash) { await using var conn = await Open(); var files = await conn.QueryAsync(@" SELECT 0 as Parent, i.Hash, i.Size, null as Path FROM IndexedFile i WHERE Hash = @Hash UNION ALL SELECT a.Parent, i.Hash, i.Size, a.Path FROM AllArchiveContent a LEFT JOIN IndexedFile i ON i.Hash = a.Child WHERE TopParent = @Hash", new {Hash = hash}); var grouped = files.GroupBy(f => f.Parent).ToDictionary(f => f.Key, f=> (IEnumerable)f); List Build(long parent) { if (grouped.TryGetValue(parent, out var children)) { return children.Select(f => new IndexedVirtualFile { Name = (RelativePath)f.Path, Hash = Hash.FromLong(f.Hash), Size = f.Size, Children = Build(f.Hash) }).ToList(); } return new List(); } return Build(0).FirstOrDefault(); } public async Task IngestAllMetrics(IEnumerable allMetrics) { await using var conn = await Open(); await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", allMetrics); } public async Task IngestMetric(Metric metric) { await using var conn = await Open(); await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", metric); } public async Task> MetricsReport(string action) { await using var conn = await Open(); return (await conn.QueryAsync(@" SELECT d.Date, d.GroupingSubject as Subject, Count(*) as Count FROM (select DISTINCT CONVERT(date, Timestamp) as Date, GroupingSubject, Action, MetricsKey from dbo.Metrics) m RIGHT OUTER JOIN (SELECT CONVERT(date, DATEADD(DAY, number + 1, dbo.MinMetricDate())) as Date, GroupingSubject, Action FROM master..spt_values CROSS JOIN ( SELECT DISTINCT GroupingSubject, Action FROM dbo.Metrics WHERE MetricsKey is not null AND Subject != 'Default' AND TRY_CONVERT(uniqueidentifier, Subject) is null) as keys WHERE type = 'P' AND DATEADD(DAY, number+1, dbo.MinMetricDate()) < dbo.MaxMetricDate()) as d ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action WHERE d.Action = @action group by d.Date, d.GroupingSubject, d.Action ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action})) .ToList(); } #region JobRoutines /// /// Enqueue a Job into the Job queue to be run at a later time /// /// /// public async Task EnqueueJob(Job job) { await using var conn = await Open(); await conn.ExecuteAsync( @"INSERT INTO dbo.Jobs (Created, Priority, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @Payload, @OnSuccess)", new { job.Priority, Payload = job.Payload.ToJson(), OnSuccess = job.OnSuccess?.ToJson() ?? null}); } /// /// Enqueue a Job into the Job queue to be run at a later time /// /// /// public async Task FinishJob(Job job) { await using var conn = await Open(); await conn.ExecuteAsync( @"UPDATE dbo.Jobs SET Ended = GETDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id", new { job.Id, Success = job.Result.ResultType == JobResultType.Success, ResultContent = job.Result.ToJson() }); if (job.OnSuccess != null) await EnqueueJob(job.OnSuccess); } /// /// Get a Job from the Job queue to run. /// /// public async Task GetJob() { await using var conn = await Open(); var result = await conn.QueryAsync( @"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy WHERE ID in (SELECT TOP(1) ID FROM Jobs WHERE Started is NULL ORDER BY Priority DESC, Created); SELECT TOP(1) * FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC", new {RunBy = Guid.NewGuid().ToString()}); return result.FirstOrDefault(); } public async Task> GetRunningJobs() { await using var conn = await Open(); var results = await conn.QueryAsync("SELECT * from dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL "); return results; } public async Task> GetUnfinishedJobs() { await using var conn = await Open(); var results = await conn.QueryAsync("SELECT * from dbo.Jobs WHERE Ended IS NULL "); return results; } #endregion #region TypeMappers static SqlService() { SqlMapper.AddTypeHandler(new PayloadMapper()); SqlMapper.AddTypeHandler(new HashMapper()); } public class PayloadMapper : SqlMapper.TypeHandler { public override void SetValue(IDbDataParameter parameter, AJobPayload value) { parameter.Value = value.ToJson(); } public override AJobPayload Parse(object value) { return ((string)value).FromJsonString(); } } class HashMapper : SqlMapper.TypeHandler { public override void SetValue(IDbDataParameter parameter, Hash value) { parameter.Value = (long)value; } public override Hash Parse(object value) { return Hash.FromLong((long)value); } } #endregion public async Task AddUploadedFile(UploadedFile uf) { await using var conn = await Open(); await conn.ExecuteAsync( "INSERT INTO dbo.UploadedFiles (Id, Name, Size, UploadedBy, Hash, UploadDate, CDNName) VALUES " + "(@Id, @Name, @Size, @UploadedBy, @Hash, @UploadDate, @CDNName)", new { Id = uf.Id.ToString(), uf.Name, uf.Size, UploadedBy = uf.Uploader, Hash = (long)uf.Hash, uf.UploadDate, uf.CDNName }); } public async Task UploadedFileById(Guid fileId) { await using var conn = await Open(); return await conn.QueryFirstAsync("SELECT * FROM dbo.UploadedFiles WHERE Id = @Id", new {Id = fileId.ToString()}); } public async Task> AllUploadedFilesForUser(string user) { await using var conn = await Open(); return await conn.QueryAsync("SELECT * FROM dbo.UploadedFiles WHERE UploadedBy = @uploadedBy", new {UploadedBy = user}); } public async Task> AllUploadedFiles() { await using var conn = await Open(); return await conn.QueryAsync("SELECT * FROM dbo.UploadedFiles ORDER BY UploadDate DESC"); } public async Task DeleteUploadedFile(Guid dupId) { await using var conn = await Open(); await conn.ExecuteAsync("SELECT * FROM dbo.UploadedFiles WHERE Id = @id", new { Id = dupId.ToString() }); } public async Task AddDownloadState(Hash hash, AbstractDownloadState state) { await using var conn = await Open(); await conn.ExecuteAsync("INSERT INTO dbo.DownloadStates (Id, Hash, PrimaryKey, IniState, JsonState) " + "VALUES (@Id, @Hash, @PrimaryKey, @IniState, @JsonState)", new { Id = state.PrimaryKeyString.StringSha256Hex().FromHex(), Hash = hash, PrimaryKey = state.PrimaryKeyString, IniState = string.Join("\n", state.GetMetaIni()), JsonState = state.ToJson() }); } public async Task GetIniForHash(Hash id) { await using var conn = await Open(); var results = await conn.QueryAsync("SELECT IniState FROM dbo.DownloadStates WHERE Hash = @Hash", new { Hash = id }); return results.FirstOrDefault(); } public async Task HaveIndexedArchivePrimaryKey(string key) { await using var conn = await Open(); var results = await conn.QueryFirstOrDefaultAsync( "SELECT PrimaryKey FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey", new {PrimaryKey = key}); return results != null; } public async Task AddNexusFileInfo(Game game, long modId, long fileId, DateTime lastCheckedUtc, NexusFileInfo data) { await using var conn = await Open(); await conn.ExecuteAsync("INSERT INTO dbo.NexusFileInfos (Game, ModId, FileId, LastChecked, Data) VALUES " + "(@Game, @ModId, @FileId, @LastChecked, @Data)", new { Game = game.MetaData().NexusGameId, ModId = modId, FileId = fileId, LastChecked = lastCheckedUtc, Data = JsonConvert.SerializeObject(data) }); } public async Task AddNexusModInfo(Game game, long modId, DateTime lastCheckedUtc, ModInfo data) { await using var conn = await Open(); await conn.ExecuteAsync( @"MERGE dbo.NexusModInfos AS Target USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source ON Target.Game = Source.Game AND Target.ModId = Source.ModId WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);", new { Game = game.MetaData().NexusGameId, ModId = modId, LastChecked = lastCheckedUtc, Data = JsonConvert.SerializeObject(data) }); } public async Task AddNexusModFiles(Game game, long modId, DateTime lastCheckedUtc, NexusApiClient.GetModFilesResponse data) { await using var conn = await Open(); await conn.ExecuteAsync( @"MERGE dbo.NexusModFiles AS Target USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source ON Target.Game = Source.Game AND Target.ModId = Source.ModId WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);", new { Game = game.MetaData().NexusGameId, ModId = modId, LastChecked = lastCheckedUtc, Data = JsonConvert.SerializeObject(data) }); } public async Task GetNexusModInfoString(Game game, long modId) { await using var conn = await Open(); var result = await conn.QueryFirstOrDefaultAsync( "SELECT Data FROM dbo.NexusModInfos WHERE Game = @Game AND @ModId = ModId", new {Game = game.MetaData().NexusGameId, ModId = modId}); return result == null ? null : JsonConvert.DeserializeObject(result); } public async Task GetModFiles(Game game, long modId) { await using var conn = await Open(); var result = await conn.QueryFirstOrDefaultAsync( "SELECT Data FROM dbo.NexusModFiles WHERE Game = @Game AND @ModId = ModId", new {Game = game.MetaData().NexusGameId, ModId = modId}); return result == null ? null : JsonConvert.DeserializeObject(result); } #region ModLists public async Task> GetModListSummaries() { await using var conn = await Open(); var results = await conn.QueryAsync("SELECT Summary from dbo.ModLists"); return results.Select(s => s.FromJsonString()).ToList(); } public async Task GetDetailedModlistStatus(string machineUrl) { await using var conn = await Open(); var result = await conn.QueryFirstOrDefaultAsync("SELECT DetailedStatus from dbo.ModLists WHERE MachineURL = @MachineURL", new { machineUrl }); return result.FromJsonString(); } public async Task> GetDetailedModlistStatuses() { await using var conn = await Open(); var results = await conn.QueryAsync("SELECT DetailedStatus from dbo.ModLists"); return results.Select(s => s.FromJsonString()).ToList(); } #endregion #region Logins public async Task AddLogin(string name) { var key = NewAPIKey(); await using var conn = await Open(); await conn.ExecuteAsync("INSERT INTO dbo.ApiKeys (Owner, ApiKey) VALUES (@Owner, @ApiKey)", new {Owner = name, ApiKey = key}); return key; } public static string NewAPIKey() { var arr = new byte[128]; new Random().NextBytes(arr); return arr.ToHex(); } public async Task LoginByAPIKey(string key) { await using var conn = await Open(); var result = await conn.QueryAsync(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @ApiKey", new {ApiKey = key}); return result.FirstOrDefault(); } public async Task> GetAllUserKeys() { await using var conn = await Open(); var result = await conn.QueryAsync<(string Owner, string Key)>("SELECT Owner, ApiKey FROM dbo.ApiKeys"); return result; } #endregion #region Auto-healing routines public async Task GetNexusStateByHash(Hash startingHash) { await using var conn = await Open(); var result = await conn.QueryFirstOrDefaultAsync(@"SELECT JsonState FROM dbo.DownloadStates WHERE Hash = @hash AND PrimaryKey like 'NexusDownloader+State|%'", new {Hash = (long)startingHash}); return result == null ? null : new Archive { State = result.FromJsonString(), Hash = startingHash }; } public async Task DownloadStateByPrimaryKey(string primaryKey) { await using var conn = await Open(); var result = await conn.QueryFirstOrDefaultAsync<(long Hash, string State)>(@"SELECT Hash, JsonState FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey", new {PrimaryKey = primaryKey}); return result == default ? null : new Archive { State = result.State.FromJsonString(), Hash = Hash.FromLong(result.Hash) }; } #endregion /// /// Returns a hashset the only contains hashes from the input that do not exist in IndexedArchives /// /// /// /// public async Task> FilterByExistingIndexedArchives(HashSet searching) { await using var conn = await Open(); var found = await conn.QueryAsync("SELECT Hash from dbo.IndexedFile WHERE Hash in @Hashes", new {Hashes = searching.Select(h => (long)h)}); return searching.Except(found.Select(h => Hash.FromLong(h)).ToHashSet()).ToHashSet(); } /// /// Returns a hashset the only contains primary keys from the input that do not exist in IndexedArchives /// /// /// /// public async Task> FilterByExistingPrimaryKeys(HashSet pks) { await using var conn = await Open(); var found = await conn.QueryAsync("SELECT Hash from dbo.IndexedFile WHERE PrimaryKey in @PrimaryKeys", new {PrimaryKeys = pks.ToList()}); return pks.Except(found.ToHashSet()).ToHashSet(); } public async Task DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date) { await using var conn = await Open(); var deleted = await conn.ExecuteScalarAsync( @"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date SELECT @@ROWCOUNT AS Deleted", new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date}); return deleted; } public async Task DeleteNexusModFilesUpdatedBeforeDate(Game game, long modId, DateTime date) { await using var conn = await Open(); var deleted = await conn.ExecuteScalarAsync( @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date SELECT @@ROWCOUNT AS Deleted", new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date}); return deleted; } public async Task UpdateModListStatus(ModListStatus dto) { await using var conn = await Open(); await conn.ExecuteAsync(@"MERGE dbo.ModLists AS Target USING (SELECT @MachineUrl MachineUrl, @Metadata Metadata, @Summary Summary, @DetailedStatus DetailedStatus) AS Source ON Target.MachineUrl = Source.MachineUrl WHEN MATCHED THEN UPDATE SET Target.Summary = Source.Summary, Target.Metadata = Source.Metadata, Target.DetailedStatus = Source.DetailedStatus WHEN NOT MATCHED THEN INSERT (MachineUrl, Summary, Metadata, DetailedStatus) VALUES (@MachineUrl, @Summary, @Metadata, @DetailedStatus);", new { MachineUrl = dto.Metadata.Links.MachineURL, Metadata = dto.Metadata.ToJson(), Summary = dto.Summary.ToJson(), DetailedStatus = dto.DetailedStatus.ToJson() }); } } }