wabbajack/Wabbajack.BuildServer/Models/Sql/SqlService.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.Data;
using System.Data.SqlClient;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Wabbajack.BuildServer.Model.Models.Results;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
using Wabbajack.VirtualFileSystem;

namespace Wabbajack.BuildServer.Model.Models
{
    public class SqlService
    {
        private AppSettings _settings;

        public SqlService(AppSettings settings)
        {
            _settings = settings;

        }

        public async Task<SqlConnection> Open()
        {
            var conn = new SqlConnection(_settings.SqlConnection);
            await conn.OpenAsync();
            return conn;
        }

        public async Task MergeVirtualFile(VirtualFile vfile)
        {
            var files = new List<IndexedFile>();
            var contents = new List<ArchiveContent>();
            
            IngestFile(vfile, files, contents);

            files = files.DistinctBy(f => f.Hash).ToList();
            contents = contents.DistinctBy(c => (c.Parent, c.Path)).ToList();

            await using var conn = await Open();
            await conn.ExecuteAsync("dbo.MergeIndexedFiles", new {Files = files.ToDataTable(), Contents = contents.ToDataTable()},
                commandType: CommandType.StoredProcedure);
        }
        
        private static void IngestFile(VirtualFile root, ICollection<IndexedFile> files, ICollection<ArchiveContent> contents)
        {
            files.Add(new IndexedFile
            {
                Hash = (long)root.Hash,
                Sha256 = root.ExtendedHashes.SHA256.FromHex(),
                Sha1 = root.ExtendedHashes.SHA1.FromHex(),
                Md5 = root.ExtendedHashes.MD5.FromHex(),
                Crc32 = BitConverter.ToInt32(root.ExtendedHashes.CRC.FromHex()),
                Size = root.Size
            });

            if (root.Children == null) return;

            foreach (var child in root.Children)
            {
                IngestFile(child, files, contents);

                contents.Add(new ArchiveContent
                {
                    Parent = (long)root.Hash,
                    Child = (long)child.Hash,
                    Path = (RelativePath)child.Name
                });
            }

        }

        public async Task<bool> HaveIndexdFile(Hash hash)
        {
            await using var conn = await Open();
            var row = await conn.QueryAsync(@"SELECT * FROM IndexedFile WHERE Hash = @Hash",
                new {Hash = (long)hash});
            return row.Any();
        }
        
        
                    
        class ArchiveContentsResult
        {
            public long Parent { get; set; }
            public long Hash { get; set; }
            public long Size { get; set; }
            public string Path { get; set; }
        }

        
        /// <summary>
        /// Get the name, path, hash and size of the file with the provided hash, and all files perhaps
        /// contained inside this file. Note: files themselves do not have paths, so the top level result
        /// will have a null path
        /// </summary>
        /// <param name="hash">The xxHash64 of the file to look up</param>
        /// <returns></returns>
        public async Task<IndexedVirtualFile> AllArchiveContents(long hash)
        {
            await using var conn = await Open();
            var files = await conn.QueryAsync<ArchiveContentsResult>(@"
              SELECT 0 as Parent, i.Hash, i.Size, null as Path FROM IndexedFile i WHERE Hash = @Hash
              UNION ALL
              SELECT a.Parent, i.Hash, i.Size, a.Path FROM AllArchiveContent a 
              LEFT JOIN IndexedFile i ON i.Hash = a.Child
              WHERE TopParent = @Hash",
                new {Hash = hash});

            var grouped = files.GroupBy(f => f.Parent).ToDictionary(f => f.Key, f=> (IEnumerable<ArchiveContentsResult>)f);

            List<IndexedVirtualFile> Build(long parent)
            {
                if (grouped.TryGetValue(parent, out var children))
                {
                    return children.Select(f => new IndexedVirtualFile
                    {
                        Name = (RelativePath)f.Path,
                        Hash = Hash.FromLong(f.Hash),
                        Size = f.Size,
                        Children = Build(f.Hash)
                    }).ToList();
                }
                return new List<IndexedVirtualFile>();
            }
            return Build(0).FirstOrDefault();
        }

        public async Task IngestAllMetrics(IEnumerable<Metric> allMetrics)
        {
            await using var conn = await Open();
            await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", allMetrics);
        }
        public async Task IngestMetric(Metric metric)
        {
            await using var conn = await Open();
            await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", metric);
        }

        public async Task<IEnumerable<AggregateMetric>> MetricsReport(string action)
        {
            await using var conn = await Open();
            return (await conn.QueryAsync<AggregateMetric>(@"
                        SELECT d.Date, d.GroupingSubject as Subject, Count(*) as Count FROM 
                        (select DISTINCT CONVERT(date, Timestamp) as Date, GroupingSubject, Action, MetricsKey from dbo.Metrics) m
                        RIGHT OUTER JOIN
                        (SELECT CONVERT(date, DATEADD(DAY, number + 1, dbo.MinMetricDate())) as Date, GroupingSubject, Action
                        FROM master..spt_values
                        CROSS JOIN (
                          SELECT DISTINCT GroupingSubject, Action FROM dbo.Metrics 
                          WHERE MetricsKey is not null 
                          AND Subject != 'Default'
                          AND TRY_CONVERT(uniqueidentifier, Subject) is null) as keys
                        WHERE type = 'P'
                        AND DATEADD(DAY, number+1, dbo.MinMetricDate()) < dbo.MaxMetricDate()) as d
                        ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action
                        WHERE d.Action = @action
                        group by d.Date, d.GroupingSubject, d.Action
                        ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action}))
                .ToList();
        }
        
        #region JobRoutines

        /// <summary>
        /// Enqueue a Job into the Job queue to be run at a later time
        /// </summary>
        /// <param name="job"></param>
        /// <returns></returns>
        public async Task EnqueueJob(Job job)
        {
            await using var conn = await Open();
            await conn.ExecuteAsync(
                @"INSERT INTO dbo.Jobs (Created, Priority, PrimaryKeyString, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @PrimaryKeyString, @Payload, @OnSuccess)",
                new {
                    job.Priority,
                    PrimaryKeyString = job.Payload.PrimaryKeyString,
                    Payload = job.Payload.ToJson(), 
                    OnSuccess = job.OnSuccess?.ToJson() ?? null});
        }
        
        /// <summary>
        /// Enqueue a Job into the Job queue to be run at a later time
        /// </summary>
        /// <param name="job"></param>
        /// <returns></returns>
        public async Task FinishJob(Job job)
        {
            await using var conn = await Open();
            await conn.ExecuteAsync(
                @"UPDATE dbo.Jobs SET Ended = GETDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id",
                new {
                    job.Id,
                    Success = job.Result.ResultType == JobResultType.Success,
                    ResultContent = job.Result.ToJson()
                });
            
            if (job.OnSuccess != null)
                await EnqueueJob(job.OnSuccess);
        }

        
        /// <summary>
        /// Get a Job from the Job queue to run. 
        /// </summary>
        /// <returns></returns>
        public async Task<Job> GetJob()
        {
            await using var conn = await Open();
            var result = await conn.QueryAsync<Job>(
                @"UPDATE jobs SET Started = GETDATE(), RunBy = @RunBy 
                        WHERE ID in (SELECT TOP(1) ID FROM Jobs 
                                       WHERE Started is NULL
                                       AND PrimaryKeyString NOT IN (SELECT PrimaryKeyString from jobs WHERE Started IS NOT NULL and Ended IS NULL)
                                       ORDER BY Priority DESC, Created);
                      SELECT TOP(1) * FROM jobs WHERE RunBy = @RunBy ORDER BY Started DESC",
                new {RunBy = Guid.NewGuid().ToString()});
            return result.FirstOrDefault();
        }
        
        
        public async Task<IEnumerable<Job>> GetRunningJobs()
        {
            await using var conn = await Open();
            var results =
                await conn.QueryAsync<Job>("SELECT * from dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL ");
            return results;
        }


        public async Task<IEnumerable<Job>> GetUnfinishedJobs()
        {
            await using var conn = await Open();
            var results =
                await conn.QueryAsync<Job>("SELECT * from dbo.Jobs WHERE Ended IS NULL ");
            return results;
        }

        
        #endregion


        #region TypeMappers

        static SqlService()
        {
            SqlMapper.AddTypeHandler(new PayloadMapper());
            SqlMapper.AddTypeHandler(new HashMapper());
        }

        public class PayloadMapper : SqlMapper.TypeHandler<AJobPayload>
        {
            public override void SetValue(IDbDataParameter parameter, AJobPayload value)
            {
                parameter.Value = value.ToJson();
            }

            public override AJobPayload Parse(object value)
            {
                return ((string)value).FromJsonString<AJobPayload>();
            }
        }
        
        class HashMapper : SqlMapper.TypeHandler<Hash>
        {
            public override void SetValue(IDbDataParameter parameter, Hash value)
            {
                parameter.Value = (long)value;
            }

            public override Hash Parse(object value)
            {
                return Hash.FromLong((long)value);
            }
        }
        

        #endregion

        public async Task AddUploadedFile(UploadedFile uf)
        {
            await using var conn = await Open();
            await conn.ExecuteAsync(
                "INSERT INTO dbo.UploadedFiles (Id, Name, Size, UploadedBy, Hash, UploadDate, CDNName) VALUES " +
                "(@Id, @Name, @Size, @UploadedBy, @Hash, @UploadDate, @CDNName)",
                new
                {
                    Id = uf.Id.ToString(),
                    uf.Name,
                    uf.Size,
                    UploadedBy = uf.Uploader,
                    Hash = (long)uf.Hash,
                    uf.UploadDate,
                    uf.CDNName
                });
        }
        
        
        public async Task<UploadedFile> UploadedFileById(Guid fileId)
        {
            await using var conn = await Open();
            return await conn.QueryFirstAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles WHERE Id = @Id", 
                new {Id = fileId.ToString()});

        }

        public async Task<IEnumerable<UploadedFile>> AllUploadedFilesForUser(string user)
        {
            await using var conn = await Open();
            return await conn.QueryAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles WHERE UploadedBy = @uploadedBy", 
                new {UploadedBy = user});
        }
        
        
        public async Task<IEnumerable<UploadedFile>> AllUploadedFiles()
        {
            await using var conn = await Open();
            return await conn.QueryAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles ORDER BY UploadDate DESC");
        }
        
        public async Task DeleteUploadedFile(Guid dupId)
        {
            await using var conn = await Open();
            await conn.ExecuteAsync("SELECT * FROM dbo.UploadedFiles WHERE Id = @id", 
                new
                {
                    Id = dupId.ToString()
                });
        }


        public async Task AddDownloadState(Hash hash, AbstractDownloadState state)
        {
            await using var conn = await Open();
            await conn.ExecuteAsync("INSERT INTO dbo.DownloadStates (Id, Hash, PrimaryKey, IniState, JsonState) " +
                                    "VALUES (@Id, @Hash, @PrimaryKey, @IniState, @JsonState)",
                new
                {
                    Id = state.PrimaryKeyString.StringSha256Hex().FromHex(),
                    Hash = hash,
                    PrimaryKey = state.PrimaryKeyString,
                    IniState = string.Join("\n", state.GetMetaIni()),
                    JsonState = state.ToJson()
                });
        }

        public async Task<string> GetIniForHash(Hash id)
        {
            await using var conn = await Open();
            var results = await conn.QueryAsync<string>("SELECT IniState FROM dbo.DownloadStates WHERE Hash = @Hash",
            new {
                Hash = id
            });

            return results.FirstOrDefault();

        }

        public async Task<bool> HaveIndexedArchivePrimaryKey(string key)
        {
            await using var conn = await Open();
            var results = await conn.QueryFirstOrDefaultAsync<string>(
                "SELECT PrimaryKey FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey",
                new {PrimaryKey = key});
            return results != null;
        }

        public async Task AddNexusFileInfo(Game game, long modId, long fileId, DateTime lastCheckedUtc, NexusFileInfo data)
        {
            await using var conn = await Open();

            await conn.ExecuteAsync("INSERT INTO dbo.NexusFileInfos (Game, ModId, FileId, LastChecked, Data) VALUES " +
                                    "(@Game, @ModId, @FileId, @LastChecked, @Data)",
                new
                {
                    Game = game.MetaData().NexusGameId,
                    ModId = modId,
                    FileId = fileId,
                    LastChecked = lastCheckedUtc,
                    Data = JsonConvert.SerializeObject(data)
                });
            
        }

        public async Task AddNexusModInfo(Game game, long modId, DateTime lastCheckedUtc, ModInfo data)
        {
            await using var conn = await Open();

            await conn.ExecuteAsync(
                @"MERGE dbo.NexusModInfos AS Target
                      USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
                      ON Target.Game = Source.Game AND Target.ModId = Source.ModId
                      WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
                      WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
                new
                {
                    Game = game.MetaData().NexusGameId,
                    ModId = modId,
                    LastChecked = lastCheckedUtc,
                    Data = JsonConvert.SerializeObject(data)
                });
            
        }
        
        public async Task AddNexusModFiles(Game game, long modId, DateTime lastCheckedUtc, NexusApiClient.GetModFilesResponse data)
        {
            await using var conn = await Open();

            await conn.ExecuteAsync(                
                @"MERGE dbo.NexusModFiles AS Target
                      USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
                      ON Target.Game = Source.Game AND Target.ModId = Source.ModId
                      WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
                      WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
                new
                {
                    Game = game.MetaData().NexusGameId,
                    ModId = modId,
                    LastChecked = lastCheckedUtc,
                    Data = JsonConvert.SerializeObject(data)
                });
            
        }

        public async Task<ModInfo> GetNexusModInfoString(Game game, long modId)
        {
            await using var conn = await Open();
            var result = await conn.QueryFirstOrDefaultAsync<string>(
                "SELECT Data FROM dbo.NexusModInfos WHERE Game = @Game AND @ModId = ModId",
                new {Game = game.MetaData().NexusGameId, ModId = modId});
            return result == null ? null : JsonConvert.DeserializeObject<ModInfo>(result);
        }

        public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(Game game, long modId)
        {
            await using var conn = await Open();
            var result = await conn.QueryFirstOrDefaultAsync<string>(
                "SELECT Data FROM dbo.NexusModFiles WHERE Game = @Game AND @ModId = ModId",
                new {Game = game.MetaData().NexusGameId, ModId = modId});
            return result == null ? null : JsonConvert.DeserializeObject<NexusApiClient.GetModFilesResponse>(result);
        }

        #region ModLists
        public async Task<IEnumerable<ModListSummary>> GetModListSummaries()
        {
            await using var conn = await Open();
            var results = await conn.QueryAsync<string>("SELECT Summary from dbo.ModLists");
            return results.Select(s => s.FromJsonString<ModListSummary>()).ToList();
        }
        
        public async Task<DetailedStatus> GetDetailedModlistStatus(string machineUrl)
        {
            await using var conn = await Open();
            var result = await conn.QueryFirstOrDefaultAsync<string>("SELECT DetailedStatus from dbo.ModLists WHERE MachineURL = @MachineURL",
                new
                {
                    machineUrl
                });
            return result.FromJsonString<DetailedStatus>();
        }
        public async Task<List<DetailedStatus>> GetDetailedModlistStatuses()
        {
            await using var conn = await Open();
            var results = await conn.QueryAsync<string>("SELECT DetailedStatus from dbo.ModLists");
            return results.Select(s => s.FromJsonString<DetailedStatus>()).ToList();
        }

        

        #endregion

        #region Logins
        public async Task<string> AddLogin(string name)
        {
            var key = NewAPIKey();
            await using var conn = await Open();


            await conn.ExecuteAsync("INSERT INTO dbo.ApiKeys (Owner, ApiKey) VALUES (@Owner, @ApiKey)",
                new {Owner = name, ApiKey = key});
            return key;
        }
        
        public static string NewAPIKey()
        {
            var arr = new byte[128];
            new Random().NextBytes(arr);
            return arr.ToHex();
        }
        
        public async Task<string> LoginByAPIKey(string key)
        {
            await using var conn = await Open();
            var result = await conn.QueryAsync<string>(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @ApiKey",
                new {ApiKey = key});
            return result.FirstOrDefault();
        }

        public async Task<IEnumerable<(string Owner, string Key)>> GetAllUserKeys()
        {
            await using var conn = await Open();
            var result = await conn.QueryAsync<(string Owner, string Key)>("SELECT Owner, ApiKey FROM dbo.ApiKeys");
            return result;
        }

        
        #endregion

        #region Auto-healing routines

        public async Task<Archive> GetNexusStateByHash(Hash startingHash)
        {
            await using var conn = await Open();
            var result = await conn.QueryFirstOrDefaultAsync<string>(@"SELECT JsonState FROM dbo.DownloadStates  
                                                               WHERE Hash = @hash AND PrimaryKey like 'NexusDownloader+State|%'",
                new {Hash = (long)startingHash});
            return result == null ? null : new Archive(result.FromJsonString<AbstractDownloadState>())
            {
                Hash = startingHash
            };
        }
        
        public async Task<Archive> DownloadStateByPrimaryKey(string primaryKey)
        {
            await using var conn = await Open();
            var result = await conn.QueryFirstOrDefaultAsync<(long Hash, string State)>(@"SELECT Hash, JsonState FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey",
                new {PrimaryKey = primaryKey});
            return result == default ? null : new Archive(result.State.FromJsonString<AbstractDownloadState>())
            {
                Hash = Hash.FromLong(result.Hash)
            };
        }
        

        #endregion

        
        /// <summary>
        /// Returns a hashset the only contains hashes from the input that do not exist in IndexedArchives
        /// </summary>
        /// <param name="searching"></param>
        /// <returns></returns>
        /// <exception cref="NotImplementedException"></exception>
        public async Task<HashSet<Hash>> FilterByExistingIndexedArchives(HashSet<Hash> searching)
        {
            await using var conn = await Open();
            var found = await conn.QueryAsync<long>("SELECT Hash from dbo.IndexedFile WHERE Hash in @Hashes",
                new {Hashes = searching.Select(h => (long)h)});
            return searching.Except(found.Select(h => Hash.FromLong(h)).ToHashSet()).ToHashSet();
       }

        
        /// <summary>
        /// Returns a hashset the only contains primary keys from the input that do not exist in IndexedArchives
        /// </summary>
        /// <param name="searching"></param>
        /// <returns></returns>
        /// <exception cref="NotImplementedException"></exception>
        public async Task<HashSet<string>> FilterByExistingPrimaryKeys(HashSet<string> pks)
        {
            await using var conn = await Open();
            var found = await conn.QueryAsync<string>("SELECT Hash from dbo.IndexedFile WHERE PrimaryKey in @PrimaryKeys",
                new {PrimaryKeys = pks.ToList()});
            return pks.Except(found.ToHashSet()).ToHashSet();
        }

        public async Task<long> DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date)
        {
            await using var conn = await Open();
            var deleted = await conn.ExecuteScalarAsync<long>(
                @"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
                      SELECT @@ROWCOUNT AS Deleted",
                new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
            return deleted;
        }
        
        public async Task<long> DeleteNexusModFilesUpdatedBeforeDate(Game game, long modId, DateTime date)
        {
            await using var conn = await Open();
            var deleted = await conn.ExecuteScalarAsync<long>(
                @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
                      SELECT @@ROWCOUNT AS Deleted",
                new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
            return deleted;
        }

        public async Task UpdateModListStatus(ModListStatus dto)
        {
            await using var conn = await Open();
            await conn.ExecuteAsync(@"MERGE dbo.ModLists AS Target
                USING (SELECT @MachineUrl MachineUrl, @Metadata Metadata, @Summary Summary, @DetailedStatus DetailedStatus) AS Source
            ON Target.MachineUrl = Source.MachineUrl
            WHEN MATCHED THEN UPDATE SET Target.Summary = Source.Summary, Target.Metadata = Source.Metadata, Target.DetailedStatus = Source.DetailedStatus
            WHEN NOT MATCHED THEN INSERT (MachineUrl, Summary, Metadata, DetailedStatus) VALUES (@MachineUrl, @Summary, @Metadata, @DetailedStatus);",
                new
                {
                    MachineUrl = dto.Metadata.Links.MachineURL,
                    Metadata = dto.Metadata.ToJson(),
                    Summary = dto.Summary.ToJson(),
                    DetailedStatus = dto.DetailedStatus.ToJson()
                });
        }

    }
}