sqlinsight-pro/src/SQLInsight.Agent/Configuration/AgentRequestService.cs

using System.Data.Common;
using SQLInsight.Collectors.Discovery;
using SQLInsight.Connectivity;
using SQLInsight.Core.Configuration;
using SQLInsight.Core.Instances;
using SQLInsight.Repository;

namespace SQLInsight.Agent.Configuration;

/// <summary>Set by the monitoring service when the repository is migrated and the configuration file mirrored.</summary>
public sealed class RepositoryReadiness
{
    private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously);

    public Task Ready => _ready.Task;

    public void MarkReady() => _ready.TrySetResult();
}

/// <summary>
/// Runs what SQLInsight Web asks for (insight.AgentRequests): connection tests, database discovery and instance
/// discovery. They run here, as the agent's own identity, because that is what connects to the instances in production;
/// a test from the web server would prove nothing. Also refreshes every instance's database list hourly, so new
/// databases are noticed (and monitored if the instance says so), and publishes the agent's public key.
/// Nothing is written to the monitored instances; results go to the repository only.
/// </summary>
public sealed partial class AgentRequestService(
    InsightRepository repository,
    InstanceConfigurationStore store,
    AgentCredentialKey key,
    ISqlConnectionFactory connectionFactory,
    RepositoryReadiness readiness,
    TimeProvider timeProvider,
    ILogger<AgentRequestService> logger,
    Servers.ServerRequestHandler? serverRequests = null) : BackgroundService
{
    /// <summary>How often the queue is looked at. Short, so the web UI gets an answer within seconds.</summary>
    public static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(2);

    /// <summary>How often every instance's databases are listed again without being asked.</summary>
    public static readonly TimeSpan DiscoveryInterval = TimeSpan.FromHours(1);

    public static readonly TimeSpan RequestTimeout = TimeSpan.FromMinutes(2);

    /// <summary>The least time between two routine discoveries of one instance, also when the last one could not be stored.</summary>
    public static readonly TimeSpan RoutineRetry = TimeSpan.FromMinutes(15);

    private readonly Dictionary<int, DateTimeOffset> _routineAttempts = [];

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        await readiness.Ready.WaitAsync(stoppingToken).ConfigureAwait(false);
        var nextHousekeeping = DateTimeOffset.MinValue;
        using var timer = new PeriodicTimer(PollInterval, timeProvider);
        do
        {
            try
            {
                var now = timeProvider.GetUtcNow();
                if (now >= nextHousekeeping)
                {
                    nextHousekeeping = now.AddMinutes(10);
                    await repository.PublishAgentKeyAsync(key.Describe(now), stoppingToken).ConfigureAwait(false);
                    await repository.CleanUpAgentRequestsAsync(now, StaleAfter, TimeSpan.FromMinutes(30), TimeSpan.FromDays(30), stoppingToken)
                        .ConfigureAwait(false);
                }

                // Everything waiting first, then at most one routine discovery per round.
                while (await repository.ClaimAgentRequestAsync(timeProvider.GetUtcNow(), stoppingToken).ConfigureAwait(false) is { } request)
                {
                    await HandleAsync(request, stoppingToken).ConfigureAwait(false);
                }

                await RefreshOneAsync(timeProvider.GetUtcNow(), stoppingToken).ConfigureAwait(false);
            }
            catch (Exception ex) when (ex is not OperationCanceledException)
            {
                // Typically the repository is unreachable. Try again next round.
                LogRoundFailed(logger, ex);
            }
        }
        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
    }

    /// <summary>Runs one request and stores its result. A failure is the request's result, never the service's.</summary>
    public async Task HandleAsync(AgentRequestRow request, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(request);
        var limit = TimeoutOf(request.Kind);
        using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
        timeout.CancelAfter(limit);
        string? result = null;
        string? error = null;
        try
        {
            result = request.Kind switch
            {
                AgentRequestKind.ConnectionTest => AgentRequestJson.Serialize(await TestAsync(Require(request), timeout.Token).ConfigureAwait(false)),
                AgentRequestKind.DatabaseDiscovery => AgentRequestJson.Serialize(await DiscoverAsync(Require(request), timeout.Token).ConfigureAwait(false)),
                AgentRequestKind.InstanceDiscovery => AgentRequestJson.Serialize(await InstanceFinder.FindAsync(
                    AgentRequestJson.Deserialize<InstanceDiscoveryParameters>(request.Parameters)?.Host, TimeSpan.FromSeconds(3), timeout.Token).ConfigureAwait(false)),
                _ when serverRequests is not null && Servers.ServerRequestHandler.Handles(request.Kind) => await serverRequests.HandleAsync(request, timeout.Token).ConfigureAwait(false),
                _ => throw new InvalidOperationException($"Ukendt anmodning: {request.Kind}."),
            };
        }
        catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
        {
            error = $"Agenten fik ikke svar inden for {limit.TotalMinutes:0} minutter.";
        }
        // Anything else (a lock timeout, a repository error while storing a discovery …) is the request's result too, so it
        // is never left as Running for the web UI to wait on.
        catch (Exception ex) when (ex is not OperationCanceledException)
        {
            error = ex.Message;
        }

        LogHandled(logger, request.RequestId, request.Kind, error ?? "ok");
        await repository.CompleteAgentRequestAsync(request.RequestId, error is null, result, error, timeProvider.GetUtcNow(), CancellationToken.None).ConfigureAwait(false);
    }

    /// <summary>How long a request may take: a synthetic check run now may take its own timeout plus the lookup of its service.</summary>
    public TimeSpan TimeoutOf(string kind) =>
        serverRequests is not null && Servers.ServerRequestHandler.Handles(kind) ? Longest(RequestTimeout, Servers.ServerRequestHandler.RequestTimeout) : RequestTimeout;

    /// <summary>A request still Running after this was lost (the agent stopped): the longest request plus a minute.</summary>
    private static TimeSpan StaleAfter => Longest(RequestTimeout, Servers.ServerRequestHandler.RequestTimeout) + TimeSpan.FromMinutes(1);

    private static TimeSpan Longest(TimeSpan a, TimeSpan b) => a > b ? a : b;

    private static int Require(AgentRequestRow request) =>
        request.ConfiguredInstanceId ?? throw new InvalidOperationException("Anmodningen mangler en instans.");

    /// <summary>Connects as the agent and checks its rights. Not reaching the instance is a result, with a hint, not an error.</summary>
    public async Task<ConnectionTestResult> TestAsync(int instanceId, CancellationToken cancellationToken)
    {
        var (_, instance) = await store.GetAsync(instanceId, cancellationToken).ConfigureAwait(false);
        OpenedConnection opened;
        try
        {
            opened = await connectionFactory.OpenAsync(instance.ToTarget(), cancellationToken).ConfigureAwait(false);
        }
        catch (Exception ex) when (ex is not OperationCanceledException)
        {
            return new ConnectionTestResult { Connected = false, Error = ex.Message, Hint = Hint(ex, instance) };
        }

        await using (opened.ConfigureAwait(false))
        {
            var info = await InstanceProbe.DetectAsync(opened.Connection, cancellationToken).ConfigureAwait(false);
            var notes = new List<string>();
            if (opened.UsedLegacyFallback)
            {
                notes.Add("Forbindelsen bruger den ældre ODBC-driver, fordi instansen er for gammel til SqlClient. Krypteringen kan være svagere.");
            }

            var result = new ConnectionTestResult
            {
                Connected = true,
                Provider = opened.Provider.ToString(),
                UsedLegacyFallback = opened.UsedLegacyFallback,
                ProductVersion = info.Version.ToString(),
                Release = info.Release.DisplayName(),
                Edition = info.Edition,
                ServerName = info.ServerName,
                Notes = notes,
            };
            return await PermissionCheck.CheckAsync(opened.Connection, info, result, cancellationToken).ConfigureAwait(false);
        }
    }

    /// <summary>Lists the instance's databases and stores them. New ones are monitored when the instance monitors new databases.</summary>
    public async Task<DatabaseDiscoveryResult> DiscoverAsync(int instanceId, CancellationToken cancellationToken)
    {
        var (_, instance) = await store.GetAsync(instanceId, cancellationToken).ConfigureAwait(false);
        List<DiscoveredDatabase> databases;
        try
        {
            await using var opened = await connectionFactory.OpenAsync(instance.ToTarget(), cancellationToken).ConfigureAwait(false);
            var info = await InstanceProbe.DetectAsync(opened.Connection, cancellationToken).ConfigureAwait(false);
            databases = await DatabaseDiscovery.DiscoverAsync(opened.Connection, info, cancellationToken).ConfigureAwait(false);
        }
        catch (Exception ex) when (ex is not OperationCanceledException)
        {
            await repository.RecordDiscoveryErrorAsync(instanceId, ex.Message, timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
            throw new DiscoveryFailedException(ex.Message + " " + Hint(ex, instance), ex);
        }

        return await repository.SaveDiscoveredDatabasesAsync(instanceId, databases, timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
    }

    /// <summary>The routine discovery of the active instance that waited longest, if one is due.</summary>
    private async Task RefreshOneAsync(DateTimeOffset now, CancellationToken cancellationToken)
    {
        // Also waits on the agent's own last attempt: if storing the result fails (the repository), LastDiscoveryAt is
        // not set, and without this the same instance would be listed again every round (every 2 seconds).
        var due = (await repository.GetConfiguredInstancesAsync(cancellationToken).ConfigureAwait(false))
            .Where(i => !i.Paused && i.CredentialError is null && (i.LastDiscoveryAt is null || now.UtcDateTime - i.LastDiscoveryAt.Value >= DiscoveryInterval))
            .Where(i => !_routineAttempts.TryGetValue(i.ConfiguredInstanceId, out var attempted) || now - attempted >= RoutineRetry)
            .OrderBy(i => i.LastDiscoveryAt ?? DateTime.MinValue)
            .FirstOrDefault();
        if (due is null)
        {
            return;
        }

        _routineAttempts[due.ConfiguredInstanceId] = now;
        using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
        timeout.CancelAfter(RequestTimeout);
        try
        {
            var result = await DiscoverAsync(due.ConfiguredInstanceId, timeout.Token).ConfigureAwait(false);
            if (result.New.Count > 0)
            {
                LogNewDatabases(logger, due.Name, result.New, result.NewMonitored);
            }
        }
        catch (Exception ex) when (ex is DiscoveryFailedException or CredentialException or KeyNotFoundException
                                   || ex is OperationCanceledException && !cancellationToken.IsCancellationRequested)
        {
            // Recorded on the instance (or the password is reported by the monitor); the next round tries the next instance.
            await repository.RecordDiscoveryErrorAsync(due.ConfiguredInstanceId, ex.Message, now, cancellationToken).ConfigureAwait(false);
        }
    }

    /// <summary>What to do when the agent could not connect (Danish).</summary>
    public static string Hint(Exception exception, MonitoredInstance instance)
    {
        ArgumentNullException.ThrowIfNull(exception);
        ArgumentNullException.ThrowIfNull(instance);
        var number = exception.GetType().GetProperty("Number")?.GetValue(exception) as int?;
        var account = AgentCredentialKey.CurrentAccount();
        return number switch
        {
            18456 when instance.Authentication == AuthenticationMode.Windows =>
                $"Agentens konto ({account}, på en anden server typisk computerkontoen DOMÆNE\\{Environment.MachineName}$) kan ikke logge ind. Kør GRANT-scriptet nedenfor som sysadmin; det opretter login'et.",
            18456 => "SQL-login'et eller adgangskoden blev afvist. Tjek dem, og at instansen tillader SQL Server-godkendelse.",
            4060 => "Login'et findes, men kan ikke åbne master. Kør GRANT-scriptet nedenfor.",
            2 or 53 or -1 or 258 =>
                "Instansen kunne ikke nås fra agentens server. Tjek servernavnet, at instansen kører, og firewallen (TCP 1433, eller UDP 1434 og instansens port for en navngiven instans).",
            -2146893019 => "SQL Server bruger et certifikat, agenten ikke har tillid til. Installér et betroet certifikat, eller slå \"Stol på servercertifikatet\" til.",
            _ when exception.InnerException is System.Security.Authentication.AuthenticationException =>
                "TLS-forbindelsen fejlede. Gamle instanser kan kun TLS 1.0: vælg kryptering \"Valgfri\", eller installér et betroet certifikat.",
            _ => "Se fejlteksten. Forbindelsen blev forsøgt fra agentens server som agentens konto.",
        };
    }

    [LoggerMessage(Level = LogLevel.Information, Message = "Request {RequestId} ({Kind}): {Outcome}")]
    private static partial void LogHandled(ILogger logger, long requestId, string kind, string outcome);

    [LoggerMessage(Level = LogLevel.Information, Message = "{Instance}: new databases {Databases} (monitored at once: {Monitored})")]
    private static partial void LogNewDatabases(ILogger logger, string instance, IReadOnlyList<string> databases, bool monitored);

    [LoggerMessage(Level = LogLevel.Error, Message = "Handling configuration requests failed; retrying.")]
    private static partial void LogRoundFailed(ILogger logger, Exception exception);
}

/// <summary>A discovery that could not reach the instance; the message says why and what to do (Danish).</summary>
public sealed class DiscoveryFailedException : Exception
{
    public DiscoveryFailedException()
    {
    }

    public DiscoveryFailedException(string message)
        : base(message)
    {
    }

    public DiscoveryFailedException(string message, Exception innerException)
        : base(message, innerException)
    {
    }
}