sqlinsight-pro/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs

using SQLInsight.Collectors;
using SQLInsight.Connectivity;
using SQLInsight.Core.Collection;
using SQLInsight.Core.Instances;
using SQLInsight.Repository;
using SQLInsight.Rules.Engine;

namespace SQLInsight.Agent.Monitoring;

/// <summary>
/// Monitors one instance: runs collectors when they are due, keeps the latest data and each collector's previous
/// result (for deltas), evaluates the rules and stores everything. Not thread-safe; one cycle at a time.
/// </summary>
public sealed partial class InstanceMonitor(
    MonitoredInstance instance,
    IReadOnlyList<ICollector> collectors,
    IReadOnlyDictionary<string, TimeSpan> intervalOverrides,
    ISqlConnectionFactory connectionFactory,
    RuleEngine ruleEngine,
    InsightRepository repository,
    ILogger logger)
{
    private readonly CollectedData _data = new();
    private readonly Dictionary<string, object> _previous = [];
    private readonly Dictionary<string, DateTimeOffset> _nextDue = [];
    private DateTimeOffset _nextBaseline = DateTimeOffset.MinValue;

    /// <summary>Metrics a baseline is kept for. Must match what the repository stores (see <see cref="Metrics"/>).</summary>
    public static readonly string[] BaselineMetrics = [Metrics.CpuSqlServer, Metrics.BatchRequestsPerSecond, Metrics.PageLifeExpectancy];

    public string Name => instance.Name;

    public TimeSpan IntervalOf(ICollector collector) =>
        intervalOverrides.TryGetValue(collector.Name, out var interval) ? interval : collector.DefaultInterval;

    public bool IsDue(DateTimeOffset now) => collectors.Any(c => !_nextDue.TryGetValue(c.Name, out var due) || due <= now);

    /// <summary>Runs the collectors that are due, then evaluates all rules. Returns the rule results, or null if nothing was due.</summary>
    public async Task<IReadOnlyList<RuleResult>?> RunCycleAsync(DateTimeOffset now, CancellationToken cancellationToken)
    {
        var due = collectors.Where(c => !_nextDue.TryGetValue(c.Name, out var at) || at <= now).ToList();
        if (due.Count == 0)
        {
            return null;
        }

        foreach (var collector in due)
        {
            _nextDue[collector.Name] = now + IntervalOf(collector);
        }

        OpenedConnection opened;
        InstanceInfo info;
        int instanceId;
        try
        {
            opened = await connectionFactory.OpenAsync(instance.ToTarget(), cancellationToken).ConfigureAwait(false);
            info = await InstanceProbe.DetectAsync(opened.Connection, cancellationToken).ConfigureAwait(false);
            instanceId = await repository.UpsertInstanceAsync(instance.Name, info, opened.Provider.ToString(), opened.UsedLegacyFallback, now, cancellationToken)
                .ConfigureAwait(false);
        }
        catch (Exception ex) when (ex is not OperationCanceledException)
        {
            LogConnectFailed(logger, ex, instance.Name);
            await repository.RecordInstanceErrorAsync(instance.Name, ex.Message, now, cancellationToken).ConfigureAwait(false);
            return null;
        }

        await using (opened.ConfigureAwait(false))
        {
            if (opened.UsedLegacyFallback)
            {
                LogLegacyFallback(logger, instance.Name);
            }

            var capabilities = InstanceCapabilities.From(info);
            foreach (var collector in due)
            {
                await CollectAsync(collector, opened, info, capabilities, instanceId, now, cancellationToken).ConfigureAwait(false);
            }

            await RefreshBaselineAsync(instanceId, now, cancellationToken).ConfigureAwait(false);
            var results = ruleEngine.Evaluate(new RuleContext(info, capabilities, now, _data));
            await repository.SaveRuleResultsAsync(instanceId, results, now, cancellationToken).ConfigureAwait(false);
            var findings = 0;
            var notMeasurable = 0;
            foreach (var result in results)
            {
                findings += result.Findings.Count;
                notMeasurable += result.Outcome is RuleOutcome.NotMeasurable or RuleOutcome.Error ? 1 : 0;
            }

            LogCycle(logger, instance.Name, due.Count, findings, notMeasurable);
            return results;
        }
    }

    private async Task CollectAsync(
        ICollector collector, OpenedConnection opened, InstanceInfo info, InstanceCapabilities capabilities, int instanceId, DateTimeOffset now, CancellationToken cancellationToken)
    {
        var missing = capabilities.Missing(collector.RequiredCapabilities);
        if (missing.Count > 0)
        {
            _data.MarkUnavailable(collector.DataType, $"Ikke målbar på {info.Release.DisplayName()}: kræver {string.Join(", ", missing)}.");
            return;
        }

        try
        {
            if (!_previous.TryGetValue(collector.Name, out var previous))
            {
                previous = await SeedAsync(collector, instanceId, now, cancellationToken).ConfigureAwait(false);
            }

            var result = await collector.CollectAsync(new CollectorContext(opened.Connection, info, capabilities, now, previous, cancellationToken)).ConfigureAwait(false);
            _previous[collector.Name] = result;
            _data.Set(collector.DataType, result);
            await repository.SaveCollectedAsync(instanceId, collector.Name, result, cancellationToken).ConfigureAwait(false);
        }
        catch (Exception ex) when (ex is not OperationCanceledException)
        {
            // Typically a permission the agent lacks on this instance. Rules depending on it say why.
            LogCollectorFailed(logger, ex, instance.Name, collector.Name);
            _data.MarkUnavailable(collector.DataType, $"Collector '{collector.Name}' fejlede: {ex.Message}");
        }
    }

    /// <summary>
    /// The baseline for the current hour of the week, from the repository's hourly history. Refreshed at each new hour.
    /// </summary>
    private async Task RefreshBaselineAsync(int instanceId, DateTimeOffset now, CancellationToken cancellationToken)
    {
        if (now < _nextBaseline)
        {
            return;
        }

        var history = await repository.GetHourlyHistoryAsync(instanceId, BaselineMetrics, now, cancellationToken: cancellationToken).ConfigureAwait(false);
        _data.Set(Core.Data.BaselineCalculator.Calculate(history, now));
        var hour = new DateTimeOffset(now.UtcDateTime.Year, now.UtcDateTime.Month, now.UtcDateTime.Day, now.UtcDateTime.Hour, 0, 0, TimeSpan.Zero);
        _nextBaseline = hour.AddHours(1);
    }

    /// <summary>
    /// State a collector would otherwise lose when the agent restarts. Deadlocks continue after the newest one
    /// already stored; the others simply start over (cumulative collectors need one extra sample).
    /// </summary>
    private async Task<object?> SeedAsync(ICollector collector, int instanceId, DateTimeOffset now, CancellationToken cancellationToken)
    {
        if (collector is not Collectors.Concurrency.DeadlockCollector)
        {
            return null;
        }

        var latest = await repository.GetLatestDeadlockTimeAsync(instanceId, cancellationToken).ConfigureAwait(false);
        return latest is null ? null : new Core.Data.DeadlockSnapshot { CapturedAt = now, NewDeadlocks = [], LastEventTime = latest };
    }

    [LoggerMessage(Level = LogLevel.Error, Message = "{Instance}: could not connect")]
    private static partial void LogConnectFailed(ILogger logger, Exception exception, string instance);

    [LoggerMessage(Level = LogLevel.Warning, Message = "{Instance}: SqlClient could not connect, using the legacy ODBC connection. Encryption may be weaker.")]
    private static partial void LogLegacyFallback(ILogger logger, string instance);

    [LoggerMessage(Level = LogLevel.Warning, Message = "{Instance}: collector {Collector} failed")]
    private static partial void LogCollectorFailed(ILogger logger, Exception exception, string instance, string collector);

    [LoggerMessage(Level = LogLevel.Information, Message = "{Instance}: {Collectors} collectors, {Findings} findings, {NotMeasurable} rules not measurable")]
    private static partial void LogCycle(ILogger logger, string instance, int collectors, int findings, int notMeasurable);
}