sqlinsight-pro/src/SQLInsight.Agent/Monitoring/MonitoringService.cs

using Microsoft.Extensions.Options;
using SQLInsight.Collectors;
using SQLInsight.Connectivity;
using SQLInsight.Notifications;
using SQLInsight.Repository;
using SQLInsight.Rules.Engine;

namespace SQLInsight.Agent.Monitoring;

/// <summary>
/// The agent's main loop: migrates the repository, then checks every instance on each tick and runs maintenance hourly.
/// </summary>
public sealed partial class MonitoringService(
    IOptions<AgentOptions> options,
    ISqlConnectionFactory connectionFactory,
    RuleEngine ruleEngine,
    InsightRepository repository,
    TimeProvider timeProvider,
    ILoggerFactory loggerFactory,
    NotificationDispatcher notifications,
    IOptions<NotificationOptions> notificationOptions,
    Hub.HubOutbox hubOutbox,
    Core.CustomRules.CustomRuleCatalog customRules,
    Core.Data.IGroupMemberResolver groupResolver) : BackgroundService
{
    private readonly ILogger _logger = loggerFactory.CreateLogger<MonitoringService>();

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        var settings = options.Value;
        await MigrateWithRetryAsync(stoppingToken).ConfigureAwait(false);

        var intervals = new Dictionary<string, TimeSpan>(settings.CollectorIntervals, StringComparer.OrdinalIgnoreCase);
        var monitors = settings.Instances
            .Select(i => new InstanceMonitor(i, BuiltInCollectors.Create(customRules, new CollectorSettings(groupResolver, settings.Privileges.MaxSysadminAccounts, settings.TopSql.ActivityThresholdPercent, settings.ErrorLog.LoginFailureBurst,
                    settings.Activity.TransactionMinutes, settings.Activity.TransactionLogMegabytes, settings.Activity.RequestMinutes,
                    settings.Tempdb.VersionStoreMegabytes, settings.Tempdb.SessionMegabytes)), intervals, connectionFactory, ruleEngine, repository,
                loggerFactory.CreateLogger($"SQLInsight.Instance.{i.Name}"), notifications, notificationOptions.Value.WebUrl, hubOutbox))
            .ToList();
        if (monitors.Count == 0)
        {
            LogNoInstances(_logger);
        }

        using var throttle = new SemaphoreSlim(Math.Max(1, settings.MaxParallelInstances));
        var running = new Dictionary<InstanceMonitor, Task>();
        var nextMaintenance = timeProvider.GetUtcNow();
        using var timer = new PeriodicTimer(settings.Tick, timeProvider);

        do
        {
            var now = timeProvider.GetUtcNow();
            foreach (var monitor in monitors)
            {
                // A slow instance keeps its own cycle; it never delays the others.
                if (running.TryGetValue(monitor, out var task) && !task.IsCompleted || !monitor.IsDue(now))
                {
                    continue;
                }

                running[monitor] = RunThrottledAsync(monitor, now, throttle, stoppingToken);
            }

            if (now >= nextMaintenance)
            {
                nextMaintenance = now.AddHours(1);
                await RunMaintenanceAsync(settings, now, stoppingToken).ConfigureAwait(false);
            }
        }
        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));

        await Task.WhenAll(running.Values).ConfigureAwait(false);
    }

    private async Task RunThrottledAsync(InstanceMonitor monitor, DateTimeOffset now, SemaphoreSlim throttle, CancellationToken cancellationToken)
    {
        await throttle.WaitAsync(cancellationToken).ConfigureAwait(false);
        try
        {
            await monitor.RunCycleAsync(now, cancellationToken).ConfigureAwait(false);
        }
        catch (Exception ex) when (ex is not OperationCanceledException)
        {
            // Typically the repository is unreachable. Try again next time.
            LogCycleFailed(_logger, ex, monitor.Name);
        }
        finally
        {
            throttle.Release();
        }
    }

    private async Task MigrateWithRetryAsync(CancellationToken cancellationToken)
    {
        for (var attempt = 1; ; attempt++)
        {
            try
            {
                var applied = await repository.MigrateAsync(cancellationToken).ConfigureAwait(false);
                LogMigrated(_logger, applied);
                return;
            }
            catch (Exception ex) when (ex is not OperationCanceledException)
            {
                LogMigrationFailed(_logger, ex, attempt);
                await Task.Delay(TimeSpan.FromSeconds(Math.Min(60, 5 * attempt)), timeProvider, cancellationToken).ConfigureAwait(false);
            }
        }
    }

    private async Task RunMaintenanceAsync(AgentOptions settings, DateTimeOffset now, CancellationToken cancellationToken)
    {
        try
        {
            await repository.RunMaintenanceAsync(settings.Retention, now, cancellationToken).ConfigureAwait(false);
        }
        catch (Exception ex) when (ex is not OperationCanceledException)
        {
            LogMaintenanceFailed(_logger, ex);
        }
    }

    [LoggerMessage(Level = LogLevel.Warning, Message = "No instances configured under 'SQLInsight:Instances'.")]
    private static partial void LogNoInstances(ILogger logger);

    [LoggerMessage(Level = LogLevel.Information, Message = "Repository ready ({Applied} migrations applied).")]
    private static partial void LogMigrated(ILogger logger, int applied);

    [LoggerMessage(Level = LogLevel.Error, Message = "Repository migration failed (attempt {Attempt}); retrying.")]
    private static partial void LogMigrationFailed(ILogger logger, Exception exception, int attempt);

    [LoggerMessage(Level = LogLevel.Error, Message = "{Instance}: cycle failed")]
    private static partial void LogCycleFailed(ILogger logger, Exception exception, string instance);

    [LoggerMessage(Level = LogLevel.Error, Message = "Repository maintenance failed.")]
    private static partial void LogMaintenanceFailed(ILogger logger, Exception exception);
}