using Microsoft.Extensions.Options; using SQLInsight.Collectors; using SQLInsight.Connectivity; using SQLInsight.Notifications; using SQLInsight.Repository; using SQLInsight.Rules.Engine; namespace SQLInsight.Agent.Monitoring; /// /// The agent's main loop: migrates the repository, then checks every instance on each tick and runs maintenance hourly. /// public sealed partial class MonitoringService( IOptions options, ISqlConnectionFactory connectionFactory, RuleEngine ruleEngine, InsightRepository repository, TimeProvider timeProvider, ILoggerFactory loggerFactory, NotificationDispatcher notifications, IOptions notificationOptions, Hub.HubOutbox hubOutbox, Core.CustomRules.CustomRuleCatalog customRules, Core.Data.IGroupMemberResolver groupResolver) : BackgroundService { private readonly ILogger _logger = loggerFactory.CreateLogger(); protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var settings = options.Value; await MigrateWithRetryAsync(stoppingToken).ConfigureAwait(false); var intervals = new Dictionary(settings.CollectorIntervals, StringComparer.OrdinalIgnoreCase); var monitors = settings.Instances .Select(i => new InstanceMonitor(i, BuiltInCollectors.Create(customRules, new CollectorSettings(groupResolver, settings.Privileges.MaxSysadminAccounts, settings.TopSql.ActivityThresholdPercent)), intervals, connectionFactory, ruleEngine, repository, loggerFactory.CreateLogger($"SQLInsight.Instance.{i.Name}"), notifications, notificationOptions.Value.WebUrl, hubOutbox)) .ToList(); if (monitors.Count == 0) { LogNoInstances(_logger); } using var throttle = new SemaphoreSlim(Math.Max(1, settings.MaxParallelInstances)); var running = new Dictionary(); var nextMaintenance = timeProvider.GetUtcNow(); using var timer = new PeriodicTimer(settings.Tick, timeProvider); do { var now = timeProvider.GetUtcNow(); foreach (var monitor in monitors) { // A slow instance keeps its own cycle; it never delays the others. if (running.TryGetValue(monitor, out var task) && !task.IsCompleted || !monitor.IsDue(now)) { continue; } running[monitor] = RunThrottledAsync(monitor, now, throttle, stoppingToken); } if (now >= nextMaintenance) { nextMaintenance = now.AddHours(1); await RunMaintenanceAsync(settings, now, stoppingToken).ConfigureAwait(false); } } while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); await Task.WhenAll(running.Values).ConfigureAwait(false); } private async Task RunThrottledAsync(InstanceMonitor monitor, DateTimeOffset now, SemaphoreSlim throttle, CancellationToken cancellationToken) { await throttle.WaitAsync(cancellationToken).ConfigureAwait(false); try { await monitor.RunCycleAsync(now, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { // Typically the repository is unreachable. Try again next time. LogCycleFailed(_logger, ex, monitor.Name); } finally { throttle.Release(); } } private async Task MigrateWithRetryAsync(CancellationToken cancellationToken) { for (var attempt = 1; ; attempt++) { try { var applied = await repository.MigrateAsync(cancellationToken).ConfigureAwait(false); LogMigrated(_logger, applied); return; } catch (Exception ex) when (ex is not OperationCanceledException) { LogMigrationFailed(_logger, ex, attempt); await Task.Delay(TimeSpan.FromSeconds(Math.Min(60, 5 * attempt)), timeProvider, cancellationToken).ConfigureAwait(false); } } } private async Task RunMaintenanceAsync(AgentOptions settings, DateTimeOffset now, CancellationToken cancellationToken) { try { await repository.RunMaintenanceAsync(settings.Retention, now, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { LogMaintenanceFailed(_logger, ex); } } [LoggerMessage(Level = LogLevel.Warning, Message = "No instances configured under 'SQLInsight:Instances'.")] private static partial void LogNoInstances(ILogger logger); [LoggerMessage(Level = LogLevel.Information, Message = "Repository ready ({Applied} migrations applied).")] private static partial void LogMigrated(ILogger logger, int applied); [LoggerMessage(Level = LogLevel.Error, Message = "Repository migration failed (attempt {Attempt}); retrying.")] private static partial void LogMigrationFailed(ILogger logger, Exception exception, int attempt); [LoggerMessage(Level = LogLevel.Error, Message = "{Instance}: cycle failed")] private static partial void LogCycleFailed(ILogger logger, Exception exception, string instance); [LoggerMessage(Level = LogLevel.Error, Message = "Repository maintenance failed.")] private static partial void LogMaintenanceFailed(ILogger logger, Exception exception); }