using Microsoft.Extensions.Options; using SQLInsight.Collectors; using SQLInsight.Connectivity; using SQLInsight.Notifications; using SQLInsight.Repository; using SQLInsight.Rules.Engine; namespace SQLInsight.Agent.Monitoring; /// /// The agent's main loop: migrates the repository, then checks every instance on each tick and runs maintenance hourly. /// The instances are read from the configuration (file and SQLInsight Web) on every tick, so changes need no restart. /// public sealed partial class MonitoringService( IOptions options, ISqlConnectionFactory connectionFactory, RuleEngine ruleEngine, InsightRepository repository, TimeProvider timeProvider, ILoggerFactory loggerFactory, NotificationDispatcher notifications, IOptions notificationOptions, Hub.HubOutbox hubOutbox, Core.CustomRules.CustomRuleCatalog customRules, Core.Data.IGroupMemberResolver groupResolver, Configuration.InstanceConfigurationStore store, Configuration.RepositoryReadiness readiness) : BackgroundService { private readonly ILogger _logger = loggerFactory.CreateLogger(); private readonly Dictionary _credentialErrors = new(StringComparer.OrdinalIgnoreCase); private bool _configurationLoaded; private bool _warnedNoInstances; protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var settings = options.Value; await MigrateWithRetryAsync(stoppingToken).ConfigureAwait(false); await SyncConfigFileAsync(stoppingToken).ConfigureAwait(false); readiness.MarkReady(); var intervals = new Dictionary(settings.CollectorIntervals, StringComparer.OrdinalIgnoreCase); var monitors = new Dictionary(StringComparer.OrdinalIgnoreCase); using var throttle = new SemaphoreSlim(Math.Max(1, settings.MaxParallelInstances)); var running = new Dictionary(StringComparer.OrdinalIgnoreCase); var nextMaintenance = timeProvider.GetUtcNow(); using var timer = new PeriodicTimer(settings.Tick, timeProvider); do { var now = timeProvider.GetUtcNow(); // Instances added, paused, removed or changed in SQLInsight Web take effect here, without a restart. await RefreshMonitorsAsync(monitors, intervals, settings, now, stoppingToken).ConfigureAwait(false); foreach (var (name, (monitor, _)) in monitors) { // A slow instance keeps its own cycle; it never delays the others. A rebuilt monitor waits for the old cycle. if (running.TryGetValue(name, out var task) && !task.IsCompleted || !monitor.IsDue(now)) { continue; } running[name] = RunThrottledAsync(monitor, now, throttle, stoppingToken); } foreach (var done in running.Where(r => r.Value.IsCompleted && !monitors.ContainsKey(r.Key)).Select(r => r.Key).ToList()) { running.Remove(done); } if (now >= nextMaintenance) { nextMaintenance = now.AddHours(1); await RunMaintenanceAsync(settings, now, stoppingToken).ConfigureAwait(false); } } while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); await Task.WhenAll(running.Values).ConfigureAwait(false); } private InstanceMonitor CreateMonitor(MonitoredInstance instance, AgentOptions settings, IReadOnlyDictionary intervals) => new(instance, BuiltInCollectors.Create(customRules, new CollectorSettings(groupResolver, settings.Privileges.MaxSysadminAccounts, settings.TopSql.ActivityThresholdPercent, settings.ErrorLog.LoginFailureBurst, settings.Activity.TransactionMinutes, settings.Activity.TransactionLogMegabytes, settings.Activity.RequestMinutes, settings.Tempdb.VersionStoreMegabytes, settings.Tempdb.SessionMegabytes, settings.DynamicsAx.ToThresholds())), intervals, connectionFactory, ruleEngine, repository, loggerFactory.CreateLogger($"SQLInsight.Instance.{instance.Name}"), notifications, notificationOptions.Value.WebUrl, hubOutbox); /// /// Brings the monitors in step with the configuration (the configuration file and SQLInsight Web). When the /// repository cannot be read, the monitors keep running as they are; before the first successful read the /// configuration file alone is used, as before 1.3. /// private async Task RefreshMonitorsAsync(Dictionary monitors, IReadOnlyDictionary intervals, AgentOptions settings, DateTimeOffset now, CancellationToken cancellationToken) { IReadOnlyList configured; try { configured = await store.LoadAsync(cancellationToken).ConfigureAwait(false); _configurationLoaded = true; } catch (Exception ex) when (ex is not OperationCanceledException) { LogConfigurationFailed(_logger, ex); if (_configurationLoaded) { return; } configured = store.FromConfigFileOnly(); } var active = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (var instance in configured) { var name = instance.Instance.Name; if (instance.Paused) { continue; } if (instance.CredentialError is { } error) { // Shown on the overview like any other connection problem, once per change. if (!_credentialErrors.TryGetValue(name, out var reported) || reported != error) { _credentialErrors[name] = error; await RecordQuietlyAsync(name, "Adgangskoden kan ikke bruges: " + error, now, cancellationToken).ConfigureAwait(false); } continue; } _credentialErrors.Remove(name); active.Add(name); if (!monitors.TryGetValue(name, out var current) || current.Fingerprint != instance.Fingerprint) { current = (CreateMonitor(instance.Instance, settings, intervals), instance.Fingerprint); monitors[name] = current; LogMonitorStarted(_logger, name); } current.Monitor.Databases = instance.Databases; } foreach (var gone in monitors.Keys.Where(n => !active.Contains(n)).ToList()) { monitors.Remove(gone); LogMonitorStopped(_logger, gone); } if (monitors.Count == 0 && !_warnedNoInstances) { LogNoInstances(_logger); } _warnedNoInstances = monitors.Count == 0; } private async Task RecordQuietlyAsync(string name, string error, DateTimeOffset now, CancellationToken cancellationToken) { try { await repository.RecordInstanceErrorAsync(name, error, now, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { LogConfigurationFailed(_logger, ex); } } private async Task SyncConfigFileAsync(CancellationToken cancellationToken) { try { await store.SyncConfigFileAsync(timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { // The instances in the file are monitored anyway (InstanceConfigurationStore.LoadAsync adds them). LogConfigurationFailed(_logger, ex); } } private async Task RunThrottledAsync(InstanceMonitor monitor, DateTimeOffset now, SemaphoreSlim throttle, CancellationToken cancellationToken) { await throttle.WaitAsync(cancellationToken).ConfigureAwait(false); try { await monitor.RunCycleAsync(now, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { // Typically the repository is unreachable. Try again next time. LogCycleFailed(_logger, ex, monitor.Name); } finally { throttle.Release(); } } private async Task MigrateWithRetryAsync(CancellationToken cancellationToken) { for (var attempt = 1; ; attempt++) { try { var applied = await repository.MigrateAsync(cancellationToken).ConfigureAwait(false); LogMigrated(_logger, applied); return; } catch (Exception ex) when (ex is not OperationCanceledException) { LogMigrationFailed(_logger, ex, attempt); await Task.Delay(TimeSpan.FromSeconds(Math.Min(60, 5 * attempt)), timeProvider, cancellationToken).ConfigureAwait(false); } } } private async Task RunMaintenanceAsync(AgentOptions settings, DateTimeOffset now, CancellationToken cancellationToken) { try { await repository.RunMaintenanceAsync(settings.Retention, now, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { LogMaintenanceFailed(_logger, ex); } } [LoggerMessage(Level = LogLevel.Warning, Message = "No instances configured under 'SQLInsight:Instances'.")] private static partial void LogNoInstances(ILogger logger); [LoggerMessage(Level = LogLevel.Information, Message = "Repository ready ({Applied} migrations applied).")] private static partial void LogMigrated(ILogger logger, int applied); [LoggerMessage(Level = LogLevel.Error, Message = "Repository migration failed (attempt {Attempt}); retrying.")] private static partial void LogMigrationFailed(ILogger logger, Exception exception, int attempt); [LoggerMessage(Level = LogLevel.Error, Message = "{Instance}: cycle failed")] private static partial void LogCycleFailed(ILogger logger, Exception exception, string instance); [LoggerMessage(Level = LogLevel.Information, Message = "{Instance}: monitoring started")] private static partial void LogMonitorStarted(ILogger logger, string instance); [LoggerMessage(Level = LogLevel.Information, Message = "{Instance}: monitoring stopped (paused, removed or no longer configured)")] private static partial void LogMonitorStopped(ILogger logger, string instance); [LoggerMessage(Level = LogLevel.Warning, Message = "The instance configuration could not be read from the repository.")] private static partial void LogConfigurationFailed(ILogger logger, Exception exception); [LoggerMessage(Level = LogLevel.Error, Message = "Repository maintenance failed.")] private static partial void LogMaintenanceFailed(ILogger logger, Exception exception); }