using Microsoft.Extensions.Options; using SQLInsight.Core.Servers; using SQLInsight.Notifications; using SQLInsight.Repository; using SQLInsight.Servers; namespace SQLInsight.Agent.Servers; /// /// The agent's loop for Windows servers: keeps the SQL Server hosts in step with the monitored instances, reads the /// configured servers (changes in SQLInsight Web apply within a minute, without a restart), runs each server's due /// areas with a limit on how many at once, and deletes old samples and check results hourly (also when only the /// synthetic checks are on). /// public sealed partial class ServerMonitoringService( IOptions options, IOptions checkOptions, IWindowsServerSource source, InsightRepository repository, TimeProvider time, ILoggerFactory loggerFactory, NotificationDispatcher notifications, IOptions notificationOptions, IOptions agentOptions, Configuration.RepositoryReadiness readiness) : BackgroundService { public static readonly TimeSpan RefreshInterval = TimeSpan.FromMinutes(1); public static readonly TimeSpan MaintenanceInterval = TimeSpan.FromHours(1); private readonly ILogger _logger = loggerFactory.CreateLogger(); protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var settings = options.Value; if (!settings.Enabled) { LogDisabled(_logger); if (checkOptions.Value.Enabled) { // The synthetic checks keep writing results: their retention still applies. await MaintainOnlyAsync(settings, stoppingToken).ConfigureAwait(false); } return; } await readiness.Ready.WaitAsync(stoppingToken).ConfigureAwait(false); var monitors = new Dictionary(); var running = new Dictionary(); using var throttle = new SemaphoreSlim(Math.Clamp(settings.MaxParallelServers, 1, 32)); var nextRefresh = DateTimeOffset.MinValue; var nextMaintenance = time.GetUtcNow().AddMinutes(5); using var timer = new PeriodicTimer(agentOptions.Value.Tick, time); try { do { var now = time.GetUtcNow(); try { if (now >= nextRefresh) { nextRefresh = now + RefreshInterval; await RefreshAsync(monitors, settings, now, stoppingToken).ConfigureAwait(false); } foreach (var (id, monitor) in monitors) { if (running.TryGetValue(id, out var task) && !task.IsCompleted || !monitor.IsDue(now)) { continue; } running[id] = RunThrottledAsync(monitor, throttle, stoppingToken); } foreach (var done in running.Where(r => r.Value.IsCompleted && !monitors.ContainsKey(r.Key)).Select(r => r.Key).ToList()) { running.Remove(done); } if (now >= nextMaintenance) { nextMaintenance = now + MaintenanceInterval; await MaintainAsync(settings, now, stoppingToken).ConfigureAwait(false); } } catch (Exception ex) when (!stoppingToken.IsCancellationRequested) { // Typically the repository is unreachable. Try again next round. LogRoundFailed(_logger, ex); } } while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { // Stopping: the running cycles end below, before the throttle they release is disposed. } await WaitQuietlyAsync(running.Values).ConfigureAwait(false); } /// Servers off, synthetic checks on: only the hourly maintenance. private async Task MaintainOnlyAsync(ServerOptions settings, CancellationToken stoppingToken) { try { await readiness.Ready.WaitAsync(stoppingToken).ConfigureAwait(false); using var timer = new PeriodicTimer(MaintenanceInterval, time); do { try { await MaintainAsync(settings, time.GetUtcNow(), stoppingToken).ConfigureAwait(false); } catch (Exception ex) when (!stoppingToken.IsCancellationRequested) { LogRoundFailed(_logger, ex); } } while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { // Stopping. } } /// Deletes what is past retention, in small batches; every period is at least a day (see ). private Task MaintainAsync(ServerOptions settings, DateTimeOffset now, CancellationToken cancellationToken) => repository.RunServerMaintenanceAsync(new ServerRetention { SampleDays = Math.Max(1, settings.SampleDays), DiskHistoryDays = Math.Max(Math.Max(1, settings.DiskHistoryDays), settings.Disks.ForecastHistoryDays + 1), EventDays = Math.Max(1, settings.EventDays), CheckResultDays = Math.Max(1, checkOptions.Value.ResultDays), ResolvedFindingDays = Math.Max(1, agentOptions.Value.Retention.ResolvedFindingDays), }, now, cancellationToken); /// /// Waits, when the agent stops, for tasks that end with it: a cancellation (or the error a cancelled database call /// reports instead) is how they end, not news. /// internal static async Task WaitQuietlyAsync(IEnumerable tasks) { foreach (var task in tasks) { try { await task.ConfigureAwait(false); } catch (Exception ex) when (ex is OperationCanceledException or System.Data.Common.DbException) { // Stopping. } } } /// The SQL Server hosts first, then the servers to look at (not paused). private async Task RefreshAsync(Dictionary monitors, ServerOptions settings, DateTimeOffset now, CancellationToken cancellationToken) { if (settings.IncludeSqlHosts) { var hosts = (await repository.GetConfiguredInstancesAsync(cancellationToken).ConfigureAwait(false)) .Select(i => ServerHost.FromDataSource(i.DataSource, Environment.MachineName).Host) .OfType() .Distinct(StringComparer.OrdinalIgnoreCase) .Select(h => (h, ServerHost.ShortName(h))) .ToList(); await repository.SyncSqlHostServersAsync(hosts, now, cancellationToken).ConfigureAwait(false); } var servers = (await repository.GetWindowsServersAsync(cancellationToken).ConfigureAwait(false)).Where(s => !s.Paused).ToList(); foreach (var row in servers) { if (monitors.TryGetValue(row.ServerId, out var monitor) && string.Equals(monitor.Server.Host, row.Host, StringComparison.OrdinalIgnoreCase)) { monitor.Update(row); continue; } monitors[row.ServerId] = new ServerMonitor(row, source, repository, settings, loggerFactory.CreateLogger($"SQLInsight.Server.{row.Name}"), notifications, notificationOptions.Value.WebUrl); LogStarted(_logger, row.Name, row.Host); } foreach (var gone in monitors.Keys.Where(id => servers.All(s => s.ServerId != id)).ToList()) { LogStopped(_logger, monitors[gone].Server.Name); monitors.Remove(gone); } } /// /// One server's cycle when a slot is free, with the time taken then (not when it was queued). The cycle bounds /// itself (), so a hung server gives its slot back. /// private async Task RunThrottledAsync(ServerMonitor monitor, SemaphoreSlim throttle, CancellationToken cancellationToken) { await throttle.WaitAsync(cancellationToken).ConfigureAwait(false); try { await monitor.RunCycleAsync(time.GetUtcNow(), cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (!cancellationToken.IsCancellationRequested) { LogCycleFailed(_logger, ex, monitor.Server.Name); } finally { throttle.Release(); } } [LoggerMessage(Level = LogLevel.Information, Message = "Windows servers are not monitored (SQLInsight:Servers:Enabled is false).")] private static partial void LogDisabled(ILogger logger); [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server} ({Host}): monitoring started")] private static partial void LogStarted(ILogger logger, string server, string host); [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: monitoring stopped (paused or removed)")] private static partial void LogStopped(ILogger logger, string server); [LoggerMessage(Level = LogLevel.Error, Message = "Windows server {Server}: cycle failed")] private static partial void LogCycleFailed(ILogger logger, Exception exception, string server); [LoggerMessage(Level = LogLevel.Error, Message = "Monitoring Windows servers failed this round; retrying.")] private static partial void LogRoundFailed(ILogger logger, Exception exception); }