using Microsoft.Extensions.Options; using SQLInsight.Core.Servers; using SQLInsight.Notifications; using SQLInsight.Repository; using SQLInsight.Servers; namespace SQLInsight.Agent.Servers; /// /// The agent's loop for Windows servers: keeps the SQL Server hosts in step with the monitored instances, reads the /// configured servers (changes in SQLInsight Web apply within a minute, without a restart), runs each server's due /// areas with a limit on how many at once, and deletes old samples hourly. /// public sealed partial class ServerMonitoringService( IOptions options, IOptions checkOptions, IWindowsServerSource source, InsightRepository repository, TimeProvider time, ILoggerFactory loggerFactory, NotificationDispatcher notifications, IOptions notificationOptions, IOptions agentOptions, Configuration.RepositoryReadiness readiness) : BackgroundService { public static readonly TimeSpan RefreshInterval = TimeSpan.FromMinutes(1); private readonly ILogger _logger = loggerFactory.CreateLogger(); protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var settings = options.Value; if (!settings.Enabled) { LogDisabled(_logger); return; } await readiness.Ready.WaitAsync(stoppingToken).ConfigureAwait(false); var monitors = new Dictionary(); var running = new Dictionary(); using var throttle = new SemaphoreSlim(Math.Max(1, settings.MaxParallelServers)); var nextRefresh = DateTimeOffset.MinValue; var nextMaintenance = time.GetUtcNow().AddMinutes(5); using var timer = new PeriodicTimer(agentOptions.Value.Tick, time); do { var now = time.GetUtcNow(); try { if (now >= nextRefresh) { nextRefresh = now + RefreshInterval; await RefreshAsync(monitors, settings, now, stoppingToken).ConfigureAwait(false); } foreach (var (id, monitor) in monitors) { if (running.TryGetValue(id, out var task) && !task.IsCompleted || !monitor.IsDue(now)) { continue; } running[id] = RunThrottledAsync(monitor, now, throttle, stoppingToken); } foreach (var done in running.Where(r => r.Value.IsCompleted && !monitors.ContainsKey(r.Key)).Select(r => r.Key).ToList()) { running.Remove(done); } if (now >= nextMaintenance) { nextMaintenance = now.AddHours(1); await repository.RunServerMaintenanceAsync(new ServerRetention { SampleDays = settings.SampleDays, DiskHistoryDays = Math.Max(settings.DiskHistoryDays, settings.Disks.ForecastHistoryDays + 1), EventDays = settings.EventDays, CheckResultDays = checkOptions.Value.ResultDays, ResolvedFindingDays = agentOptions.Value.Retention.ResolvedFindingDays, }, now, stoppingToken).ConfigureAwait(false); } } catch (Exception ex) when (ex is not OperationCanceledException) { // Typically the repository is unreachable. Try again next round. LogRoundFailed(_logger, ex); } } while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); await Task.WhenAll(running.Values).ConfigureAwait(false); } /// The SQL Server hosts first, then the servers to look at (not paused). private async Task RefreshAsync(Dictionary monitors, ServerOptions settings, DateTimeOffset now, CancellationToken cancellationToken) { if (settings.IncludeSqlHosts) { var hosts = (await repository.GetConfiguredInstancesAsync(cancellationToken).ConfigureAwait(false)) .Select(i => ServerHost.FromDataSource(i.DataSource, Environment.MachineName).Host) .OfType() .Distinct(StringComparer.OrdinalIgnoreCase) .Select(h => (h, ServerHost.ShortName(h))) .ToList(); await repository.SyncSqlHostServersAsync(hosts, now, cancellationToken).ConfigureAwait(false); } var servers = (await repository.GetWindowsServersAsync(cancellationToken).ConfigureAwait(false)).Where(s => !s.Paused).ToList(); foreach (var row in servers) { if (monitors.TryGetValue(row.ServerId, out var monitor) && string.Equals(monitor.Server.Host, row.Host, StringComparison.OrdinalIgnoreCase)) { monitor.Update(row); continue; } monitors[row.ServerId] = new ServerMonitor(row, source, repository, settings, loggerFactory.CreateLogger($"SQLInsight.Server.{row.Name}"), notifications, notificationOptions.Value.WebUrl); LogStarted(_logger, row.Name, row.Host); } foreach (var gone in monitors.Keys.Where(id => servers.All(s => s.ServerId != id)).ToList()) { LogStopped(_logger, monitors[gone].Server.Name); monitors.Remove(gone); } } private async Task RunThrottledAsync(ServerMonitor monitor, DateTimeOffset now, SemaphoreSlim throttle, CancellationToken cancellationToken) { await throttle.WaitAsync(cancellationToken).ConfigureAwait(false); try { await monitor.RunCycleAsync(now, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { LogCycleFailed(_logger, ex, monitor.Server.Name); } finally { throttle.Release(); } } [LoggerMessage(Level = LogLevel.Information, Message = "Windows servers are not monitored (SQLInsight:Servers:Enabled is false).")] private static partial void LogDisabled(ILogger logger); [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server} ({Host}): monitoring started")] private static partial void LogStarted(ILogger logger, string server, string host); [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: monitoring stopped (paused or removed)")] private static partial void LogStopped(ILogger logger, string server); [LoggerMessage(Level = LogLevel.Error, Message = "Windows server {Server}: cycle failed")] private static partial void LogCycleFailed(ILogger logger, Exception exception, string server); [LoggerMessage(Level = LogLevel.Error, Message = "Monitoring Windows servers failed this round; retrying.")] private static partial void LogRoundFailed(ILogger logger, Exception exception); }