sqlinsight-pro/src/SQLInsight.Agent/Servers/ServerMonitoringService.cs

using Microsoft.Extensions.Options;
using SQLInsight.Core.Servers;
using SQLInsight.Notifications;
using SQLInsight.Repository;
using SQLInsight.Servers;

namespace SQLInsight.Agent.Servers;

/// <summary>
/// The agent's loop for Windows servers: keeps the SQL Server hosts in step with the monitored instances, reads the
/// configured servers (changes in SQLInsight Web apply within a minute, without a restart), runs each server's due
/// areas with a limit on how many at once, and deletes old samples hourly.
/// </summary>
public sealed partial class ServerMonitoringService(
    IOptions<ServerOptions> options,
    IOptions<SyntheticCheckOptions> checkOptions,
    IWindowsServerSource source,
    InsightRepository repository,
    TimeProvider time,
    ILoggerFactory loggerFactory,
    NotificationDispatcher notifications,
    IOptions<NotificationOptions> notificationOptions,
    IOptions<AgentOptions> agentOptions,
    Configuration.RepositoryReadiness readiness) : BackgroundService
{
    public static readonly TimeSpan RefreshInterval = TimeSpan.FromMinutes(1);

    private readonly ILogger _logger = loggerFactory.CreateLogger<ServerMonitoringService>();

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        var settings = options.Value;
        if (!settings.Enabled)
        {
            LogDisabled(_logger);
            return;
        }

        await readiness.Ready.WaitAsync(stoppingToken).ConfigureAwait(false);
        var monitors = new Dictionary<int, ServerMonitor>();
        var running = new Dictionary<int, Task>();
        using var throttle = new SemaphoreSlim(Math.Max(1, settings.MaxParallelServers));
        var nextRefresh = DateTimeOffset.MinValue;
        var nextMaintenance = time.GetUtcNow().AddMinutes(5);
        using var timer = new PeriodicTimer(agentOptions.Value.Tick, time);
        do
        {
            var now = time.GetUtcNow();
            try
            {
                if (now >= nextRefresh)
                {
                    nextRefresh = now + RefreshInterval;
                    await RefreshAsync(monitors, settings, now, stoppingToken).ConfigureAwait(false);
                }

                foreach (var (id, monitor) in monitors)
                {
                    if (running.TryGetValue(id, out var task) && !task.IsCompleted || !monitor.IsDue(now))
                    {
                        continue;
                    }

                    running[id] = RunThrottledAsync(monitor, now, throttle, stoppingToken);
                }

                foreach (var done in running.Where(r => r.Value.IsCompleted && !monitors.ContainsKey(r.Key)).Select(r => r.Key).ToList())
                {
                    running.Remove(done);
                }

                if (now >= nextMaintenance)
                {
                    nextMaintenance = now.AddHours(1);
                    await repository.RunServerMaintenanceAsync(new ServerRetention
                    {
                        SampleDays = settings.SampleDays,
                        DiskHistoryDays = Math.Max(settings.DiskHistoryDays, settings.Disks.ForecastHistoryDays + 1),
                        EventDays = settings.EventDays,
                        CheckResultDays = checkOptions.Value.ResultDays,
                        ResolvedFindingDays = agentOptions.Value.Retention.ResolvedFindingDays,
                    }, now, stoppingToken).ConfigureAwait(false);
                }
            }
            catch (Exception ex) when (ex is not OperationCanceledException)
            {
                // Typically the repository is unreachable. Try again next round.
                LogRoundFailed(_logger, ex);
            }
        }
        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));

        await Task.WhenAll(running.Values).ConfigureAwait(false);
    }

    /// <summary>The SQL Server hosts first, then the servers to look at (not paused).</summary>
    private async Task RefreshAsync(Dictionary<int, ServerMonitor> monitors, ServerOptions settings, DateTimeOffset now, CancellationToken cancellationToken)
    {
        if (settings.IncludeSqlHosts)
        {
            var hosts = (await repository.GetConfiguredInstancesAsync(cancellationToken).ConfigureAwait(false))
                .Select(i => ServerHost.FromDataSource(i.DataSource, Environment.MachineName).Host)
                .OfType<string>()
                .Distinct(StringComparer.OrdinalIgnoreCase)
                .Select(h => (h, ServerHost.ShortName(h)))
                .ToList();
            await repository.SyncSqlHostServersAsync(hosts, now, cancellationToken).ConfigureAwait(false);
        }

        var servers = (await repository.GetWindowsServersAsync(cancellationToken).ConfigureAwait(false)).Where(s => !s.Paused).ToList();
        foreach (var row in servers)
        {
            if (monitors.TryGetValue(row.ServerId, out var monitor) && string.Equals(monitor.Server.Host, row.Host, StringComparison.OrdinalIgnoreCase))
            {
                monitor.Update(row);
                continue;
            }

            monitors[row.ServerId] = new ServerMonitor(row, source, repository, settings, loggerFactory.CreateLogger($"SQLInsight.Server.{row.Name}"), notifications,
                notificationOptions.Value.WebUrl);
            LogStarted(_logger, row.Name, row.Host);
        }

        foreach (var gone in monitors.Keys.Where(id => servers.All(s => s.ServerId != id)).ToList())
        {
            LogStopped(_logger, monitors[gone].Server.Name);
            monitors.Remove(gone);
        }
    }

    private async Task RunThrottledAsync(ServerMonitor monitor, DateTimeOffset now, SemaphoreSlim throttle, CancellationToken cancellationToken)
    {
        await throttle.WaitAsync(cancellationToken).ConfigureAwait(false);
        try
        {
            await monitor.RunCycleAsync(now, cancellationToken).ConfigureAwait(false);
        }
        catch (Exception ex) when (ex is not OperationCanceledException)
        {
            LogCycleFailed(_logger, ex, monitor.Server.Name);
        }
        finally
        {
            throttle.Release();
        }
    }

    [LoggerMessage(Level = LogLevel.Information, Message = "Windows servers are not monitored (SQLInsight:Servers:Enabled is false).")]
    private static partial void LogDisabled(ILogger logger);

    [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server} ({Host}): monitoring started")]
    private static partial void LogStarted(ILogger logger, string server, string host);

    [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: monitoring stopped (paused or removed)")]
    private static partial void LogStopped(ILogger logger, string server);

    [LoggerMessage(Level = LogLevel.Error, Message = "Windows server {Server}: cycle failed")]
    private static partial void LogCycleFailed(ILogger logger, Exception exception, string server);

    [LoggerMessage(Level = LogLevel.Error, Message = "Monitoring Windows servers failed this round; retrying.")]
    private static partial void LogRoundFailed(ILogger logger, Exception exception);
}