using Microsoft.Extensions.Options;
using SQLInsight.Core.Servers;
using SQLInsight.Notifications;
using SQLInsight.Repository;
using SQLInsight.Servers;
namespace SQLInsight.Agent.Servers;
///
/// The agent's loop for Windows servers: keeps the SQL Server hosts in step with the monitored instances, reads the
/// configured servers (changes in SQLInsight Web apply within a minute, without a restart), runs each server's due
/// areas with a limit on how many at once, and deletes old samples hourly.
///
public sealed partial class ServerMonitoringService(
IOptions options,
IOptions checkOptions,
IWindowsServerSource source,
InsightRepository repository,
TimeProvider time,
ILoggerFactory loggerFactory,
NotificationDispatcher notifications,
IOptions notificationOptions,
IOptions agentOptions,
Configuration.RepositoryReadiness readiness) : BackgroundService
{
public static readonly TimeSpan RefreshInterval = TimeSpan.FromMinutes(1);
private readonly ILogger _logger = loggerFactory.CreateLogger();
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var settings = options.Value;
if (!settings.Enabled)
{
LogDisabled(_logger);
return;
}
await readiness.Ready.WaitAsync(stoppingToken).ConfigureAwait(false);
var monitors = new Dictionary();
var running = new Dictionary();
using var throttle = new SemaphoreSlim(Math.Max(1, settings.MaxParallelServers));
var nextRefresh = DateTimeOffset.MinValue;
var nextMaintenance = time.GetUtcNow().AddMinutes(5);
using var timer = new PeriodicTimer(agentOptions.Value.Tick, time);
do
{
var now = time.GetUtcNow();
try
{
if (now >= nextRefresh)
{
nextRefresh = now + RefreshInterval;
await RefreshAsync(monitors, settings, now, stoppingToken).ConfigureAwait(false);
}
foreach (var (id, monitor) in monitors)
{
if (running.TryGetValue(id, out var task) && !task.IsCompleted || !monitor.IsDue(now))
{
continue;
}
running[id] = RunThrottledAsync(monitor, now, throttle, stoppingToken);
}
foreach (var done in running.Where(r => r.Value.IsCompleted && !monitors.ContainsKey(r.Key)).Select(r => r.Key).ToList())
{
running.Remove(done);
}
if (now >= nextMaintenance)
{
nextMaintenance = now.AddHours(1);
await repository.RunServerMaintenanceAsync(new ServerRetention
{
SampleDays = settings.SampleDays,
DiskHistoryDays = Math.Max(settings.DiskHistoryDays, settings.Disks.ForecastHistoryDays + 1),
EventDays = settings.EventDays,
CheckResultDays = checkOptions.Value.ResultDays,
ResolvedFindingDays = agentOptions.Value.Retention.ResolvedFindingDays,
}, now, stoppingToken).ConfigureAwait(false);
}
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
// Typically the repository is unreachable. Try again next round.
LogRoundFailed(_logger, ex);
}
}
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
await Task.WhenAll(running.Values).ConfigureAwait(false);
}
/// The SQL Server hosts first, then the servers to look at (not paused).
private async Task RefreshAsync(Dictionary monitors, ServerOptions settings, DateTimeOffset now, CancellationToken cancellationToken)
{
if (settings.IncludeSqlHosts)
{
var hosts = (await repository.GetConfiguredInstancesAsync(cancellationToken).ConfigureAwait(false))
.Select(i => ServerHost.FromDataSource(i.DataSource, Environment.MachineName).Host)
.OfType()
.Distinct(StringComparer.OrdinalIgnoreCase)
.Select(h => (h, ServerHost.ShortName(h)))
.ToList();
await repository.SyncSqlHostServersAsync(hosts, now, cancellationToken).ConfigureAwait(false);
}
var servers = (await repository.GetWindowsServersAsync(cancellationToken).ConfigureAwait(false)).Where(s => !s.Paused).ToList();
foreach (var row in servers)
{
if (monitors.TryGetValue(row.ServerId, out var monitor) && string.Equals(monitor.Server.Host, row.Host, StringComparison.OrdinalIgnoreCase))
{
monitor.Update(row);
continue;
}
monitors[row.ServerId] = new ServerMonitor(row, source, repository, settings, loggerFactory.CreateLogger($"SQLInsight.Server.{row.Name}"), notifications,
notificationOptions.Value.WebUrl);
LogStarted(_logger, row.Name, row.Host);
}
foreach (var gone in monitors.Keys.Where(id => servers.All(s => s.ServerId != id)).ToList())
{
LogStopped(_logger, monitors[gone].Server.Name);
monitors.Remove(gone);
}
}
private async Task RunThrottledAsync(ServerMonitor monitor, DateTimeOffset now, SemaphoreSlim throttle, CancellationToken cancellationToken)
{
await throttle.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
await monitor.RunCycleAsync(now, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
LogCycleFailed(_logger, ex, monitor.Server.Name);
}
finally
{
throttle.Release();
}
}
[LoggerMessage(Level = LogLevel.Information, Message = "Windows servers are not monitored (SQLInsight:Servers:Enabled is false).")]
private static partial void LogDisabled(ILogger logger);
[LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server} ({Host}): monitoring started")]
private static partial void LogStarted(ILogger logger, string server, string host);
[LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: monitoring stopped (paused or removed)")]
private static partial void LogStopped(ILogger logger, string server);
[LoggerMessage(Level = LogLevel.Error, Message = "Windows server {Server}: cycle failed")]
private static partial void LogCycleFailed(ILogger logger, Exception exception, string server);
[LoggerMessage(Level = LogLevel.Error, Message = "Monitoring Windows servers failed this round; retrying.")]
private static partial void LogRoundFailed(ILogger logger, Exception exception);
}