using Microsoft.Extensions.Options;
using SQLInsight.Collectors;
using SQLInsight.Connectivity;
using SQLInsight.Notifications;
using SQLInsight.Repository;
using SQLInsight.Rules.Engine;
namespace SQLInsight.Agent.Monitoring;
///
/// The agent's main loop: migrates the repository, then checks every instance on each tick and runs maintenance hourly.
///
public sealed partial class MonitoringService(
IOptions options,
ISqlConnectionFactory connectionFactory,
RuleEngine ruleEngine,
InsightRepository repository,
TimeProvider timeProvider,
ILoggerFactory loggerFactory,
NotificationDispatcher notifications,
IOptions notificationOptions,
Hub.HubOutbox hubOutbox,
Core.CustomRules.CustomRuleCatalog customRules,
Core.Data.IGroupMemberResolver groupResolver) : BackgroundService
{
private readonly ILogger _logger = loggerFactory.CreateLogger();
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var settings = options.Value;
await MigrateWithRetryAsync(stoppingToken).ConfigureAwait(false);
var intervals = new Dictionary(settings.CollectorIntervals, StringComparer.OrdinalIgnoreCase);
var monitors = settings.Instances
.Select(i => new InstanceMonitor(i, BuiltInCollectors.Create(customRules, new CollectorSettings(groupResolver, settings.Privileges.MaxSysadminAccounts, settings.TopSql.ActivityThresholdPercent, settings.ErrorLog.LoginFailureBurst,
settings.Activity.TransactionMinutes, settings.Activity.TransactionLogMegabytes, settings.Activity.RequestMinutes,
settings.Tempdb.VersionStoreMegabytes, settings.Tempdb.SessionMegabytes)), intervals, connectionFactory, ruleEngine, repository,
loggerFactory.CreateLogger($"SQLInsight.Instance.{i.Name}"), notifications, notificationOptions.Value.WebUrl, hubOutbox))
.ToList();
if (monitors.Count == 0)
{
LogNoInstances(_logger);
}
using var throttle = new SemaphoreSlim(Math.Max(1, settings.MaxParallelInstances));
var running = new Dictionary();
var nextMaintenance = timeProvider.GetUtcNow();
using var timer = new PeriodicTimer(settings.Tick, timeProvider);
do
{
var now = timeProvider.GetUtcNow();
foreach (var monitor in monitors)
{
// A slow instance keeps its own cycle; it never delays the others.
if (running.TryGetValue(monitor, out var task) && !task.IsCompleted || !monitor.IsDue(now))
{
continue;
}
running[monitor] = RunThrottledAsync(monitor, now, throttle, stoppingToken);
}
if (now >= nextMaintenance)
{
nextMaintenance = now.AddHours(1);
await RunMaintenanceAsync(settings, now, stoppingToken).ConfigureAwait(false);
}
}
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
await Task.WhenAll(running.Values).ConfigureAwait(false);
}
private async Task RunThrottledAsync(InstanceMonitor monitor, DateTimeOffset now, SemaphoreSlim throttle, CancellationToken cancellationToken)
{
await throttle.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
await monitor.RunCycleAsync(now, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
// Typically the repository is unreachable. Try again next time.
LogCycleFailed(_logger, ex, monitor.Name);
}
finally
{
throttle.Release();
}
}
private async Task MigrateWithRetryAsync(CancellationToken cancellationToken)
{
for (var attempt = 1; ; attempt++)
{
try
{
var applied = await repository.MigrateAsync(cancellationToken).ConfigureAwait(false);
LogMigrated(_logger, applied);
return;
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
LogMigrationFailed(_logger, ex, attempt);
await Task.Delay(TimeSpan.FromSeconds(Math.Min(60, 5 * attempt)), timeProvider, cancellationToken).ConfigureAwait(false);
}
}
}
private async Task RunMaintenanceAsync(AgentOptions settings, DateTimeOffset now, CancellationToken cancellationToken)
{
try
{
await repository.RunMaintenanceAsync(settings.Retention, now, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
LogMaintenanceFailed(_logger, ex);
}
}
[LoggerMessage(Level = LogLevel.Warning, Message = "No instances configured under 'SQLInsight:Instances'.")]
private static partial void LogNoInstances(ILogger logger);
[LoggerMessage(Level = LogLevel.Information, Message = "Repository ready ({Applied} migrations applied).")]
private static partial void LogMigrated(ILogger logger, int applied);
[LoggerMessage(Level = LogLevel.Error, Message = "Repository migration failed (attempt {Attempt}); retrying.")]
private static partial void LogMigrationFailed(ILogger logger, Exception exception, int attempt);
[LoggerMessage(Level = LogLevel.Error, Message = "{Instance}: cycle failed")]
private static partial void LogCycleFailed(ILogger logger, Exception exception, string instance);
[LoggerMessage(Level = LogLevel.Error, Message = "Repository maintenance failed.")]
private static partial void LogMaintenanceFailed(ILogger logger, Exception exception);
}