using SQLInsight.Collectors;
using SQLInsight.Connectivity;
using SQLInsight.Core.Collection;
using SQLInsight.Core.Instances;
using SQLInsight.Notifications;
using SQLInsight.Repository;
using SQLInsight.Rules.Engine;
namespace SQLInsight.Agent.Monitoring;
///
/// Monitors one instance: runs collectors when they are due, keeps the latest data and each collector's previous
/// result (for deltas), evaluates the rules and stores everything. Not thread-safe; one cycle at a time.
///
public sealed partial class InstanceMonitor(
MonitoredInstance instance,
IReadOnlyList collectors,
IReadOnlyDictionary intervalOverrides,
ISqlConnectionFactory connectionFactory,
RuleEngine ruleEngine,
InsightRepository repository,
ILogger logger,
NotificationDispatcher? notifications = null,
Uri? webUrl = null,
Hub.HubOutbox? hubOutbox = null)
{
private bool? _wasReachable;
private readonly CollectedData _data = new();
private readonly Dictionary _previous = [];
private readonly Dictionary _nextDue = [];
private DateTimeOffset _nextBaseline = DateTimeOffset.MinValue;
private (InstanceInfo Info, int InstanceId)? _last;
/// Metrics a baseline is kept for. Must match what the repository stores (see ).
public static readonly string[] BaselineMetrics = [Metrics.CpuSqlServer, Metrics.BatchRequestsPerSecond, Metrics.PageLifeExpectancy];
public string Name => instance.Name;
/// The databases chosen for monitoring; changed by the monitoring service when the choice changes.
public Core.Configuration.DatabaseSelection Databases { get; set; } = Core.Configuration.DatabaseSelection.All;
public TimeSpan IntervalOf(ICollector collector) =>
intervalOverrides.TryGetValue(collector.Name, out var interval) ? interval : collector.DefaultInterval;
public bool IsDue(DateTimeOffset now) => collectors.Any(c => !_nextDue.TryGetValue(c.Name, out var due) || due <= now);
/// Runs the collectors that are due, then evaluates all rules. Returns the rule results, or null if nothing was due.
public async Task?> RunCycleAsync(DateTimeOffset now, CancellationToken cancellationToken)
{
var due = collectors.Where(c => !_nextDue.TryGetValue(c.Name, out var at) || at <= now).ToList();
if (due.Count == 0)
{
return null;
}
foreach (var collector in due)
{
_nextDue[collector.Name] = now + IntervalOf(collector);
}
// Only frequent samples are due: store them without probing the instance again or evaluating the rules.
if (_last is { } last && _wasReachable == true && due.All(c => c.IsSampling))
{
await SampleAsync(due, last, now, cancellationToken).ConfigureAwait(false);
return null;
}
OpenedConnection opened;
InstanceInfo info;
int instanceId;
try
{
opened = await connectionFactory.OpenAsync(instance.ToTarget(), cancellationToken).ConfigureAwait(false);
info = await InstanceProbe.DetectAsync(opened.Connection, cancellationToken).ConfigureAwait(false);
instanceId = await repository.UpsertInstanceAsync(instance.Name, info, opened.Provider.ToString(), opened.UsedLegacyFallback, now, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
LogConnectFailed(logger, ex, instance.Name);
await repository.RecordInstanceErrorAsync(instance.Name, ex.Message, now, cancellationToken).ConfigureAwait(false);
if (_wasReachable != false)
{
await NotifyAsync([NotificationFactory.Unreachable(instance.Name, ex.Message, webUrl, now)], cancellationToken).ConfigureAwait(false);
}
_wasReachable = false;
return null;
}
await using (opened.ConfigureAwait(false))
{
if (opened.UsedLegacyFallback)
{
LogLegacyFallback(logger, instance.Name);
}
_last = (info, instanceId);
var capabilities = InstanceCapabilities.From(info);
foreach (var collector in due)
{
await CollectAsync(collector, opened, info, capabilities, instanceId, now, cancellationToken).ConfigureAwait(false);
}
await RefreshBaselineAsync(instanceId, now, cancellationToken).ConfigureAwait(false);
var results = ruleEngine.Evaluate(new RuleContext(info, capabilities, now, _data));
var changes = await repository.SaveRuleResultsAsync(instanceId, results, now, cancellationToken).ConfigureAwait(false);
var events = new List();
if (_wasReachable == false)
{
events.Add(NotificationFactory.Recovered(instance.Name, webUrl, now));
}
_wasReachable = true;
events.AddRange(changes.Opened.Select(o => NotificationFactory.Opened(instance.Name, o.FindingId, o.Finding, webUrl, now)));
events.AddRange(changes.Resolved.Select(r => NotificationFactory.Resolved(instance.Name, r.FindingId, r.RuleId, r.Title, (Core.Findings.Severity)r.Severity, webUrl, now)));
await NotifyAsync(events, cancellationToken).ConfigureAwait(false);
if (hubOutbox is not null)
{
await hubOutbox.QueueAsync(instance.Name, instanceId, info, results, changes.Opened.Count + changes.Resolved.Count > 0, now, cancellationToken, Databases).ConfigureAwait(false);
}
var findings = 0;
var notMeasurable = 0;
foreach (var result in results)
{
findings += result.Findings.Count;
notMeasurable += result.Outcome is RuleOutcome.NotMeasurable or RuleOutcome.Error ? 1 : 0;
}
LogCycle(logger, instance.Name, due.Count, findings, notMeasurable);
return results;
}
}
///
/// Runs sampling collectors with what the last full cycle learned about the instance. A failed connection is left
/// to the next full cycle to report.
///
private async Task SampleAsync(List due, (InstanceInfo Info, int InstanceId) last, DateTimeOffset now, CancellationToken cancellationToken)
{
OpenedConnection opened;
try
{
opened = await connectionFactory.OpenAsync(instance.ToTarget(), cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
LogSampleConnectFailed(logger, instance.Name, ex.Message);
return;
}
await using (opened.ConfigureAwait(false))
{
var capabilities = InstanceCapabilities.From(last.Info);
foreach (var collector in due)
{
await CollectAsync(collector, opened, last.Info, capabilities, last.InstanceId, now, cancellationToken).ConfigureAwait(false);
}
}
}
[LoggerMessage(Level = LogLevel.Debug, Message = "{Instance}: sampling skipped, could not connect: {Error}")]
private static partial void LogSampleConnectFailed(ILogger logger, string instance, string error);
private async Task CollectAsync(
ICollector collector, OpenedConnection opened, InstanceInfo info, InstanceCapabilities capabilities, int instanceId, DateTimeOffset now, CancellationToken cancellationToken)
{
var missing = capabilities.Missing(collector.RequiredCapabilities);
if (missing.Count > 0)
{
if (missing.All(InstanceCapabilities.IsInherent))
{
// The version lacks it (Query Store on 2005 …): not applicable, not a gap in the monitoring.
_data.MarkNotApplicable(collector.DataType, $"Ikke relevant for {info.Release.DisplayName()}: kræver {InstanceCapabilities.Describe(missing)}.");
}
else
{
_data.MarkUnavailable(collector.DataType, $"Ikke målbar på {info.Release.DisplayName()}: kræver {string.Join(", ", missing)}.");
}
return;
}
try
{
if (!_previous.TryGetValue(collector.Name, out var previous))
{
previous = await SeedAsync(collector, instanceId, now, cancellationToken).ConfigureAwait(false);
}
var result = await collector.CollectAsync(new CollectorContext(opened.Connection, info, capabilities, now, previous, cancellationToken, Databases)).ConfigureAwait(false);
_previous[collector.Name] = result;
_data.Set(collector.DataType, result);
await repository.SaveCollectedAsync(instanceId, collector.Name, result, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
// Typically a permission the agent lacks on this instance. Rules depending on it say why.
LogCollectorFailed(logger, ex, instance.Name, collector.Name);
_data.MarkUnavailable(collector.DataType, $"Collector '{collector.Name}' fejlede: {ex.Message}");
}
}
private async Task NotifyAsync(List events, CancellationToken cancellationToken)
{
if (notifications is not null && events.Count > 0)
{
await notifications.DispatchAsync(events, cancellationToken).ConfigureAwait(false);
}
}
///
/// The baseline for the current hour of the week, from the repository's hourly history. Refreshed at each new hour.
///
private async Task RefreshBaselineAsync(int instanceId, DateTimeOffset now, CancellationToken cancellationToken)
{
if (now < _nextBaseline)
{
return;
}
var history = await repository.GetHourlyHistoryAsync(instanceId, BaselineMetrics, now, cancellationToken: cancellationToken).ConfigureAwait(false);
_data.Set(Core.Data.BaselineCalculator.Calculate(history, now));
var hour = new DateTimeOffset(now.UtcDateTime.Year, now.UtcDateTime.Month, now.UtcDateTime.Day, now.UtcDateTime.Hour, 0, 0, TimeSpan.Zero);
_nextBaseline = hour.AddHours(1);
}
///
/// State a collector would otherwise lose when the agent restarts. Deadlocks continue after the newest one
/// already stored; the others simply start over (cumulative collectors need one extra sample).
///
private async Task