using SQLInsight.Core.Collection;
using SQLInsight.Core.Servers;
using SQLInsight.Notifications;
using SQLInsight.Repository;
using SQLInsight.Rules.Engine;
using SQLInsight.Rules.Servers;
using SQLInsight.Servers;
namespace SQLInsight.Agent.Servers;
///
/// Looks at one Windows server: reads the areas that are due over one CIM session, keeps the latest data for the rules,
/// evaluates them and stores findings. Only reads; a missing right makes that area "ikke målbar" with the exact reason
/// (logged once per change, never every round). Not thread-safe; one cycle at a time.
///
public sealed partial class ServerMonitor(
WindowsServerRow server,
IWindowsServerSource source,
InsightRepository repository,
ServerOptions options,
ILogger logger,
NotificationDispatcher? notifications = null,
Uri? webUrl = null)
{
/// Rules are evaluated at least this often even when only performance was collected.
public static readonly TimeSpan EvaluationInterval = TimeSpan.FromMinutes(5);
private readonly CollectedData _data = new();
private readonly Dictionary _nextDue = new(StringComparer.Ordinal);
private readonly Dictionary _areas = new(StringComparer.Ordinal);
private readonly Dictionary _reported = new(StringComparer.Ordinal);
private ServerPerformanceRaw? _previousRaw;
private IReadOnlyList _samples = [];
private DateTimeOffset? _lastBoot;
private DateTimeOffset? _restartSeenAt;
private ServerEventSnapshot? _events;
private DateTimeOffset _nextEvaluation = DateTimeOffset.MinValue;
private bool _seeded;
private bool? _wasReachable;
public WindowsServerRow Server { get; private set; } = server;
public int ServerId => Server.ServerId;
/// The configuration changed in SQLInsight Web (role, watch lists): used from the next cycle.
public void Update(WindowsServerRow row) => Server = row;
public bool IsDue(DateTimeOffset now) => ServerArea.All.Any(a => !_nextDue.TryGetValue(a, out var due) || due <= now);
public async Task RunCycleAsync(DateTimeOffset now, CancellationToken cancellationToken)
{
var due = ServerArea.All.Where(a => !_nextDue.TryGetValue(a, out var at) || at <= now).ToList();
if (due.Count == 0)
{
return;
}
foreach (var area in due)
{
_nextDue[area] = now + options.IntervalOf(area);
}
if (!_seeded)
{
await SeedAsync(now, cancellationToken).ConfigureAwait(false);
_seeded = true;
}
IWindowsServerSession session;
try
{
session = await source.ConnectAsync(Server.Host, cancellationToken).ConfigureAwait(false);
}
catch (ServerAccessException ex)
{
await NotReachableAsync(ex, due, now, cancellationToken).ConfigureAwait(false);
return;
}
using (session)
{
if (_wasReachable == false)
{
await NotifyAsync([NotificationFactory.Recovered(Server.Name, webUrl, now) with { Url = ServerNotifications.ServerPage(webUrl, ServerId) }], cancellationToken)
.ConfigureAwait(false);
}
_wasReachable = true;
_reported.Remove("connect");
foreach (var area in due)
{
await CollectAsync(session, area, now, cancellationToken).ConfigureAwait(false);
}
if (due.Any(a => a != ServerArea.Performance) || now >= _nextEvaluation)
{
_nextEvaluation = now + EvaluationInterval;
await EvaluateAsync(now, cancellationToken).ConfigureAwait(false);
}
}
}
/// Unreachable: recorded and notified once. Refused by rights at the door: every area is "ikke målbar" with the reason.
private async Task NotReachableAsync(ServerAccessException ex, IReadOnlyList due, DateTimeOffset now, CancellationToken cancellationToken)
{
if (Report("connect", ex.Message))
{
LogNotReachable(logger, Server.Name, ex.Message);
}
await repository.RecordServerErrorAsync(ServerId, ex.Message, now, cancellationToken).ConfigureAwait(false);
if (ex.Problem is ServerAccessProblem.AccessDenied or ServerAccessProblem.NotSupported)
{
foreach (var area in ServerArea.All)
{
Unavailable(area, ex.Message);
}
await repository.RecordServerAreasAsync(ServerId, _areas.Values.ToList(), cancellationToken).ConfigureAwait(false);
if (due.Any(a => a != ServerArea.Performance) || now >= _nextEvaluation)
{
_nextEvaluation = now + EvaluationInterval;
await EvaluateAsync(now, cancellationToken).ConfigureAwait(false);
}
return;
}
if (_wasReachable != false)
{
await NotifyAsync([NotificationFactory.Unreachable(Server.Name, ex.Message, webUrl, now) with { Url = ServerNotifications.ServerPage(webUrl, ServerId) }],
cancellationToken).ConfigureAwait(false);
}
_wasReachable = false;
}
private async Task CollectAsync(IWindowsServerSession session, string area, DateTimeOffset now, CancellationToken cancellationToken)
{
try
{
var detail = area switch
{
ServerArea.Performance => await CollectPerformanceAsync(session, now, cancellationToken).ConfigureAwait(false),
ServerArea.Disks => await CollectDisksAsync(session, now, cancellationToken).ConfigureAwait(false),
ServerArea.Services => await CollectServicesAsync(session, now, cancellationToken).ConfigureAwait(false),
ServerArea.Events => await CollectEventsAsync(session, now, cancellationToken).ConfigureAwait(false),
ServerArea.Tasks => await CollectTasksAsync(session, now, cancellationToken).ConfigureAwait(false),
_ => null,
};
_areas[area] = new ServerAreaStatus(area, true, detail, ServerArea.Right(area));
if (_reported.Remove(area))
{
LogAreaRecovered(logger, Server.Name, area);
}
}
catch (ServerAccessException ex)
{
Unavailable(area, ex.Message);
if (Report(area, ex.Message))
{
LogAreaUnavailable(logger, Server.Name, area, ex.Message);
}
}
}
private void Unavailable(string area, string reason)
{
_areas[area] = new ServerAreaStatus(area, false, reason, ServerArea.Right(area));
switch (area)
{
case ServerArea.Performance:
_data.MarkUnavailable(reason);
break;
case ServerArea.Disks:
_data.MarkUnavailable(reason);
break;
case ServerArea.Services:
_data.MarkUnavailable(reason);
break;
case ServerArea.Events:
_data.MarkUnavailable(reason);
break;
case ServerArea.Tasks:
_data.MarkUnavailable(reason);
break;
}
}
/// Whether a problem is new (or changed) and should be logged: the same problem every minute is not news.
private bool Report(string key, string message)
{
if (_reported.TryGetValue(key, out var previous) && previous == message)
{
return false;
}
_reported[key] = message;
return true;
}
public async Task CollectPerformanceAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(session);
var os = await session.GetOperatingSystemAsync(cancellationToken).ConfigureAwait(false);
var raw = await session.GetPerformanceAsync(now, cancellationToken).ConfigureAwait(false);
DateTimeOffset? previousBoot = null;
if (os.LastBootUpTime is { } boot && _lastBoot is { } before && Math.Abs((boot - before).TotalMinutes) > 2)
{
// Restarted since the last look: the raw counters started over too.
previousBoot = before;
_restartSeenAt = now;
_previousRaw = null;
}
_lastBoot = os.LastBootUpTime ?? _lastBoot;
var sample = ServerPerformanceMath.Sample(_previousRaw, raw, os, options.IntervalOf(ServerArea.Performance));
_previousRaw = raw;
_samples = ServerPerformanceMath.Append(_samples, sample);
var snapshot = new ServerPerformanceSnapshot
{
CapturedAt = now,
Os = os,
Raw = raw,
Samples = _samples,
PreviousBootUpTime = previousBoot,
RestartSeenAt = _restartSeenAt,
Protocol = session.Protocol,
};
_data.Set(snapshot);
await repository.SaveServerPerformanceAsync(ServerId, sample, cancellationToken).ConfigureAwait(false);
await repository.RecordServerContactAsync(ServerId, os, session.Protocol, null, now, cancellationToken).ConfigureAwait(false);
await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Performance, snapshot with { Samples = [] }, now, cancellationToken).ConfigureAwait(false);
return sample.CpuPercent is { } cpu ? $"CPU {cpu:0} %" : null;
}
public async Task CollectDisksAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(session);
var disks = await session.GetDisksAsync(cancellationToken).ConfigureAwait(false);
await repository.SaveServerDisksAsync(ServerId, now, disks, cancellationToken).ConfigureAwait(false);
var historyDays = Math.Max(1, options.Disks.ForecastHistoryDays);
var history = await repository.GetServerDiskHistoryAsync(ServerId, now.AddDays(-historyDays), cancellationToken).ConfigureAwait(false);
var settings = new DiskForecastSettings { HistoryDays = historyDays };
var forecasts = disks.Select(d => DiskForecaster.Forecast(d.Volume, history.TryGetValue(d.Volume, out var points) ? points : [], now, settings)).ToList();
var snapshot = new ServerDiskSnapshot { CapturedAt = now, Disks = disks, Forecasts = forecasts };
_data.Set(snapshot);
await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Disks, snapshot, now, cancellationToken).ConfigureAwait(false);
return $"{disks.Count} drev";
}
public async Task CollectServicesAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(session);
var services = await session.GetServicesAsync(cancellationToken).ConfigureAwait(false);
var snapshot = new ServerServiceSnapshot { CapturedAt = now, Services = services };
_data.Set(snapshot);
await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Services, snapshot, now, cancellationToken).ConfigureAwait(false);
return $"{services.Count} tjenester";
}
/// The first time, the last day is read, so recent problems show at once; then from where the last read stopped.
public async Task CollectEventsAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(session);
var since = _events?.ReadUntil ?? now - ServerEventAnalysis.RecentWindow;
var read = await session.GetEventsAsync(since, Math.Max(10, options.EventLog.MaxEventsPerCollection), cancellationToken).ConfigureAwait(false);
var (groups, ignored) = ServerEventAnalysis.Group(read.Entries, ServerEventAnalysis.DefaultIgnore.Concat(options.EventLog.Ignore));
// Strictly after the newest event read; without events, keep ten minutes' margin for events written late.
var readUntil = read.Entries.Count > 0 ? read.Entries.Max(e => e.At) : Max(since, now.AddMinutes(-10));
var snapshot = new ServerEventSnapshot
{
CapturedAt = now,
ReadUntil = readUntil,
Method = read.Method,
Groups = groups,
Recent = ServerEventAnalysis.Recent(_events?.Recent ?? [], groups, now),
Ignored = ignored,
Coverage = read.Truncated
? $"Der var flere end {options.EventLog.MaxEventsPerCollection} hændelser i en log siden sidst; resten læses ved næste indsamling."
: read.Note,
};
_events = snapshot;
_data.Set(snapshot);
await repository.SaveServerEventsAsync(ServerId, snapshot, cancellationToken).ConfigureAwait(false);
await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Events, snapshot with { Groups = [] }, now, cancellationToken).ConfigureAwait(false);
return $"{read.Entries.Count} hændelser ({read.Method})";
static DateTimeOffset Max(DateTimeOffset a, DateTimeOffset b) => a > b ? a : b;
}
public async Task CollectTasksAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(session);
var tasks = options.Tasks;
var read = await session.GetTasksAsync(
tasks.Watches,
Math.Max(1, tasks.MaxWatched), cancellationToken).ConfigureAwait(false);
var snapshot = new ServerTaskSnapshot { CapturedAt = now, Tasks = read.Watched, Visible = read.Visible, Coverage = read.Note };
_data.Set(snapshot);
await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Tasks, snapshot, now, cancellationToken).ConfigureAwait(false);
return $"{read.Watched.Count} af {read.Visible} opgaver overvåges";
}
private async Task EvaluateAsync(DateTimeOffset now, CancellationToken cancellationToken)
{
await repository.RecordServerAreasAsync(ServerId, ServerArea.All.Where(_areas.ContainsKey).Select(a => _areas[a]).ToList(), cancellationToken).ConfigureAwait(false);
var context = new ServerRuleContext(Server.ToIdentity(), now, _data, options) { PerformanceInterval = options.IntervalOf(ServerArea.Performance) };
var results = ScopedRuleEngine.Evaluate(WindowsServerRules.Create(), context);
var changes = await repository.SaveScopedRuleResultsAsync(FindingScope.Server, ServerId, results, now, cancellationToken).ConfigureAwait(false);
await NotifyAsync(ServerNotifications.For(Server.Name, changes, webUrl, now).ToList(), cancellationToken).ConfigureAwait(false);
if (logger.IsEnabled(LogLevel.Debug))
{
var findings = results.Sum(r => r.Findings.Count);
var notMeasurable = results.Count(r => r.Outcome is RuleOutcome.NotMeasurable or RuleOutcome.Error);
LogCycle(logger, Server.Name, findings, notMeasurable);
}
}
/// What a restart of the agent would lose: the last hour's samples, the boot time and where the event log was read to.
private async Task SeedAsync(DateTimeOffset now, CancellationToken cancellationToken)
{
var stored = await repository.GetServerPerformanceAsync(ServerId, now - ServerPerformanceMath.Window, cancellationToken).ConfigureAwait(false);
_samples = stored.Select(s => new ServerPerformanceSample(new DateTimeOffset(DateTime.SpecifyKind(s.SampledAt, DateTimeKind.Utc)), s.CpuPercent, s.AvailableMb,
s.AvailableMb is { } mb && s.AvailablePct is > 0 ? mb * 100 / s.AvailablePct.Value : null, s.PagesPerSec, s.CommitPct)).ToList();
var performance = await repository.GetServerSnapshotAsync(ServerId, ServerArea.Performance, cancellationToken).ConfigureAwait(false);
_lastBoot = performance?.Os.LastBootUpTime;
_restartSeenAt = performance?.RestartSeenAt;
_events = await repository.GetServerSnapshotAsync(ServerId, ServerArea.Events, cancellationToken).ConfigureAwait(false);
}
private async Task NotifyAsync(List events, CancellationToken cancellationToken)
{
if (notifications is not null && events.Count > 0)
{
await notifications.DispatchAsync(events, cancellationToken).ConfigureAwait(false);
}
}
[LoggerMessage(Level = LogLevel.Warning, Message = "Windows server {Server} cannot be read: {Reason}")]
private static partial void LogNotReachable(ILogger logger, string server, string reason);
[LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: {Area} is not measurable: {Reason}")]
private static partial void LogAreaUnavailable(ILogger logger, string server, string area, string reason);
[LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: {Area} can be read again")]
private static partial void LogAreaRecovered(ILogger logger, string server, string area);
[LoggerMessage(Level = LogLevel.Debug, Message = "Windows server {Server}: {Findings} findings, {NotMeasurable} rules not measurable")]
private static partial void LogCycle(ILogger logger, string server, int findings, int notMeasurable);
}