using SQLInsight.Core.Collection; using SQLInsight.Core.Servers; using SQLInsight.Notifications; using SQLInsight.Repository; using SQLInsight.Rules.Engine; using SQLInsight.Rules.Servers; using SQLInsight.Servers; namespace SQLInsight.Agent.Servers; /// /// Looks at one Windows server: reads the areas that are due over one CIM session, keeps the latest data for the rules, /// evaluates them and stores findings. Only reads; a missing right makes that area "ikke målbar" with the exact reason /// (logged once per change, never every round). Not thread-safe; one cycle at a time. /// public sealed partial class ServerMonitor( WindowsServerRow server, IWindowsServerSource source, InsightRepository repository, ServerOptions options, ILogger logger, NotificationDispatcher? notifications = null, Uri? webUrl = null) { /// Rules are evaluated at least this often even when only performance was collected. public static readonly TimeSpan EvaluationInterval = TimeSpan.FromMinutes(5); private readonly CollectedData _data = new(); private readonly Dictionary _nextDue = new(StringComparer.Ordinal); private readonly Dictionary _areas = new(StringComparer.Ordinal); private readonly Dictionary _reported = new(StringComparer.Ordinal); private ServerPerformanceRaw? _previousRaw; private IReadOnlyList _samples = []; private DateTimeOffset? _lastBoot; private DateTimeOffset? _restartSeenAt; private ServerEventSnapshot? _events; private DateTimeOffset _nextEvaluation = DateTimeOffset.MinValue; private bool _seeded; private bool? _wasReachable; public WindowsServerRow Server { get; private set; } = server; public int ServerId => Server.ServerId; /// The configuration changed in SQLInsight Web (role, watch lists): used from the next cycle. public void Update(WindowsServerRow row) => Server = row; public bool IsDue(DateTimeOffset now) => ServerArea.All.Any(a => !_nextDue.TryGetValue(a, out var due) || due <= now); public async Task RunCycleAsync(DateTimeOffset now, CancellationToken cancellationToken) { var due = ServerArea.All.Where(a => !_nextDue.TryGetValue(a, out var at) || at <= now).ToList(); if (due.Count == 0) { return; } foreach (var area in due) { _nextDue[area] = now + options.IntervalOf(area); } if (!_seeded) { await SeedAsync(now, cancellationToken).ConfigureAwait(false); _seeded = true; } IWindowsServerSession session; try { session = await source.ConnectAsync(Server.Host, cancellationToken).ConfigureAwait(false); } catch (ServerAccessException ex) { await NotReachableAsync(ex, due, now, cancellationToken).ConfigureAwait(false); return; } using (session) { if (_wasReachable == false) { await NotifyAsync([NotificationFactory.Recovered(Server.Name, webUrl, now) with { Url = ServerNotifications.ServerPage(webUrl, ServerId) }], cancellationToken) .ConfigureAwait(false); } _wasReachable = true; _reported.Remove("connect"); foreach (var area in due) { await CollectAsync(session, area, now, cancellationToken).ConfigureAwait(false); } if (due.Any(a => a != ServerArea.Performance) || now >= _nextEvaluation) { _nextEvaluation = now + EvaluationInterval; await EvaluateAsync(now, cancellationToken).ConfigureAwait(false); } } } /// Unreachable: recorded and notified once. Refused by rights at the door: every area is "ikke målbar" with the reason. private async Task NotReachableAsync(ServerAccessException ex, IReadOnlyList due, DateTimeOffset now, CancellationToken cancellationToken) { if (Report("connect", ex.Message)) { LogNotReachable(logger, Server.Name, ex.Message); } await repository.RecordServerErrorAsync(ServerId, ex.Message, now, cancellationToken).ConfigureAwait(false); if (ex.Problem is ServerAccessProblem.AccessDenied or ServerAccessProblem.NotSupported) { foreach (var area in ServerArea.All) { Unavailable(area, ex.Message); } await repository.RecordServerAreasAsync(ServerId, _areas.Values.ToList(), cancellationToken).ConfigureAwait(false); if (due.Any(a => a != ServerArea.Performance) || now >= _nextEvaluation) { _nextEvaluation = now + EvaluationInterval; await EvaluateAsync(now, cancellationToken).ConfigureAwait(false); } return; } if (_wasReachable != false) { await NotifyAsync([NotificationFactory.Unreachable(Server.Name, ex.Message, webUrl, now) with { Url = ServerNotifications.ServerPage(webUrl, ServerId) }], cancellationToken).ConfigureAwait(false); } _wasReachable = false; } private async Task CollectAsync(IWindowsServerSession session, string area, DateTimeOffset now, CancellationToken cancellationToken) { try { var detail = area switch { ServerArea.Performance => await CollectPerformanceAsync(session, now, cancellationToken).ConfigureAwait(false), ServerArea.Disks => await CollectDisksAsync(session, now, cancellationToken).ConfigureAwait(false), ServerArea.Services => await CollectServicesAsync(session, now, cancellationToken).ConfigureAwait(false), ServerArea.Events => await CollectEventsAsync(session, now, cancellationToken).ConfigureAwait(false), ServerArea.Tasks => await CollectTasksAsync(session, now, cancellationToken).ConfigureAwait(false), _ => null, }; _areas[area] = new ServerAreaStatus(area, true, detail, ServerArea.Right(area)); if (_reported.Remove(area)) { LogAreaRecovered(logger, Server.Name, area); } } catch (ServerAccessException ex) { Unavailable(area, ex.Message); if (Report(area, ex.Message)) { LogAreaUnavailable(logger, Server.Name, area, ex.Message); } } } private void Unavailable(string area, string reason) { _areas[area] = new ServerAreaStatus(area, false, reason, ServerArea.Right(area)); switch (area) { case ServerArea.Performance: _data.MarkUnavailable(reason); break; case ServerArea.Disks: _data.MarkUnavailable(reason); break; case ServerArea.Services: _data.MarkUnavailable(reason); break; case ServerArea.Events: _data.MarkUnavailable(reason); break; case ServerArea.Tasks: _data.MarkUnavailable(reason); break; } } /// Whether a problem is new (or changed) and should be logged: the same problem every minute is not news. private bool Report(string key, string message) { if (_reported.TryGetValue(key, out var previous) && previous == message) { return false; } _reported[key] = message; return true; } public async Task CollectPerformanceAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(session); var os = await session.GetOperatingSystemAsync(cancellationToken).ConfigureAwait(false); var raw = await session.GetPerformanceAsync(now, cancellationToken).ConfigureAwait(false); DateTimeOffset? previousBoot = null; if (os.LastBootUpTime is { } boot && _lastBoot is { } before && Math.Abs((boot - before).TotalMinutes) > 2) { // Restarted since the last look: the raw counters started over too. previousBoot = before; _restartSeenAt = now; _previousRaw = null; } _lastBoot = os.LastBootUpTime ?? _lastBoot; var sample = ServerPerformanceMath.Sample(_previousRaw, raw, os, options.IntervalOf(ServerArea.Performance)); _previousRaw = raw; _samples = ServerPerformanceMath.Append(_samples, sample); var snapshot = new ServerPerformanceSnapshot { CapturedAt = now, Os = os, Raw = raw, Samples = _samples, PreviousBootUpTime = previousBoot, RestartSeenAt = _restartSeenAt, Protocol = session.Protocol, }; _data.Set(snapshot); await repository.SaveServerPerformanceAsync(ServerId, sample, cancellationToken).ConfigureAwait(false); await repository.RecordServerContactAsync(ServerId, os, session.Protocol, null, now, cancellationToken).ConfigureAwait(false); await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Performance, snapshot with { Samples = [] }, now, cancellationToken).ConfigureAwait(false); return sample.CpuPercent is { } cpu ? $"CPU {cpu:0} %" : null; } public async Task CollectDisksAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(session); var disks = await session.GetDisksAsync(cancellationToken).ConfigureAwait(false); await repository.SaveServerDisksAsync(ServerId, now, disks, cancellationToken).ConfigureAwait(false); var historyDays = Math.Max(1, options.Disks.ForecastHistoryDays); var history = await repository.GetServerDiskHistoryAsync(ServerId, now.AddDays(-historyDays), cancellationToken).ConfigureAwait(false); var settings = new DiskForecastSettings { HistoryDays = historyDays }; var forecasts = disks.Select(d => DiskForecaster.Forecast(d.Volume, history.TryGetValue(d.Volume, out var points) ? points : [], now, settings)).ToList(); var snapshot = new ServerDiskSnapshot { CapturedAt = now, Disks = disks, Forecasts = forecasts }; _data.Set(snapshot); await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Disks, snapshot, now, cancellationToken).ConfigureAwait(false); return $"{disks.Count} drev"; } public async Task CollectServicesAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(session); var services = await session.GetServicesAsync(cancellationToken).ConfigureAwait(false); var snapshot = new ServerServiceSnapshot { CapturedAt = now, Services = services }; _data.Set(snapshot); await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Services, snapshot, now, cancellationToken).ConfigureAwait(false); return $"{services.Count} tjenester"; } /// The first time, the last day is read, so recent problems show at once; then from where the last read stopped. public async Task CollectEventsAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(session); var since = _events?.ReadUntil ?? now - ServerEventAnalysis.RecentWindow; var read = await session.GetEventsAsync(since, Math.Max(10, options.EventLog.MaxEventsPerCollection), cancellationToken).ConfigureAwait(false); var (groups, ignored) = ServerEventAnalysis.Group(read.Entries, ServerEventAnalysis.DefaultIgnore.Concat(options.EventLog.Ignore)); // Strictly after the newest event read; without events, keep ten minutes' margin for events written late. var readUntil = read.Entries.Count > 0 ? read.Entries.Max(e => e.At) : Max(since, now.AddMinutes(-10)); var snapshot = new ServerEventSnapshot { CapturedAt = now, ReadUntil = readUntil, Method = read.Method, Groups = groups, Recent = ServerEventAnalysis.Recent(_events?.Recent ?? [], groups, now), Ignored = ignored, Coverage = read.Truncated ? $"Der var flere end {options.EventLog.MaxEventsPerCollection} hændelser i en log siden sidst; resten læses ved næste indsamling." : read.Note, }; _events = snapshot; _data.Set(snapshot); await repository.SaveServerEventsAsync(ServerId, snapshot, cancellationToken).ConfigureAwait(false); await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Events, snapshot with { Groups = [] }, now, cancellationToken).ConfigureAwait(false); return $"{read.Entries.Count} hændelser ({read.Method})"; static DateTimeOffset Max(DateTimeOffset a, DateTimeOffset b) => a > b ? a : b; } public async Task CollectTasksAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(session); var tasks = options.Tasks; var read = await session.GetTasksAsync( tasks.Watches, Math.Max(1, tasks.MaxWatched), cancellationToken).ConfigureAwait(false); var snapshot = new ServerTaskSnapshot { CapturedAt = now, Tasks = read.Watched, Visible = read.Visible, Coverage = read.Note }; _data.Set(snapshot); await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Tasks, snapshot, now, cancellationToken).ConfigureAwait(false); return $"{read.Watched.Count} af {read.Visible} opgaver overvåges"; } private async Task EvaluateAsync(DateTimeOffset now, CancellationToken cancellationToken) { await repository.RecordServerAreasAsync(ServerId, ServerArea.All.Where(_areas.ContainsKey).Select(a => _areas[a]).ToList(), cancellationToken).ConfigureAwait(false); var context = new ServerRuleContext(Server.ToIdentity(), now, _data, options) { PerformanceInterval = options.IntervalOf(ServerArea.Performance) }; var results = ScopedRuleEngine.Evaluate(WindowsServerRules.Create(), context); var changes = await repository.SaveScopedRuleResultsAsync(FindingScope.Server, ServerId, results, now, cancellationToken).ConfigureAwait(false); await NotifyAsync(ServerNotifications.For(Server.Name, changes, webUrl, now).ToList(), cancellationToken).ConfigureAwait(false); if (logger.IsEnabled(LogLevel.Debug)) { var findings = results.Sum(r => r.Findings.Count); var notMeasurable = results.Count(r => r.Outcome is RuleOutcome.NotMeasurable or RuleOutcome.Error); LogCycle(logger, Server.Name, findings, notMeasurable); } } /// What a restart of the agent would lose: the last hour's samples, the boot time and where the event log was read to. private async Task SeedAsync(DateTimeOffset now, CancellationToken cancellationToken) { var stored = await repository.GetServerPerformanceAsync(ServerId, now - ServerPerformanceMath.Window, cancellationToken).ConfigureAwait(false); _samples = stored.Select(s => new ServerPerformanceSample(new DateTimeOffset(DateTime.SpecifyKind(s.SampledAt, DateTimeKind.Utc)), s.CpuPercent, s.AvailableMb, s.AvailableMb is { } mb && s.AvailablePct is > 0 ? mb * 100 / s.AvailablePct.Value : null, s.PagesPerSec, s.CommitPct)).ToList(); var performance = await repository.GetServerSnapshotAsync(ServerId, ServerArea.Performance, cancellationToken).ConfigureAwait(false); _lastBoot = performance?.Os.LastBootUpTime; _restartSeenAt = performance?.RestartSeenAt; _events = await repository.GetServerSnapshotAsync(ServerId, ServerArea.Events, cancellationToken).ConfigureAwait(false); } private async Task NotifyAsync(List events, CancellationToken cancellationToken) { if (notifications is not null && events.Count > 0) { await notifications.DispatchAsync(events, cancellationToken).ConfigureAwait(false); } } [LoggerMessage(Level = LogLevel.Warning, Message = "Windows server {Server} cannot be read: {Reason}")] private static partial void LogNotReachable(ILogger logger, string server, string reason); [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: {Area} is not measurable: {Reason}")] private static partial void LogAreaUnavailable(ILogger logger, string server, string area, string reason); [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: {Area} can be read again")] private static partial void LogAreaRecovered(ILogger logger, string server, string area); [LoggerMessage(Level = LogLevel.Debug, Message = "Windows server {Server}: {Findings} findings, {NotMeasurable} rules not measurable")] private static partial void LogCycle(ILogger logger, string server, int findings, int notMeasurable); }