From 6eae719e959652eb0a39e023de13e40d2e704ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Eggert=20Lundsteen=20Olsen?= Date: Fri, 2 Oct 2026 13:32:29 +0200 Subject: [PATCH] Agent: monitor Windows servers and run synthetic checks; server tests and check runs on request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ServerMonitoringService keeps the SQL Server hosts in step with the monitored instances (SQLInsight:Servers:IncludeSqlHosts), picks up servers added, paused or changed in SQLInsight Web within a minute, and runs each server's due areas with a limit on parallelism. Default intervals: 1 min CPU/memory, 15 min disks and tasks, 5 min services and events (SQLInsight:Servers:Intervals). - ServerMonitor: one CIM session per cycle; per-minute samples, disk samples and the "fuld om ca. N dage" forecast from the stored history, grouped events (read from where the last read stopped), services and tasks. A missing right makes only that area "ikke målbar" with the exact reason, logged once per change; a server that does not answer is notified once, and again when it answers. Survives an agent restart (samples, boot time, event position). - SyntheticCheckService runs checks on their interval, opens a SQL login password sealed for the agent (AgentCredentialKey.OpenContext), looks up the linked Windows service when a check fails, stores results and findings, notifies with links to /servere/fund/{id}. - AgentRequestService hands ServerTest and SyntheticCheckRun to ServerRequestHandler. - appsettings.json: SQLInsight:Servers and SQLInsight:SyntheticChecks with every threshold, interval and retention. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Configuration/AgentCredentialKey.cs | 3 + .../Configuration/AgentRequestService.cs | 4 +- src/SQLInsight.Agent/Program.cs | 2 + src/SQLInsight.Agent/SQLInsight.Agent.csproj | 1 + src/SQLInsight.Agent/Servers/ServerMonitor.cs | 359 ++++++++++++++++++ .../Servers/ServerMonitoringService.cs | 162 ++++++++ src/SQLInsight.Agent/Servers/ServerOptions.cs | 86 +++++ .../Servers/ServerRequestHandler.cs | 51 +++ .../Servers/ServerServices.cs | 29 ++ .../Servers/SyntheticCheckService.cs | 222 +++++++++++ src/SQLInsight.Agent/appsettings.json | 29 ++ .../Servers/ServerRuleSettings.cs | 6 + .../ServerAgentTests.cs | 318 ++++++++++++++++ .../SQLInsight.Servers.Tests/LocalCimTests.cs | 2 +- 14 files changed, 1272 insertions(+), 2 deletions(-) create mode 100644 src/SQLInsight.Agent/Servers/ServerMonitor.cs create mode 100644 src/SQLInsight.Agent/Servers/ServerMonitoringService.cs create mode 100644 src/SQLInsight.Agent/Servers/ServerOptions.cs create mode 100644 src/SQLInsight.Agent/Servers/ServerRequestHandler.cs create mode 100644 src/SQLInsight.Agent/Servers/ServerServices.cs create mode 100644 src/SQLInsight.Agent/Servers/SyntheticCheckService.cs create mode 100644 tests/SQLInsight.IntegrationTests/ServerAgentTests.cs diff --git a/src/SQLInsight.Agent/Configuration/AgentCredentialKey.cs b/src/SQLInsight.Agent/Configuration/AgentCredentialKey.cs index 82be929..fe11d4a 100644 --- a/src/SQLInsight.Agent/Configuration/AgentCredentialKey.cs +++ b/src/SQLInsight.Agent/Configuration/AgentCredentialKey.cs @@ -87,6 +87,9 @@ public sealed partial class AgentCredentialKey : IDisposable public string Open(byte[] envelope, string server, string? userName) => CredentialEnvelope.Open(envelope, KeyId, _rsa, CredentialEnvelope.Context(server, userName)); + /// Opens a secret sealed for this agent and the given context (e.g. a synthetic check's server and login). + public string OpenContext(byte[] envelope, string context) => CredentialEnvelope.Open(envelope, KeyId, _rsa, context); + /// What the agent publishes in insight.AgentKeys: the key, and the account and computer it runs as. public AgentKeyRow Describe(DateTimeOffset now) => new() { diff --git a/src/SQLInsight.Agent/Configuration/AgentRequestService.cs b/src/SQLInsight.Agent/Configuration/AgentRequestService.cs index 37da56e..d6636c0 100644 --- a/src/SQLInsight.Agent/Configuration/AgentRequestService.cs +++ b/src/SQLInsight.Agent/Configuration/AgentRequestService.cs @@ -31,7 +31,8 @@ public sealed partial class AgentRequestService( ISqlConnectionFactory connectionFactory, RepositoryReadiness readiness, TimeProvider timeProvider, - ILogger logger) : BackgroundService + ILogger logger, + Servers.ServerRequestHandler? serverRequests = null) : BackgroundService { /// How often the queue is looked at. Short, so the web UI gets an answer within seconds. public static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(2); @@ -97,6 +98,7 @@ public sealed partial class AgentRequestService( AgentRequestKind.DatabaseDiscovery => AgentRequestJson.Serialize(await DiscoverAsync(Require(request), timeout.Token).ConfigureAwait(false)), AgentRequestKind.InstanceDiscovery => AgentRequestJson.Serialize(await InstanceFinder.FindAsync( AgentRequestJson.Deserialize(request.Parameters)?.Host, TimeSpan.FromSeconds(3), timeout.Token).ConfigureAwait(false)), + _ when serverRequests is not null && Servers.ServerRequestHandler.Handles(request.Kind) => await serverRequests.HandleAsync(request, timeout.Token).ConfigureAwait(false), _ => throw new InvalidOperationException($"Ukendt anmodning: {request.Kind}."), }; } diff --git a/src/SQLInsight.Agent/Program.cs b/src/SQLInsight.Agent/Program.cs index 5241882..d215209 100644 --- a/src/SQLInsight.Agent/Program.cs +++ b/src/SQLInsight.Agent/Program.cs @@ -116,6 +116,8 @@ builder.Services.AddSingleton(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); +// 1.5: Windows servers (CIM, as the agent's identity) and synthetic checks. +SQLInsight.Agent.Servers.ServerServices.Add(builder.Services, builder.Configuration); var host = builder.Build(); await host.RunAsync(); diff --git a/src/SQLInsight.Agent/SQLInsight.Agent.csproj b/src/SQLInsight.Agent/SQLInsight.Agent.csproj index 7e207bd..61d9b5a 100644 --- a/src/SQLInsight.Agent/SQLInsight.Agent.csproj +++ b/src/SQLInsight.Agent/SQLInsight.Agent.csproj @@ -14,6 +14,7 @@ + diff --git a/src/SQLInsight.Agent/Servers/ServerMonitor.cs b/src/SQLInsight.Agent/Servers/ServerMonitor.cs new file mode 100644 index 0000000..fd7f99a --- /dev/null +++ b/src/SQLInsight.Agent/Servers/ServerMonitor.cs @@ -0,0 +1,359 @@ +using SQLInsight.Core.Collection; +using SQLInsight.Core.Servers; +using SQLInsight.Notifications; +using SQLInsight.Repository; +using SQLInsight.Rules.Engine; +using SQLInsight.Rules.Servers; +using SQLInsight.Servers; + +namespace SQLInsight.Agent.Servers; + +/// +/// Looks at one Windows server: reads the areas that are due over one CIM session, keeps the latest data for the rules, +/// evaluates them and stores findings. Only reads; a missing right makes that area "ikke målbar" with the exact reason +/// (logged once per change, never every round). Not thread-safe; one cycle at a time. +/// +public sealed partial class ServerMonitor( + WindowsServerRow server, + IWindowsServerSource source, + InsightRepository repository, + ServerOptions options, + ILogger logger, + NotificationDispatcher? notifications = null, + Uri? webUrl = null) +{ + /// Rules are evaluated at least this often even when only performance was collected. + public static readonly TimeSpan EvaluationInterval = TimeSpan.FromMinutes(5); + + private readonly CollectedData _data = new(); + private readonly Dictionary _nextDue = new(StringComparer.Ordinal); + private readonly Dictionary _areas = new(StringComparer.Ordinal); + private readonly Dictionary _reported = new(StringComparer.Ordinal); + private ServerPerformanceRaw? _previousRaw; + private IReadOnlyList _samples = []; + private DateTimeOffset? _lastBoot; + private DateTimeOffset? _restartSeenAt; + private ServerEventSnapshot? _events; + private DateTimeOffset _nextEvaluation = DateTimeOffset.MinValue; + private bool _seeded; + private bool? _wasReachable; + + public WindowsServerRow Server { get; private set; } = server; + + public int ServerId => Server.ServerId; + + /// The configuration changed in SQLInsight Web (role, watch lists): used from the next cycle. + public void Update(WindowsServerRow row) => Server = row; + + public bool IsDue(DateTimeOffset now) => ServerArea.All.Any(a => !_nextDue.TryGetValue(a, out var due) || due <= now); + + public async Task RunCycleAsync(DateTimeOffset now, CancellationToken cancellationToken) + { + var due = ServerArea.All.Where(a => !_nextDue.TryGetValue(a, out var at) || at <= now).ToList(); + if (due.Count == 0) + { + return; + } + + foreach (var area in due) + { + _nextDue[area] = now + options.IntervalOf(area); + } + + if (!_seeded) + { + await SeedAsync(now, cancellationToken).ConfigureAwait(false); + _seeded = true; + } + + IWindowsServerSession session; + try + { + session = await source.ConnectAsync(Server.Host, cancellationToken).ConfigureAwait(false); + } + catch (ServerAccessException ex) + { + await NotReachableAsync(ex, due, now, cancellationToken).ConfigureAwait(false); + return; + } + + using (session) + { + if (_wasReachable == false) + { + await NotifyAsync([NotificationFactory.Recovered(Server.Name, webUrl, now) with { Url = ServerNotifications.ServerPage(webUrl, ServerId) }], cancellationToken) + .ConfigureAwait(false); + } + + _wasReachable = true; + _reported.Remove("connect"); + foreach (var area in due) + { + await CollectAsync(session, area, now, cancellationToken).ConfigureAwait(false); + } + + if (due.Any(a => a != ServerArea.Performance) || now >= _nextEvaluation) + { + _nextEvaluation = now + EvaluationInterval; + await EvaluateAsync(now, cancellationToken).ConfigureAwait(false); + } + } + } + + /// Unreachable: recorded and notified once. Refused by rights at the door: every area is "ikke målbar" with the reason. + private async Task NotReachableAsync(ServerAccessException ex, IReadOnlyList due, DateTimeOffset now, CancellationToken cancellationToken) + { + if (Report("connect", ex.Message)) + { + LogNotReachable(logger, Server.Name, ex.Message); + } + + await repository.RecordServerErrorAsync(ServerId, ex.Message, now, cancellationToken).ConfigureAwait(false); + if (ex.Problem is ServerAccessProblem.AccessDenied or ServerAccessProblem.NotSupported) + { + foreach (var area in ServerArea.All) + { + Unavailable(area, ex.Message); + } + + await repository.RecordServerAreasAsync(ServerId, _areas.Values.ToList(), cancellationToken).ConfigureAwait(false); + if (due.Any(a => a != ServerArea.Performance) || now >= _nextEvaluation) + { + _nextEvaluation = now + EvaluationInterval; + await EvaluateAsync(now, cancellationToken).ConfigureAwait(false); + } + + return; + } + + if (_wasReachable != false) + { + await NotifyAsync([NotificationFactory.Unreachable(Server.Name, ex.Message, webUrl, now) with { Url = ServerNotifications.ServerPage(webUrl, ServerId) }], + cancellationToken).ConfigureAwait(false); + } + + _wasReachable = false; + } + + private async Task CollectAsync(IWindowsServerSession session, string area, DateTimeOffset now, CancellationToken cancellationToken) + { + try + { + var detail = area switch + { + ServerArea.Performance => await CollectPerformanceAsync(session, now, cancellationToken).ConfigureAwait(false), + ServerArea.Disks => await CollectDisksAsync(session, now, cancellationToken).ConfigureAwait(false), + ServerArea.Services => await CollectServicesAsync(session, now, cancellationToken).ConfigureAwait(false), + ServerArea.Events => await CollectEventsAsync(session, now, cancellationToken).ConfigureAwait(false), + ServerArea.Tasks => await CollectTasksAsync(session, now, cancellationToken).ConfigureAwait(false), + _ => null, + }; + _areas[area] = new ServerAreaStatus(area, true, detail, ServerArea.Right(area)); + if (_reported.Remove(area)) + { + LogAreaRecovered(logger, Server.Name, area); + } + } + catch (ServerAccessException ex) + { + Unavailable(area, ex.Message); + if (Report(area, ex.Message)) + { + LogAreaUnavailable(logger, Server.Name, area, ex.Message); + } + } + } + + private void Unavailable(string area, string reason) + { + _areas[area] = new ServerAreaStatus(area, false, reason, ServerArea.Right(area)); + switch (area) + { + case ServerArea.Performance: + _data.MarkUnavailable(reason); + break; + case ServerArea.Disks: + _data.MarkUnavailable(reason); + break; + case ServerArea.Services: + _data.MarkUnavailable(reason); + break; + case ServerArea.Events: + _data.MarkUnavailable(reason); + break; + case ServerArea.Tasks: + _data.MarkUnavailable(reason); + break; + } + } + + /// Whether a problem is new (or changed) and should be logged: the same problem every minute is not news. + private bool Report(string key, string message) + { + if (_reported.TryGetValue(key, out var previous) && previous == message) + { + return false; + } + + _reported[key] = message; + return true; + } + + public async Task CollectPerformanceAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(session); + var os = await session.GetOperatingSystemAsync(cancellationToken).ConfigureAwait(false); + var raw = await session.GetPerformanceAsync(now, cancellationToken).ConfigureAwait(false); + DateTimeOffset? previousBoot = null; + if (os.LastBootUpTime is { } boot && _lastBoot is { } before && Math.Abs((boot - before).TotalMinutes) > 2) + { + // Restarted since the last look: the raw counters started over too. + previousBoot = before; + _restartSeenAt = now; + _previousRaw = null; + } + + _lastBoot = os.LastBootUpTime ?? _lastBoot; + var sample = ServerPerformanceMath.Sample(_previousRaw, raw, os); + _previousRaw = raw; + _samples = ServerPerformanceMath.Append(_samples, sample); + var snapshot = new ServerPerformanceSnapshot + { + CapturedAt = now, + Os = os, + Raw = raw, + Samples = _samples, + PreviousBootUpTime = previousBoot, + RestartSeenAt = _restartSeenAt, + Protocol = session.Protocol, + }; + _data.Set(snapshot); + await repository.SaveServerPerformanceAsync(ServerId, sample, cancellationToken).ConfigureAwait(false); + await repository.RecordServerContactAsync(ServerId, os, session.Protocol, null, now, cancellationToken).ConfigureAwait(false); + await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Performance, snapshot with { Samples = [] }, now, cancellationToken).ConfigureAwait(false); + return sample.CpuPercent is { } cpu ? $"CPU {cpu:0} %" : null; + } + + public async Task CollectDisksAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(session); + var disks = await session.GetDisksAsync(cancellationToken).ConfigureAwait(false); + await repository.SaveServerDisksAsync(ServerId, now, disks, cancellationToken).ConfigureAwait(false); + var historyDays = Math.Max(1, options.Disks.ForecastHistoryDays); + var history = await repository.GetServerDiskHistoryAsync(ServerId, now.AddDays(-historyDays), cancellationToken).ConfigureAwait(false); + var settings = new DiskForecastSettings { HistoryDays = historyDays }; + var forecasts = disks.Select(d => DiskForecaster.Forecast(d.Volume, history.TryGetValue(d.Volume, out var points) ? points : [], now, settings)).ToList(); + var snapshot = new ServerDiskSnapshot { CapturedAt = now, Disks = disks, Forecasts = forecasts }; + _data.Set(snapshot); + await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Disks, snapshot, now, cancellationToken).ConfigureAwait(false); + return $"{disks.Count} drev"; + } + + public async Task CollectServicesAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(session); + var services = await session.GetServicesAsync(cancellationToken).ConfigureAwait(false); + var snapshot = new ServerServiceSnapshot { CapturedAt = now, Services = services }; + _data.Set(snapshot); + await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Services, snapshot, now, cancellationToken).ConfigureAwait(false); + return $"{services.Count} tjenester"; + } + + /// The first time, the last day is read, so recent problems show at once; then from where the last read stopped. + public async Task CollectEventsAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(session); + var since = _events?.ReadUntil ?? now - ServerEventAnalysis.RecentWindow; + var read = await session.GetEventsAsync(since, Math.Max(10, options.EventLog.MaxEventsPerCollection), cancellationToken).ConfigureAwait(false); + var (groups, ignored) = ServerEventAnalysis.Group(read.Entries, ServerEventAnalysis.DefaultIgnore.Concat(options.EventLog.Ignore)); + + // Strictly after the newest event read; without events, keep ten minutes' margin for events written late. + var readUntil = read.Entries.Count > 0 ? read.Entries.Max(e => e.At) : Max(since, now.AddMinutes(-10)); + var snapshot = new ServerEventSnapshot + { + CapturedAt = now, + ReadUntil = readUntil, + Method = read.Method, + Groups = groups, + Recent = ServerEventAnalysis.Recent(_events?.Recent ?? [], groups, now), + Ignored = ignored, + Coverage = read.Truncated + ? $"Der var flere end {options.EventLog.MaxEventsPerCollection} hændelser i en log siden sidst; resten læses ved næste indsamling." + : read.Note, + }; + _events = snapshot; + _data.Set(snapshot); + await repository.SaveServerEventsAsync(ServerId, snapshot, cancellationToken).ConfigureAwait(false); + await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Events, snapshot with { Groups = [] }, now, cancellationToken).ConfigureAwait(false); + return $"{read.Entries.Count} hændelser ({read.Method})"; + + static DateTimeOffset Max(DateTimeOffset a, DateTimeOffset b) => a > b ? a : b; + } + + public async Task CollectTasksAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(session); + var tasks = options.Tasks; + var read = await session.GetTasksAsync( + name => (tasks.IncludeMicrosoftTasks || !TaskResults.IsMicrosoftTask(name)) && !WindowsServerRulesAccess.Matches(name, tasks.Ignore), + Math.Max(1, tasks.MaxWatched), cancellationToken).ConfigureAwait(false); + var snapshot = new ServerTaskSnapshot { CapturedAt = now, Tasks = read.Watched, Visible = read.Visible, Coverage = read.Note }; + _data.Set(snapshot); + await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Tasks, snapshot, now, cancellationToken).ConfigureAwait(false); + return $"{read.Watched.Count} af {read.Visible} opgaver overvåges"; + } + + private async Task EvaluateAsync(DateTimeOffset now, CancellationToken cancellationToken) + { + await repository.RecordServerAreasAsync(ServerId, ServerArea.All.Where(_areas.ContainsKey).Select(a => _areas[a]).ToList(), cancellationToken).ConfigureAwait(false); + var context = new ServerRuleContext(Server.ToIdentity(), now, _data, options) { PerformanceInterval = options.IntervalOf(ServerArea.Performance) }; + var results = ScopedRuleEngine.Evaluate(WindowsServerRules.Create(), context); + var changes = await repository.SaveScopedRuleResultsAsync(FindingScope.Server, ServerId, results, now, cancellationToken).ConfigureAwait(false); + await NotifyAsync(ServerNotifications.For(Server.Name, changes, webUrl, now).ToList(), cancellationToken).ConfigureAwait(false); + if (logger.IsEnabled(LogLevel.Debug)) + { + var findings = results.Sum(r => r.Findings.Count); + var notMeasurable = results.Count(r => r.Outcome is RuleOutcome.NotMeasurable or RuleOutcome.Error); + LogCycle(logger, Server.Name, findings, notMeasurable); + } + } + + /// What a restart of the agent would lose: the last hour's samples, the boot time and where the event log was read to. + private async Task SeedAsync(DateTimeOffset now, CancellationToken cancellationToken) + { + var stored = await repository.GetServerPerformanceAsync(ServerId, now - ServerPerformanceMath.Window, cancellationToken).ConfigureAwait(false); + _samples = stored.Select(s => new ServerPerformanceSample(new DateTimeOffset(DateTime.SpecifyKind(s.SampledAt, DateTimeKind.Utc)), s.CpuPercent, s.AvailableMb, + s.AvailableMb is { } mb && s.AvailablePct is > 0 ? mb * 100 / s.AvailablePct.Value : null, s.PagesPerSec, s.CommitPct)).ToList(); + var performance = await repository.GetServerSnapshotAsync(ServerId, ServerArea.Performance, cancellationToken).ConfigureAwait(false); + _lastBoot = performance?.Os.LastBootUpTime; + _restartSeenAt = performance?.RestartSeenAt; + _events = await repository.GetServerSnapshotAsync(ServerId, ServerArea.Events, cancellationToken).ConfigureAwait(false); + } + + private async Task NotifyAsync(List events, CancellationToken cancellationToken) + { + if (notifications is not null && events.Count > 0) + { + await notifications.DispatchAsync(events, cancellationToken).ConfigureAwait(false); + } + } + + [LoggerMessage(Level = LogLevel.Warning, Message = "Windows server {Server} cannot be read: {Reason}")] + private static partial void LogNotReachable(ILogger logger, string server, string reason); + + [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: {Area} is not measurable: {Reason}")] + private static partial void LogAreaUnavailable(ILogger logger, string server, string area, string reason); + + [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: {Area} can be read again")] + private static partial void LogAreaRecovered(ILogger logger, string server, string area); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Windows server {Server}: {Findings} findings, {NotMeasurable} rules not measurable")] + private static partial void LogCycle(ILogger logger, string server, int findings, int notMeasurable); +} + +/// The rules' name matching (names, or prefixes ending in *), for the agent's task filter. +internal static class WindowsServerRulesAccess +{ + public static bool Matches(string name, IEnumerable patterns) => patterns.Any(p => + p.EndsWith('*') ? name.StartsWith(p[..^1], StringComparison.OrdinalIgnoreCase) : name.Equals(p, StringComparison.OrdinalIgnoreCase)); +} diff --git a/src/SQLInsight.Agent/Servers/ServerMonitoringService.cs b/src/SQLInsight.Agent/Servers/ServerMonitoringService.cs new file mode 100644 index 0000000..b3f6fa4 --- /dev/null +++ b/src/SQLInsight.Agent/Servers/ServerMonitoringService.cs @@ -0,0 +1,162 @@ +using Microsoft.Extensions.Options; +using SQLInsight.Core.Servers; +using SQLInsight.Notifications; +using SQLInsight.Repository; +using SQLInsight.Servers; + +namespace SQLInsight.Agent.Servers; + +/// +/// The agent's loop for Windows servers: keeps the SQL Server hosts in step with the monitored instances, reads the +/// configured servers (changes in SQLInsight Web apply within a minute, without a restart), runs each server's due +/// areas with a limit on how many at once, and deletes old samples hourly. +/// +public sealed partial class ServerMonitoringService( + IOptions options, + IOptions checkOptions, + IWindowsServerSource source, + InsightRepository repository, + TimeProvider time, + ILoggerFactory loggerFactory, + NotificationDispatcher notifications, + IOptions notificationOptions, + IOptions agentOptions, + Configuration.RepositoryReadiness readiness) : BackgroundService +{ + public static readonly TimeSpan RefreshInterval = TimeSpan.FromMinutes(1); + + private readonly ILogger _logger = loggerFactory.CreateLogger(); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var settings = options.Value; + if (!settings.Enabled) + { + LogDisabled(_logger); + return; + } + + await readiness.Ready.WaitAsync(stoppingToken).ConfigureAwait(false); + var monitors = new Dictionary(); + var running = new Dictionary(); + using var throttle = new SemaphoreSlim(Math.Max(1, settings.MaxParallelServers)); + var nextRefresh = DateTimeOffset.MinValue; + var nextMaintenance = time.GetUtcNow().AddMinutes(5); + using var timer = new PeriodicTimer(agentOptions.Value.Tick, time); + do + { + var now = time.GetUtcNow(); + try + { + if (now >= nextRefresh) + { + nextRefresh = now + RefreshInterval; + await RefreshAsync(monitors, settings, now, stoppingToken).ConfigureAwait(false); + } + + foreach (var (id, monitor) in monitors) + { + if (running.TryGetValue(id, out var task) && !task.IsCompleted || !monitor.IsDue(now)) + { + continue; + } + + running[id] = RunThrottledAsync(monitor, now, throttle, stoppingToken); + } + + foreach (var done in running.Where(r => r.Value.IsCompleted && !monitors.ContainsKey(r.Key)).Select(r => r.Key).ToList()) + { + running.Remove(done); + } + + if (now >= nextMaintenance) + { + nextMaintenance = now.AddHours(1); + await repository.RunServerMaintenanceAsync(new ServerRetention + { + SampleDays = settings.SampleDays, + DiskHistoryDays = Math.Max(settings.DiskHistoryDays, settings.Disks.ForecastHistoryDays + 1), + EventDays = settings.EventDays, + CheckResultDays = checkOptions.Value.ResultDays, + ResolvedFindingDays = agentOptions.Value.Retention.ResolvedFindingDays, + }, now, stoppingToken).ConfigureAwait(false); + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + // Typically the repository is unreachable. Try again next round. + LogRoundFailed(_logger, ex); + } + } + while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); + + await Task.WhenAll(running.Values).ConfigureAwait(false); + } + + /// The SQL Server hosts first, then the servers to look at (not paused). + private async Task RefreshAsync(Dictionary monitors, ServerOptions settings, DateTimeOffset now, CancellationToken cancellationToken) + { + if (settings.IncludeSqlHosts) + { + var hosts = (await repository.GetConfiguredInstancesAsync(cancellationToken).ConfigureAwait(false)) + .Select(i => ServerHost.FromDataSource(i.DataSource, Environment.MachineName).Host) + .OfType() + .Distinct(StringComparer.OrdinalIgnoreCase) + .Select(h => (h, ServerHost.ShortName(h))) + .ToList(); + await repository.SyncSqlHostServersAsync(hosts, now, cancellationToken).ConfigureAwait(false); + } + + var servers = (await repository.GetWindowsServersAsync(cancellationToken).ConfigureAwait(false)).Where(s => !s.Paused).ToList(); + foreach (var row in servers) + { + if (monitors.TryGetValue(row.ServerId, out var monitor) && string.Equals(monitor.Server.Host, row.Host, StringComparison.OrdinalIgnoreCase)) + { + monitor.Update(row); + continue; + } + + monitors[row.ServerId] = new ServerMonitor(row, source, repository, settings, loggerFactory.CreateLogger($"SQLInsight.Server.{row.Name}"), notifications, + notificationOptions.Value.WebUrl); + LogStarted(_logger, row.Name, row.Host); + } + + foreach (var gone in monitors.Keys.Where(id => servers.All(s => s.ServerId != id)).ToList()) + { + LogStopped(_logger, monitors[gone].Server.Name); + monitors.Remove(gone); + } + } + + private async Task RunThrottledAsync(ServerMonitor monitor, DateTimeOffset now, SemaphoreSlim throttle, CancellationToken cancellationToken) + { + await throttle.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await monitor.RunCycleAsync(now, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogCycleFailed(_logger, ex, monitor.Server.Name); + } + finally + { + throttle.Release(); + } + } + + [LoggerMessage(Level = LogLevel.Information, Message = "Windows servers are not monitored (SQLInsight:Servers:Enabled is false).")] + private static partial void LogDisabled(ILogger logger); + + [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server} ({Host}): monitoring started")] + private static partial void LogStarted(ILogger logger, string server, string host); + + [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: monitoring stopped (paused or removed)")] + private static partial void LogStopped(ILogger logger, string server); + + [LoggerMessage(Level = LogLevel.Error, Message = "Windows server {Server}: cycle failed")] + private static partial void LogCycleFailed(ILogger logger, Exception exception, string server); + + [LoggerMessage(Level = LogLevel.Error, Message = "Monitoring Windows servers failed this round; retrying.")] + private static partial void LogRoundFailed(ILogger logger, Exception exception); +} diff --git a/src/SQLInsight.Agent/Servers/ServerOptions.cs b/src/SQLInsight.Agent/Servers/ServerOptions.cs new file mode 100644 index 0000000..393197b --- /dev/null +++ b/src/SQLInsight.Agent/Servers/ServerOptions.cs @@ -0,0 +1,86 @@ +using SQLInsight.Core.Servers; +using SQLInsight.Notifications; +using SQLInsight.Repository; +using SQLInsight.Rules.Servers; +using SQLInsight.Servers; + +namespace SQLInsight.Agent.Servers; + +/// Windows servers (SQLInsight:Servers): how and how often to collect, how long to keep it, and the rules' thresholds. +public sealed class ServerOptions : ServerRuleSettings +{ + public const string SectionName = "SQLInsight:Servers"; + + /// Look at Windows servers at all. + public bool Enabled { get; set; } = true; + + /// The computers behind the monitored SQL Server instances are Windows servers too, without being added. + public bool IncludeSqlHosts { get; set; } = true; + + /// How many servers are read at the same time. + public int MaxParallelServers { get; set; } = 4; + + /// Auto (WS-Man, then DCOM), WSMan or DCOM. + public string Protocol { get; set; } = "Auto"; + + /// How long one CIM operation may take. + public TimeSpan OperationTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// Read event logs with the Event Log service's own protocol (filters on the server); off: CIM only. + public bool EventLogRpc { get; set; } = true; + + /// Overrides of the collection intervals by area: performance, disks, services, events, tasks. + public IDictionary Intervals { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + + /// Per-minute CPU and memory samples. + public int SampleDays { get; set; } = 14; + + /// Free space per volume: at least the forecast's history. + public int DiskHistoryDays { get; set; } = 35; + + /// Grouped Windows events. + public int EventDays { get; set; } = 90; + + public TimeSpan IntervalOf(string area) => + Intervals.TryGetValue(area, out var interval) && interval >= TimeSpan.FromSeconds(30) ? interval : ServerArea.DefaultInterval(area); + + public ServerConnectionOptions Connection => new() { Protocol = Protocol, OperationTimeout = OperationTimeout, EventLogRpc = EventLogRpc }; +} + +/// Synthetic checks (SQLInsight:SyntheticChecks): how many run at once, how long results are kept, and the rules' thresholds. +public sealed class SyntheticCheckOptions : SyntheticCheckRuleSettings +{ + public const string SectionName = "SQLInsight:SyntheticChecks"; + + public bool Enabled { get; set; } = true; + + public int MaxParallelChecks { get; set; } = 4; + + /// Every run's result. + public int ResultDays { get; set; } = 14; + + /// Where the Dynamics AX .NET Business Connector is, when not where the AX installer puts it. + public string? BusinessConnectorPath { get; set; } +} + +/// Notifications about Windows servers and synthetic checks, with links to their pages in the web UI. +internal static class ServerNotifications +{ + public static IEnumerable For(string name, FindingChanges changes, Uri? webUrl, DateTimeOffset now) + { + foreach (var (id, finding) in changes.Opened) + { + yield return NotificationFactory.Opened(name, id, finding, webUrl, now) with { Url = Link(webUrl, id) }; + } + + foreach (var resolved in changes.Resolved) + { + yield return NotificationFactory.Resolved(name, resolved.FindingId, resolved.RuleId, resolved.Title, (Core.Findings.Severity)resolved.Severity, webUrl, now) + with { Url = Link(webUrl, resolved.FindingId) }; + } + } + + public static Uri? Link(Uri? webUrl, long findingId) => webUrl is null ? null : new Uri(webUrl, $"servere/fund/{findingId}"); + + public static Uri? ServerPage(Uri? webUrl, int serverId) => webUrl is null ? null : new Uri(webUrl, $"servere/{serverId}"); +} diff --git a/src/SQLInsight.Agent/Servers/ServerRequestHandler.cs b/src/SQLInsight.Agent/Servers/ServerRequestHandler.cs new file mode 100644 index 0000000..0f0cb29 --- /dev/null +++ b/src/SQLInsight.Agent/Servers/ServerRequestHandler.cs @@ -0,0 +1,51 @@ +using SQLInsight.Core.Configuration; +using SQLInsight.Core.Servers; +using SQLInsight.Repository; +using SQLInsight.Servers; + +namespace SQLInsight.Agent.Servers; + +/// +/// Requests from SQLInsight Web about Windows servers and synthetic checks (insight.AgentRequests with a TargetId): +/// test a server's rights as the agent, or run a check now. Called by the agent's request service. +/// +public sealed class ServerRequestHandler(InsightRepository repository, IWindowsServerSource source, SyntheticCheckService checks, TimeProvider time) +{ + public static bool Handles(string kind) => ServerRequestKind.Handles(kind); + + /// The request's result as JSON. Throws for a server or check that does not exist. + public async Task HandleAsync(AgentRequestRow request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + var target = await repository.GetRequestTargetAsync(request.RequestId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException("Anmodningen mangler en server eller et tjek."); + return request.Kind switch + { + ServerRequestKind.ServerTest => AgentRequestJson.Serialize(await TestAsync(target, cancellationToken).ConfigureAwait(false)), + ServerRequestKind.SyntheticCheckRun => AgentRequestJson.Serialize(await RunAsync(target, cancellationToken).ConfigureAwait(false)), + _ => throw new InvalidOperationException($"Ukendt anmodning: {request.Kind}."), + }; + } + + /// Connects as the agent and tries every area; what could and could not be read is also stored on the server. + public async Task TestAsync(int serverId, CancellationToken cancellationToken) + { + var server = await repository.GetWindowsServerAsync(serverId, cancellationToken).ConfigureAwait(false) + ?? throw new KeyNotFoundException($"Server {serverId} findes ikke."); + var account = OperatingSystem.IsWindows() ? CimWindowsServerSource.AgentAccount(server.Host) : null; + var result = await WindowsServerTester.TestAsync(source, server.Host, account, time.GetUtcNow(), cancellationToken).ConfigureAwait(false); + if (result.Reachable) + { + await repository.RecordServerAreasAsync(serverId, result.Areas, cancellationToken).ConfigureAwait(false); + } + + return result; + } + + public async Task RunAsync(int checkId, CancellationToken cancellationToken) + { + var check = await repository.GetSyntheticCheckAsync(checkId, cancellationToken).ConfigureAwait(false) + ?? throw new KeyNotFoundException($"Tjek {checkId} findes ikke."); + return await checks.RunCheckAsync(check, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/SQLInsight.Agent/Servers/ServerServices.cs b/src/SQLInsight.Agent/Servers/ServerServices.cs new file mode 100644 index 0000000..7c155d9 --- /dev/null +++ b/src/SQLInsight.Agent/Servers/ServerServices.cs @@ -0,0 +1,29 @@ +using Microsoft.Extensions.Options; +using SQLInsight.Connectivity; +using SQLInsight.Servers; +using SQLInsight.Servers.Synthetic; + +namespace SQLInsight.Agent.Servers; + +/// 1.5: Windows servers and synthetic checks in the agent. +public static class ServerServices +{ + public static void Add(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + services.Configure(configuration.GetSection(ServerOptions.SectionName)); + services.Configure(configuration.GetSection(SyntheticCheckOptions.SectionName)); + services.AddSingleton(sp => OperatingSystem.IsWindows() + ? new CimWindowsServerSource(sp.GetRequiredService>().Value.Connection) + : new UnsupportedWindowsServerSource()); + services.AddSingleton(sp => new BusinessConnectorLocator(sp.GetRequiredService>().Value.BusinessConnectorPath)); + services.AddSingleton(); + services.AddSingleton(sp => new SyntheticCheckRunner(sp.GetRequiredService(), sp.GetRequiredService(), + sp.GetRequiredService(), sp.GetRequiredService())); + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); + services.AddSingleton(); + services.AddHostedService(); + } +} diff --git a/src/SQLInsight.Agent/Servers/SyntheticCheckService.cs b/src/SQLInsight.Agent/Servers/SyntheticCheckService.cs new file mode 100644 index 0000000..333dde5 --- /dev/null +++ b/src/SQLInsight.Agent/Servers/SyntheticCheckService.cs @@ -0,0 +1,222 @@ +using Microsoft.Extensions.Options; +using SQLInsight.Core.Configuration; +using SQLInsight.Core.SyntheticChecks; +using SQLInsight.Notifications; +using SQLInsight.Repository; +using SQLInsight.Rules.Servers; +using SQLInsight.Servers; +using SQLInsight.Servers.Synthetic; + +namespace SQLInsight.Agent.Servers; + +/// +/// Runs the synthetic checks on their schedule (and on request from SQLInsight Web), as the agent's identity: stores +/// each result, looks up the linked Windows service when a check fails ("svarer ikke, selv om tjenesten kører"), +/// evaluates the rules and notifies. Never changes anything on the servers it checks. +/// +public sealed partial class SyntheticCheckService( + IOptions options, + SyntheticCheckRunner runner, + IWindowsServerSource servers, + InsightRepository repository, + Configuration.AgentCredentialKey key, + TimeProvider time, + ILogger logger, + NotificationDispatcher notifications, + IOptions notificationOptions, + IOptions agentOptions, + Configuration.RepositoryReadiness readiness) : BackgroundService +{ + private readonly Dictionary _running = []; + private readonly Dictionary _reported = []; + private readonly Lock _lock = new(); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var settings = options.Value; + if (!settings.Enabled) + { + return; + } + + await readiness.Ready.WaitAsync(stoppingToken).ConfigureAwait(false); + using var throttle = new SemaphoreSlim(Math.Max(1, settings.MaxParallelChecks)); + using var timer = new PeriodicTimer(agentOptions.Value.Tick, time); + do + { + try + { + var now = time.GetUtcNow(); + foreach (var check in await repository.GetSyntheticChecksAsync(stoppingToken).ConfigureAwait(false)) + { + if (check.Paused || (check.LastRunAt is { } last && last.AddSeconds(check.IntervalSeconds) > now.UtcDateTime)) + { + continue; + } + + lock (_lock) + { + if (_running.TryGetValue(check.CheckId, out var task) && !task.IsCompleted) + { + continue; + } + + _running[check.CheckId] = RunThrottledAsync(check, throttle, stoppingToken); + } + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogRoundFailed(logger, ex); + } + } + while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); + + Task[] remaining; + lock (_lock) + { + remaining = [.. _running.Values]; + } + + await Task.WhenAll(remaining).ConfigureAwait(false); + } + + private async Task RunThrottledAsync(SyntheticCheckRow check, SemaphoreSlim throttle, CancellationToken cancellationToken) + { + await throttle.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await RunCheckAsync(check, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogCheckFailed(logger, ex, check.Name); + } + finally + { + throttle.Release(); + } + } + + /// Runs a check once, stores the result and its findings. Used on schedule and for "Kør nu" in SQLInsight Web. + public async Task RunCheckAsync(SyntheticCheckRow check, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(check); + var settings = check.Settings; + var now = time.GetUtcNow(); + string? password = null; + if (check.Kind == SyntheticCheckKind.Sql && settings.Authentication == "SqlLogin") + { + (password, var error) = await OpenSecretAsync(check, settings, cancellationToken).ConfigureAwait(false); + if (error is not null) + { + return await StoreAsync(check, new SyntheticCheckResult { At = now, Success = false, Message = error }, cancellationToken).ConfigureAwait(false); + } + } + + var timeout = TimeSpan.FromSeconds(Math.Clamp(check.TimeoutSeconds, 1, SyntheticCheckLimits.MaxTimeoutSeconds)); + var result = await runner.RunAsync(new SyntheticCheckRun(check.Kind, settings, timeout, password), cancellationToken).ConfigureAwait(false); + if (!result.Success && check.ServerId is { } serverId && settings.ServiceName is { Length: > 0 } service) + { + result = await WithServiceStateAsync(result, serverId, service, cancellationToken).ConfigureAwait(false); + } + + return await StoreAsync(check, result, cancellationToken).ConfigureAwait(false); + } + + private async Task<(string? Password, string? Error)> OpenSecretAsync(SyntheticCheckRow check, SyntheticCheckSettings settings, CancellationToken cancellationToken) + { + string? error = null; + string? password = null; + if (check.SecretCipher is not { Length: > 0 } cipher) + { + error = "Der er ikke gemt en adgangskode til SQL-login'et."; + } + else + { + try + { + password = key.OpenContext(cipher, settings.SecretContext()); + } + catch (CredentialException ex) + { + error = "Adgangskoden kan ikke bruges: " + ex.Message; + } + } + + if (!string.Equals(error, check.SecretError, StringComparison.Ordinal)) + { + await repository.SetSyntheticCheckSecretErrorAsync(check.CheckId, error, cancellationToken).ConfigureAwait(false); + } + + return (password, error); + } + + /// The state of the Windows service behind a failing check: the difference between "nede" and "kører, men svarer ikke". + private async Task WithServiceStateAsync(SyntheticCheckResult result, int serverId, string service, CancellationToken cancellationToken) + { + var server = await repository.GetWindowsServerAsync(serverId, cancellationToken).ConfigureAwait(false); + if (server is null) + { + return result; + } + + try + { + using var session = await servers.ConnectAsync(server.Host, cancellationToken).ConfigureAwait(false); + var state = await session.GetServiceAsync(service, cancellationToken).ConfigureAwait(false); + return result with + { + ServiceState = state?.State ?? "NotFound", + Details = [.. result.Details, new SyntheticCheckDetail("Tjeneste", $"{service} på {server.Name}: {state?.State ?? "findes ikke"}")], + }; + } + catch (ServerAccessException ex) + { + return result with { Details = [.. result.Details, new SyntheticCheckDetail("Tjeneste", $"{service} på {server.Name} kunne ikke aflæses: {ex.Message}")] }; + } + } + + private async Task StoreAsync(SyntheticCheckRow check, SyntheticCheckResult result, CancellationToken cancellationToken) + { + await repository.RecordSyntheticCheckResultAsync(check.CheckId, result, cancellationToken).ConfigureAwait(false); + var recent = (await repository.GetSyntheticCheckResultsAsync(check.CheckId, 10, cancellationToken: cancellationToken).ConfigureAwait(false)) + .Select(r => r.ToResult()).ToList(); + var settings = check.Settings; + var identity = new SyntheticCheckIdentity(check.CheckId, check.Name, check.Kind, settings.Target(check.Kind)) + { + ServerName = check.ServerName, + ServiceName = settings.ServiceName, + WarnMilliseconds = settings.WarnMilliseconds, + Https = settings.Url?.Scheme == Uri.UriSchemeHttps, + }; + var results = ScopedRuleEngine.Evaluate(SyntheticCheckRules.Create(), new SyntheticCheckRuleContext(identity, result.At, recent, options.Value)); + var changes = await repository.SaveScopedRuleResultsAsync(FindingScope.Check, check.CheckId, results, result.At, cancellationToken).ConfigureAwait(false); + var events = ServerNotifications.For(check.Name, changes, notificationOptions.Value.WebUrl, result.At).ToList(); + if (events.Count > 0) + { + await notifications.DispatchAsync(events, cancellationToken).ConfigureAwait(false); + } + + lock (_lock) + { + var state = result.Success ? "ok" : result.Message; + if (!_reported.TryGetValue(check.CheckId, out var previous) || previous != state) + { + _reported[check.CheckId] = state; + LogResult(logger, check.Name, result.Success, result.Message); + } + } + + return result; + } + + [LoggerMessage(Level = LogLevel.Information, Message = "Synthetic check {Check}: success {Success}: {Message}")] + private static partial void LogResult(ILogger logger, string check, bool success, string message); + + [LoggerMessage(Level = LogLevel.Error, Message = "Synthetic check {Check} could not be run")] + private static partial void LogCheckFailed(ILogger logger, Exception exception, string check); + + [LoggerMessage(Level = LogLevel.Error, Message = "Running synthetic checks failed this round; retrying.")] + private static partial void LogRoundFailed(ILogger logger, Exception exception); +} diff --git a/src/SQLInsight.Agent/appsettings.json b/src/SQLInsight.Agent/appsettings.json index 9ed5ddc..ad3f978 100644 --- a/src/SQLInsight.Agent/appsettings.json +++ b/src/SQLInsight.Agent/appsettings.json @@ -62,6 +62,35 @@ "RecId32WarnPercent": 50 // "LocalTimeZone": "Romance Standard Time" // AOS'ens tidszone for Axapta 3.0/4.0's lokale tider; tom: agentens egen }, + "Servers": { + // Windows-servere (se docs/windows-servere.md). Rettigheder: db/permissions/grant-windows-server.ps1 – aldrig lokal administrator. + "Enabled": true, + "IncludeSqlHosts": true, + "Protocol": "Auto", + "MaxParallelServers": 4, + "Intervals": { "performance": "00:01:00", "disks": "00:15:00", "services": "00:05:00", "events": "00:05:00", "tasks": "00:15:00" }, + "Cpu": { "HighPercent": 90, "SustainedMinutes": 15 }, + "Memory": { "AvailablePercent": 5, "AvailableMegabytes": 1024, "PagesPerSecond": 1000, "CommitPercent": 90 }, + "Disks": { "FreePercent": 10, "CriticalFreePercent": 5, "EnoughFreeGigabytes": 50, "ForecastDays": 30, "ForecastHistoryDays": 14 }, + "Services": { "Ignore": [], "StartupGraceMinutes": 15 }, + "EventLog": { "RepeatThreshold": 5, "Ignore": [], "MaxFindings": 15 }, + "Tasks": { "IncludeMicrosoftTasks": false, "LateMinutes": 60, "Ignore": [] }, + "UptimeDays": 0, + "SampleDays": 14, + "DiskHistoryDays": 35, + "EventDays": 90 + }, + "SyntheticChecks": { + // Syntetiske tjek (se docs/syntetiske-tjek.md). Tjekkene selv oprettes i webbrugerfladen eller API'et. + "Enabled": true, + "MaxParallelChecks": 4, + "FailuresBeforeFinding": 2, + "SlowRunsBeforeFinding": 3, + "CertificateWarningDays": 21, + "CertificateCriticalDays": 7, + "ResultDays": 14, + "BusinessConnectorPath": null + }, "Instances": [ // Servere tilføjes nemmest i webbrugerfladen (Servere og databaser). Her kan de også stå, fx (se docs/installation.md): // { "Name": "prod-sql01", "Server": "sql01.domæne.local", "Authentication": "Windows", "Provider": "Auto", "Encryption": "Mandatory" } diff --git a/src/SQLInsight.Rules/Servers/ServerRuleSettings.cs b/src/SQLInsight.Rules/Servers/ServerRuleSettings.cs index 020d6d8..abf97cc 100644 --- a/src/SQLInsight.Rules/Servers/ServerRuleSettings.cs +++ b/src/SQLInsight.Rules/Servers/ServerRuleSettings.cs @@ -84,6 +84,9 @@ public sealed class EventLogThresholds /// At most this many event findings per server, most important first. public int MaxFindings { get; set; } = 15; + + /// At most this many events per log are read in one collection; the rest is read next time. + public int MaxEventsPerCollection { get; set; } = 2000; } public sealed class TaskThresholds @@ -94,6 +97,9 @@ public sealed class TaskThresholds /// A task whose next run time is more than this many minutes in the past has not run as planned (SQLI-WIN-013). public int LateMinutes { get; set; } = 60; + /// At most this many tasks per server are watched (their run information is read one at a time). + public int MaxWatched { get; set; } = 50; + /// Tasks never watched (path\name, or a prefix ending in *). public IList Ignore { get; } = []; } diff --git a/tests/SQLInsight.IntegrationTests/ServerAgentTests.cs b/tests/SQLInsight.IntegrationTests/ServerAgentTests.cs new file mode 100644 index 0000000..73625fd --- /dev/null +++ b/tests/SQLInsight.IntegrationTests/ServerAgentTests.cs @@ -0,0 +1,318 @@ +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using SQLInsight.Agent; +using SQLInsight.Agent.Configuration; +using SQLInsight.Agent.Servers; +using SQLInsight.Connectivity; +using SQLInsight.Core.Configuration; +using SQLInsight.Core.Findings; +using SQLInsight.Core.Servers; +using SQLInsight.Core.SyntheticChecks; +using SQLInsight.Notifications; +using SQLInsight.Repository; +using SQLInsight.Rules.Engine; +using SQLInsight.Servers; +using SQLInsight.Servers.Synthetic; + +namespace SQLInsight.IntegrationTests; + +/// +/// The agent's side of Windows servers and synthetic checks against a real repository, with a fake Windows server (the +/// CIM implementation itself is tested against this computer in SQLInsight.Servers.Tests). +/// +[Collection(SqlServerTestGroup.Name)] +[Trait("Category", "Integration")] +public sealed class ServerAgentTests(SqlServerFixture sql) : IAsyncLifetime +{ + private readonly string _database = "SQLInsightSrvAgent_" + Guid.NewGuid().ToString("N")[..8]; + private readonly string _keys = Path.Combine(Path.GetTempPath(), "sqlinsight-keys-" + Guid.NewGuid().ToString("N")); + private InsightRepository _repository = null!; + private AgentCredentialKey _key = null!; + private readonly Recorder _recorder = new(); + private static readonly Uri WebUrl = new("https://sqlinsight.test/"); + + public async Task InitializeAsync() + { + await sql.ExecuteAsSaAsync($"CREATE DATABASE [{_database}];"); + _repository = new InsightRepository(sql.DatabaseConnectionString(_database)); + await _repository.MigrateAsync(); + _key = AgentCredentialKey.LoadOrCreate(_keys); + await _repository.PublishAgentKeyAsync(_key.Describe(DateTimeOffset.UtcNow)); + } + + public async Task DisposeAsync() + { + _key.Dispose(); + Directory.Delete(_keys, recursive: true); + await sql.ExecuteAsSaAsync($"ALTER DATABASE [{_database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{_database}];"); + } + + private sealed class Recorder : INotificationChannel + { + public List Received { get; } = []; + public string Name => "recorder"; + public Severity MinSeverity => Severity.Info; + + public Task SendAsync(IReadOnlyList notifications, CancellationToken cancellationToken) + { + Received.AddRange(notifications); + return Task.CompletedTask; + } + } + + private NotificationDispatcher Dispatcher() => new([_recorder], new NotificationOptions { MaxMessagesPerHour = 1000 }, TimeProvider.System, NullLogger.Instance); + + /// A Windows server as the tests want it: what each area returns, or the error it throws. + private sealed class FakeServer : IWindowsServerSource, IWindowsServerSession + { + public ServerAccessException? ConnectError { get; set; } + public ServerAccessException? ServicesError { get; set; } + public ulong Idle { get; set; } + public ulong Clock { get; set; } = 1_000_000_000; + public double FreeGb { get; set; } = 3; + public string AosState { get; set; } = "Running"; + public int Connects { get; private set; } + + public string Protocol => "WS-Man"; + public string? ProtocolNote => null; + + public Task ConnectAsync(string host, CancellationToken cancellationToken) + { + Connects++; + return ConnectError is { } error ? throw error : Task.FromResult(this); + } + + public Task GetOperatingSystemAsync(CancellationToken cancellationToken) => Task.FromResult(new ServerOsInfo + { + Caption = "Microsoft Windows Server 2019 Standard", Version = "10.0.17763", TotalMemoryMb = 16384, LogicalProcessors = 4, + LastBootUpTime = DateTimeOffset.UtcNow.AddDays(-10), ComputerName = "AOS01", + }); + + public Task GetPerformanceAsync(DateTimeOffset now, CancellationToken cancellationToken) + { + // Each call: one minute passed, 95 % of it busy. + Clock += 600_000_000; + Idle += 30_000_000; + return Task.FromResult(new ServerPerformanceRaw + { + CapturedAt = now, ProcessorIdle100Ns = Idle, ProcessorTimestamp100Ns = Clock, AvailableMb = 6000, CommittedBytes = 4, CommitLimit = 10, + }); + } + + public Task> GetDisksAsync(CancellationToken cancellationToken) => + Task.FromResult>([new ServerDisk(@"C:\", "System", "NTFS", 100, 40), new ServerDisk(@"D:\", "Data", "NTFS", 100, FreeGb)]); + + public Task> GetServicesAsync(CancellationToken cancellationToken) => ServicesError is { } error + ? throw error + : Task.FromResult>([new ServerService("AOS60$01", "Microsoft Dynamics AX Object Server 6.0$01", AosState, "Auto")]); + + public Task GetServiceAsync(string name, CancellationToken cancellationToken) => + Task.FromResult(new ServerService(name, name, AosState, "Auto")); + + public Task GetEventsAsync(DateTimeOffset since, int max, CancellationToken cancellationToken) => Task.FromResult(new ServerEventRead( + [new ServerEventEntry("System", "disk", 153, 2, DateTimeOffset.UtcNow.AddMinutes(-1), @"The IO operation at logical block address 0x1 for Disk 1 was retried.")], + "Event Log (RPC)", false)); + + public Task GetTasksAsync(Func watch, int maxWatched, CancellationToken cancellationToken) => Task.FromResult(new ServerTaskRead( + [new ServerTask { Path = @"\", Name = "AX export", State = "Ready", LastTaskResult = 1, LastRunTime = DateTimeOffset.UtcNow.AddHours(-1) }], 140)); + + public void Dispose() + { + } + } + + private ServerMonitor Monitor(WindowsServerRow row, FakeServer server) => + new(row, server, _repository, new ServerOptions(), NullLogger.Instance, Dispatcher(), WebUrl); + + private async Task AddServerAsync(string name = "AOS01", IReadOnlyList? watched = null) + { + var id = await _repository.AddWindowsServerAsync(new NewWindowsServer(name, name.ToLowerInvariant() + ".firma.local", ServerRoles.Application, watched ?? [], []), "anna", + DateTimeOffset.UtcNow); + return (await _repository.GetWindowsServerAsync(id))!; + } + + [Fact] + public async Task A_cycle_reads_every_area_stores_it_and_opens_findings_with_links_to_the_server_pages() + { + var row = await AddServerAsync(); + var server = new FakeServer(); + var monitor = Monitor(row, server); + var now = DateTimeOffset.UtcNow; + + await monitor.RunCycleAsync(now, CancellationToken.None); + + var findings = await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId); + Assert.Contains(findings, f => f.RuleId == "SQLI-WIN-005" && f.Subject == @"disk: D:\"); // 3 GB free + Assert.Contains(findings, f => f.RuleId == "SQLI-WIN-009" && f.Subject == "event: System/disk/153"); + Assert.Contains(findings, f => f.RuleId == "SQLI-WIN-012" && f.Subject == @"task: \AX export"); + var stored = (await _repository.GetWindowsServerAsync(row.ServerId))!; + Assert.Equal(("WS-Man", "Microsoft Windows Server 2019 Standard", 4), (stored.Protocol, stored.OperatingSystem, stored.LogicalProcessors)); + Assert.All(stored.Areas, a => Assert.True(a.Ok, a.Detail)); + Assert.Equal(ServerArea.All.Count, stored.Areas.Count); + Assert.NotNull(await _repository.GetServerSnapshotAsync(row.ServerId, ServerArea.Disks)); + Assert.Single(await _repository.GetServerEventsAsync(row.ServerId, now.AddDays(-1), now)); + var opened = _recorder.Received.Where(n => n.Kind == NotificationKind.FindingOpened).ToList(); + Assert.Equal(findings.Count, opened.Count); + Assert.All(opened, n => Assert.StartsWith("https://sqlinsight.test/servere/fund/", n.Url!.ToString(), StringComparison.Ordinal)); + Assert.All(opened, n => Assert.Equal("AOS01", n.Instance)); + + // The CPU rule needs a quarter of an hour of minutes: the samples accumulate. + for (var minute = 1; minute <= 16; minute++) + { + await monitor.RunCycleAsync(now.AddMinutes(minute), CancellationToken.None); + } + + var cpu = Assert.Single(await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId), f => f.RuleId == "SQLI-WIN-001"); + Assert.Contains("95 %", cpu.Title, StringComparison.Ordinal); + Assert.Equal(17, (await _repository.GetServerPerformanceAsync(row.ServerId, now.AddHours(-1))).Count); + } + + [Fact] + public async Task A_missing_right_is_not_measurable_with_the_reason_and_logged_findings_stay_open() + { + var row = await AddServerAsync(watched: ["AOS60$01"]); + var server = new FakeServer { AosState = "Stopped" }; + var monitor = Monitor(row, server); + var now = DateTimeOffset.UtcNow; + await monitor.RunCycleAsync(now, CancellationToken.None); + Assert.Contains(await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId), f => f.RuleId == "SQLI-WIN-008"); + + server.ServicesError = ServerErrors.Describe(ServerErrors.AccessDenied, "AccessDenied", "Access is denied.", ServerArea.Services, "aos01", @"FIRMA\AGENT01$", "WS-Man"); + await monitor.RunCycleAsync(now.AddMinutes(6), CancellationToken.None); + + var states = await _repository.GetServerRuleStatesAsync(FindingScope.Server, row.ServerId); + var watched = states.Single(s => s.RuleId == "SQLI-WIN-008"); + Assert.Equal(RuleOutcome.NotMeasurable, watched.Outcome); + Assert.Contains("Service Control Manager", watched.Reason, StringComparison.Ordinal); + Assert.Contains(await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId), f => f.RuleId == "SQLI-WIN-008"); // still open + var services = (await _repository.GetWindowsServerAsync(row.ServerId))!.Areas.Single(a => a.Area == ServerArea.Services); + Assert.False(services.Ok); + Assert.Contains("grant-windows-server.ps1", services.Detail, StringComparison.Ordinal); + } + + [Fact] + public async Task An_unreachable_server_is_notified_once_and_again_when_it_answers() + { + var row = await AddServerAsync(); + var server = new FakeServer + { + ConnectError = new ServerAccessException(ServerAccessProblem.Unreachable, "aos01.firma.local svarer ikke på WinRM (WS-Man, TCP 5985)."), + }; + var monitor = Monitor(row, server); + var now = DateTimeOffset.UtcNow; + + await monitor.RunCycleAsync(now, CancellationToken.None); + await monitor.RunCycleAsync(now.AddMinutes(16), CancellationToken.None); + server.ConnectError = null; + await monitor.RunCycleAsync(now.AddMinutes(32), CancellationToken.None); + + Assert.Single(_recorder.Received, n => n.Kind == NotificationKind.InstanceUnreachable); + Assert.Single(_recorder.Received, n => n.Kind == NotificationKind.InstanceRecovered); + Assert.Equal(new Uri(WebUrl, $"servere/{row.ServerId}"), _recorder.Received.First(n => n.Kind == NotificationKind.InstanceUnreachable).Url); + var stored = (await _repository.GetWindowsServerAsync(row.ServerId))!; + Assert.True(stored.Reachable); + } + + private SyntheticCheckService Checks(FakeServer server, IBusinessConnectorLocator? locator = null) => new( + Options.Create(new SyntheticCheckOptions()), + new SyntheticCheckRunner(new SqlConnectionFactory(), locator ?? new NoBusinessConnector(), new ProcessRunner(), TimeProvider.System), + server, _repository, _key, TimeProvider.System, NullLogger.Instance, Dispatcher(), + Options.Create(new NotificationOptions { WebUrl = WebUrl }), Options.Create(new AgentOptions()), new RepositoryReadiness()); + + private sealed class NoBusinessConnector : IBusinessConnectorLocator + { + public BusinessConnectorInstallation? Find() => null; + } + + private static int ClosedPort() + { + using var probe = new TcpListener(IPAddress.Loopback, 0); + probe.Start(); + return ((IPEndPoint)probe.LocalEndpoint).Port; + } + + [Fact] + public async Task A_check_that_fails_while_the_service_runs_says_so_after_two_failures() + { + var row = await AddServerAsync(); + var id = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("AOS01 AX", SyntheticCheckKind.AxLogon, row.ServerId, + new SyntheticCheckSettings { Host = "127.0.0.1", Port = ClosedPort(), ServiceName = "AOS60$01" }, null, null, 300, 5), "anna", DateTimeOffset.UtcNow); + var service = Checks(new FakeServer()); + var check = (await _repository.GetSyntheticCheckAsync(id))!; + + var first = await service.RunCheckAsync(check, CancellationToken.None); + Assert.Empty(await _repository.GetServerFindingsAsync(FindingScope.Check, id)); + await service.RunCheckAsync(check, CancellationToken.None); + + Assert.False(first.Success); + Assert.Equal("Running", first.ServiceState); + var finding = Assert.Single(await _repository.GetServerFindingsAsync(FindingScope.Check, id)); + Assert.Equal("AOS01 AX svarer ikke, selv om tjenesten AOS60$01 kører", finding.Title); + Assert.Equal(2, (await _repository.GetSyntheticCheckAsync(id))!.ConsecutiveFailures); + Assert.StartsWith("https://sqlinsight.test/servere/fund/", Assert.Single(_recorder.Received).Url!.ToString(), StringComparison.Ordinal); + } + + [Fact] + public async Task A_sql_check_uses_the_password_sealed_by_the_web_ui_and_only_the_agent_can_open_it() + { + var settings = new SyntheticCheckSettings + { + Server = sql.Server, Authentication = "SqlLogin", UserName = SqlServerFixture.AgentLogin, TrustServerCertificate = true, WarnMilliseconds = 60000, + }; + var key = (await _repository.GetCurrentAgentKeyAsync())!; + var cipher = CredentialEnvelope.Seal(SqlServerFixture.AgentPassword, key.KeyId, key.PublicKey, settings.SecretContext()); + var id = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("SQL01 login", SyntheticCheckKind.Sql, null, settings, cipher, key.KeyId, 300, 15), "anna", + DateTimeOffset.UtcNow); + + var result = await Checks(new FakeServer()).RunCheckAsync((await _repository.GetSyntheticCheckAsync(id))!, CancellationToken.None); + + Assert.True(result.Success, result.Message); + Assert.Equal(RuleOutcome.Passed, (await _repository.GetServerRuleStatesAsync(FindingScope.Check, id)).Single(s => s.RuleId == "SQLI-SYN-001").Outcome); + + // Pointed at another server without typing the password again: it is not sent. + await _repository.UpdateSyntheticCheckAsync(id, new SyntheticCheckInput("SQL01 login", SyntheticCheckKind.Sql, null, settings with { Server = "evil.example" }, null, null, 300, 15), + "mallory", DateTimeOffset.UtcNow); + var refused = await Checks(new FakeServer()).RunCheckAsync((await _repository.GetSyntheticCheckAsync(id))!, CancellationToken.None); + Assert.False(refused.Success); + Assert.Contains("kan ikke bruges", refused.Message, StringComparison.Ordinal); + Assert.NotNull((await _repository.GetSyntheticCheckAsync(id))!.SecretError); + } + + [Fact] + public async Task Web_requests_test_a_server_and_run_a_check_as_the_agent() + { + var row = await AddServerAsync(); + var server = new FakeServer { ServicesError = new ServerAccessException(ServerAccessProblem.AccessDenied, "Adgang nægtet til tjenester.") }; + var id = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("Port", SyntheticCheckKind.Tcp, null, + new SyntheticCheckSettings { Host = "127.0.0.1", Port = ClosedPort() }, null, null, 300, 5), "anna", DateTimeOffset.UtcNow); + var handler = new ServerRequestHandler(_repository, server, Checks(server), TimeProvider.System); + var requests = new AgentRequestService(_repository, new InstanceConfigurationStore(Options.Create(new AgentOptions()), _repository, _key, + NullLogger.Instance), _key, new SqlConnectionFactory(), new RepositoryReadiness(), TimeProvider.System, + NullLogger.Instance, handler); + + var test = await RunAsync(requests, ServerRequestKind.ServerTest, row.ServerId); + var run = await RunAsync(requests, ServerRequestKind.SyntheticCheckRun, id); + var missing = await RunAsync(requests, ServerRequestKind.ServerTest, 4711); + + Assert.Equal(AgentRequestStatus.Done, test.Status); + var result = AgentRequestJson.Deserialize(test.Result)!; + Assert.True(result.Reachable); + Assert.Equal("Adgang nægtet til tjenester.", Assert.Single(result.Missing).Detail); + Assert.Equal(AgentRequestStatus.Done, run.Status); + Assert.False(AgentRequestJson.Deserialize(run.Result)!.Success); + Assert.Equal(1, (await _repository.GetSyntheticCheckAsync(id))!.ConsecutiveFailures); + Assert.Equal(AgentRequestStatus.Failed, missing.Status); + Assert.Contains("findes ikke", missing.Error, StringComparison.Ordinal); + } + + private async Task RunAsync(AgentRequestService service, string kind, int target) + { + var id = await _repository.AddServerRequestAsync(kind, target, "anna", DateTimeOffset.UtcNow); + var claimed = await _repository.ClaimAgentRequestAsync(DateTimeOffset.UtcNow); + Assert.Equal(id, claimed!.RequestId); + await service.HandleAsync(claimed, CancellationToken.None); + return (await _repository.GetAgentRequestAsync(id))!; + } +} diff --git a/tests/SQLInsight.Servers.Tests/LocalCimTests.cs b/tests/SQLInsight.Servers.Tests/LocalCimTests.cs index 4626b36..2d748eb 100644 --- a/tests/SQLInsight.Servers.Tests/LocalCimTests.cs +++ b/tests/SQLInsight.Servers.Tests/LocalCimTests.cs @@ -87,7 +87,7 @@ public class LocalCimTests var source = new CimWindowsServerSource(new ServerConnectionOptions { OperationTimeout = TimeSpan.FromSeconds(5) }); var error = await Assert.ThrowsAsync(() => source.ConnectAsync("sqlinsight-findes-ikke.invalid", CancellationToken.None)); - Assert.Equal(ServerAccessProblem.Unreachable, error.Problem); + Assert.True(error.Problem is ServerAccessProblem.Unreachable or ServerAccessProblem.Timeout, error.Message); Assert.Contains("sqlinsight-findes-ikke.invalid", error.Message, StringComparison.Ordinal); } }