Agent: monitor Windows servers and run synthetic checks; server tests and check runs on request

- ServerMonitoringService keeps the SQL Server hosts in step with the monitored
  instances (SQLInsight:Servers:IncludeSqlHosts), picks up servers added, paused or
  changed in SQLInsight Web within a minute, and runs each server's due areas with
  a limit on parallelism. Default intervals: 1 min CPU/memory, 15 min disks and
  tasks, 5 min services and events (SQLInsight:Servers:Intervals).
- ServerMonitor: one CIM session per cycle; per-minute samples, disk samples and
  the "fuld om ca. N dage" forecast from the stored history, grouped events (read
  from where the last read stopped), services and tasks. A missing right makes
  only that area "ikke målbar" with the exact reason, logged once per change; a
  server that does not answer is notified once, and again when it answers.
  Survives an agent restart (samples, boot time, event position).
- SyntheticCheckService runs checks on their interval, opens a SQL login password
  sealed for the agent (AgentCredentialKey.OpenContext), looks up the linked
  Windows service when a check fails, stores results and findings, notifies with
  links to /servere/fund/{id}.
- AgentRequestService hands ServerTest and SyntheticCheckRun to ServerRequestHandler.
- appsettings.json: SQLInsight:Servers and SQLInsight:SyntheticChecks with every
  threshold, interval and retention.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/SQLInsight.Agent/Configuration/AgentCredentialKey.cs

@@ -87,6 +87,9 @@ public sealed partial class AgentCredentialKey : IDisposable
     public string Open(byte[] envelope, string server, string? userName) =>
         CredentialEnvelope.Open(envelope, KeyId, _rsa, CredentialEnvelope.Context(server, userName));
 
+    /// <summary>Opens a secret sealed for this agent and the given context (e.g. a synthetic check's server and login).</summary>
+    public string OpenContext(byte[] envelope, string context) => CredentialEnvelope.Open(envelope, KeyId, _rsa, context);
+
     /// <summary>What the agent publishes in insight.AgentKeys: the key, and the account and computer it runs as.</summary>
     public AgentKeyRow Describe(DateTimeOffset now) => new()
     {

src/SQLInsight.Agent/Configuration/AgentRequestService.cs

@@ -31,7 +31,8 @@ public sealed partial class AgentRequestService(
     ISqlConnectionFactory connectionFactory,
     RepositoryReadiness readiness,
     TimeProvider timeProvider,
-    ILogger<AgentRequestService> logger) : BackgroundService
+    ILogger<AgentRequestService> logger,
+    Servers.ServerRequestHandler? serverRequests = null) : BackgroundService
 {
     /// <summary>How often the queue is looked at. Short, so the web UI gets an answer within seconds.</summary>
     public static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(2);
@@ -97,6 +98,7 @@ public sealed partial class AgentRequestService(
                 AgentRequestKind.DatabaseDiscovery => AgentRequestJson.Serialize(await DiscoverAsync(Require(request), timeout.Token).ConfigureAwait(false)),
                 AgentRequestKind.InstanceDiscovery => AgentRequestJson.Serialize(await InstanceFinder.FindAsync(
                     AgentRequestJson.Deserialize<InstanceDiscoveryParameters>(request.Parameters)?.Host, TimeSpan.FromSeconds(3), timeout.Token).ConfigureAwait(false)),
+                _ when serverRequests is not null && Servers.ServerRequestHandler.Handles(request.Kind) => await serverRequests.HandleAsync(request, timeout.Token).ConfigureAwait(false),
                 _ => throw new InvalidOperationException($"Ukendt anmodning: {request.Kind}."),
             };
         }

src/SQLInsight.Agent/Program.cs

@@ -116,6 +116,8 @@ builder.Services.AddSingleton<SQLInsight.Agent.Configuration.RepositoryReadiness
 builder.Services.AddSingleton<SQLInsight.Agent.Configuration.InstanceConfigurationStore>();
 builder.Services.AddHostedService<SQLInsight.Agent.Configuration.AgentRequestService>();
 builder.Services.AddHostedService<MonitoringService>();
+// 1.5: Windows servers (CIM, as the agent's identity) and synthetic checks.
+SQLInsight.Agent.Servers.ServerServices.Add(builder.Services, builder.Configuration);
 
 var host = builder.Build();
 await host.RunAsync();

src/SQLInsight.Agent/SQLInsight.Agent.csproj

@@ -14,6 +14,7 @@
     <ProjectReference Include="..\SQLInsight.Licensing\SQLInsight.Licensing.csproj" />
     <ProjectReference Include="..\SQLInsight.Updates\SQLInsight.Updates.csproj" />
     <ProjectReference Include="..\SQLInsight.Reports\SQLInsight.Reports.csproj" />
+    <ProjectReference Include="..\SQLInsight.Servers\SQLInsight.Servers.csproj" />
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Hosting" />

src/SQLInsight.Agent/Servers/ServerMonitor.cs

@@ -0,0 +1,359 @@
+using SQLInsight.Core.Collection;
+using SQLInsight.Core.Servers;
+using SQLInsight.Notifications;
+using SQLInsight.Repository;
+using SQLInsight.Rules.Engine;
+using SQLInsight.Rules.Servers;
+using SQLInsight.Servers;
+
+namespace SQLInsight.Agent.Servers;
+
+/// <summary>
+/// Looks at one Windows server: reads the areas that are due over one CIM session, keeps the latest data for the rules,
+/// evaluates them and stores findings. Only reads; a missing right makes that area "ikke målbar" with the exact reason
+/// (logged once per change, never every round). Not thread-safe; one cycle at a time.
+/// </summary>
+public sealed partial class ServerMonitor(
+    WindowsServerRow server,
+    IWindowsServerSource source,
+    InsightRepository repository,
+    ServerOptions options,
+    ILogger logger,
+    NotificationDispatcher? notifications = null,
+    Uri? webUrl = null)
+{
+    /// <summary>Rules are evaluated at least this often even when only performance was collected.</summary>
+    public static readonly TimeSpan EvaluationInterval = TimeSpan.FromMinutes(5);
+
+    private readonly CollectedData _data = new();
+    private readonly Dictionary<string, DateTimeOffset> _nextDue = new(StringComparer.Ordinal);
+    private readonly Dictionary<string, ServerAreaStatus> _areas = new(StringComparer.Ordinal);
+    private readonly Dictionary<string, string> _reported = new(StringComparer.Ordinal);
+    private ServerPerformanceRaw? _previousRaw;
+    private IReadOnlyList<ServerPerformanceSample> _samples = [];
+    private DateTimeOffset? _lastBoot;
+    private DateTimeOffset? _restartSeenAt;
+    private ServerEventSnapshot? _events;
+    private DateTimeOffset _nextEvaluation = DateTimeOffset.MinValue;
+    private bool _seeded;
+    private bool? _wasReachable;
+
+    public WindowsServerRow Server { get; private set; } = server;
+
+    public int ServerId => Server.ServerId;
+
+    /// <summary>The configuration changed in SQLInsight Web (role, watch lists): used from the next cycle.</summary>
+    public void Update(WindowsServerRow row) => Server = row;
+
+    public bool IsDue(DateTimeOffset now) => ServerArea.All.Any(a => !_nextDue.TryGetValue(a, out var due) || due <= now);
+
+    public async Task RunCycleAsync(DateTimeOffset now, CancellationToken cancellationToken)
+    {
+        var due = ServerArea.All.Where(a => !_nextDue.TryGetValue(a, out var at) || at <= now).ToList();
+        if (due.Count == 0)
+        {
+            return;
+        }
+
+        foreach (var area in due)
+        {
+            _nextDue[area] = now + options.IntervalOf(area);
+        }
+
+        if (!_seeded)
+        {
+            await SeedAsync(now, cancellationToken).ConfigureAwait(false);
+            _seeded = true;
+        }
+
+        IWindowsServerSession session;
+        try
+        {
+            session = await source.ConnectAsync(Server.Host, cancellationToken).ConfigureAwait(false);
+        }
+        catch (ServerAccessException ex)
+        {
+            await NotReachableAsync(ex, due, now, cancellationToken).ConfigureAwait(false);
+            return;
+        }
+
+        using (session)
+        {
+            if (_wasReachable == false)
+            {
+                await NotifyAsync([NotificationFactory.Recovered(Server.Name, webUrl, now) with { Url = ServerNotifications.ServerPage(webUrl, ServerId) }], cancellationToken)
+                    .ConfigureAwait(false);
+            }
+
+            _wasReachable = true;
+            _reported.Remove("connect");
+            foreach (var area in due)
+            {
+                await CollectAsync(session, area, now, cancellationToken).ConfigureAwait(false);
+            }
+
+            if (due.Any(a => a != ServerArea.Performance) || now >= _nextEvaluation)
+            {
+                _nextEvaluation = now + EvaluationInterval;
+                await EvaluateAsync(now, cancellationToken).ConfigureAwait(false);
+            }
+        }
+    }
+
+    /// <summary>Unreachable: recorded and notified once. Refused by rights at the door: every area is "ikke målbar" with the reason.</summary>
+    private async Task NotReachableAsync(ServerAccessException ex, IReadOnlyList<string> due, DateTimeOffset now, CancellationToken cancellationToken)
+    {
+        if (Report("connect", ex.Message))
+        {
+            LogNotReachable(logger, Server.Name, ex.Message);
+        }
+
+        await repository.RecordServerErrorAsync(ServerId, ex.Message, now, cancellationToken).ConfigureAwait(false);
+        if (ex.Problem is ServerAccessProblem.AccessDenied or ServerAccessProblem.NotSupported)
+        {
+            foreach (var area in ServerArea.All)
+            {
+                Unavailable(area, ex.Message);
+            }
+
+            await repository.RecordServerAreasAsync(ServerId, _areas.Values.ToList(), cancellationToken).ConfigureAwait(false);
+            if (due.Any(a => a != ServerArea.Performance) || now >= _nextEvaluation)
+            {
+                _nextEvaluation = now + EvaluationInterval;
+                await EvaluateAsync(now, cancellationToken).ConfigureAwait(false);
+            }
+
+            return;
+        }
+
+        if (_wasReachable != false)
+        {
+            await NotifyAsync([NotificationFactory.Unreachable(Server.Name, ex.Message, webUrl, now) with { Url = ServerNotifications.ServerPage(webUrl, ServerId) }],
+                cancellationToken).ConfigureAwait(false);
+        }
+
+        _wasReachable = false;
+    }
+
+    private async Task CollectAsync(IWindowsServerSession session, string area, DateTimeOffset now, CancellationToken cancellationToken)
+    {
+        try
+        {
+            var detail = area switch
+            {
+                ServerArea.Performance => await CollectPerformanceAsync(session, now, cancellationToken).ConfigureAwait(false),
+                ServerArea.Disks => await CollectDisksAsync(session, now, cancellationToken).ConfigureAwait(false),
+                ServerArea.Services => await CollectServicesAsync(session, now, cancellationToken).ConfigureAwait(false),
+                ServerArea.Events => await CollectEventsAsync(session, now, cancellationToken).ConfigureAwait(false),
+                ServerArea.Tasks => await CollectTasksAsync(session, now, cancellationToken).ConfigureAwait(false),
+                _ => null,
+            };
+            _areas[area] = new ServerAreaStatus(area, true, detail, ServerArea.Right(area));
+            if (_reported.Remove(area))
+            {
+                LogAreaRecovered(logger, Server.Name, area);
+            }
+        }
+        catch (ServerAccessException ex)
+        {
+            Unavailable(area, ex.Message);
+            if (Report(area, ex.Message))
+            {
+                LogAreaUnavailable(logger, Server.Name, area, ex.Message);
+            }
+        }
+    }
+
+    private void Unavailable(string area, string reason)
+    {
+        _areas[area] = new ServerAreaStatus(area, false, reason, ServerArea.Right(area));
+        switch (area)
+        {
+            case ServerArea.Performance:
+                _data.MarkUnavailable<ServerPerformanceSnapshot>(reason);
+                break;
+            case ServerArea.Disks:
+                _data.MarkUnavailable<ServerDiskSnapshot>(reason);
+                break;
+            case ServerArea.Services:
+                _data.MarkUnavailable<ServerServiceSnapshot>(reason);
+                break;
+            case ServerArea.Events:
+                _data.MarkUnavailable<ServerEventSnapshot>(reason);
+                break;
+            case ServerArea.Tasks:
+                _data.MarkUnavailable<ServerTaskSnapshot>(reason);
+                break;
+        }
+    }
+
+    /// <summary>Whether a problem is new (or changed) and should be logged: the same problem every minute is not news.</summary>
+    private bool Report(string key, string message)
+    {
+        if (_reported.TryGetValue(key, out var previous) && previous == message)
+        {
+            return false;
+        }
+
+        _reported[key] = message;
+        return true;
+    }
+
+    public async Task<string?> CollectPerformanceAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(session);
+        var os = await session.GetOperatingSystemAsync(cancellationToken).ConfigureAwait(false);
+        var raw = await session.GetPerformanceAsync(now, cancellationToken).ConfigureAwait(false);
+        DateTimeOffset? previousBoot = null;
+        if (os.LastBootUpTime is { } boot && _lastBoot is { } before && Math.Abs((boot - before).TotalMinutes) > 2)
+        {
+            // Restarted since the last look: the raw counters started over too.
+            previousBoot = before;
+            _restartSeenAt = now;
+            _previousRaw = null;
+        }
+
+        _lastBoot = os.LastBootUpTime ?? _lastBoot;
+        var sample = ServerPerformanceMath.Sample(_previousRaw, raw, os);
+        _previousRaw = raw;
+        _samples = ServerPerformanceMath.Append(_samples, sample);
+        var snapshot = new ServerPerformanceSnapshot
+        {
+            CapturedAt = now,
+            Os = os,
+            Raw = raw,
+            Samples = _samples,
+            PreviousBootUpTime = previousBoot,
+            RestartSeenAt = _restartSeenAt,
+            Protocol = session.Protocol,
+        };
+        _data.Set(snapshot);
+        await repository.SaveServerPerformanceAsync(ServerId, sample, cancellationToken).ConfigureAwait(false);
+        await repository.RecordServerContactAsync(ServerId, os, session.Protocol, null, now, cancellationToken).ConfigureAwait(false);
+        await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Performance, snapshot with { Samples = [] }, now, cancellationToken).ConfigureAwait(false);
+        return sample.CpuPercent is { } cpu ? $"CPU {cpu:0} %" : null;
+    }
+
+    public async Task<string?> CollectDisksAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(session);
+        var disks = await session.GetDisksAsync(cancellationToken).ConfigureAwait(false);
+        await repository.SaveServerDisksAsync(ServerId, now, disks, cancellationToken).ConfigureAwait(false);
+        var historyDays = Math.Max(1, options.Disks.ForecastHistoryDays);
+        var history = await repository.GetServerDiskHistoryAsync(ServerId, now.AddDays(-historyDays), cancellationToken).ConfigureAwait(false);
+        var settings = new DiskForecastSettings { HistoryDays = historyDays };
+        var forecasts = disks.Select(d => DiskForecaster.Forecast(d.Volume, history.TryGetValue(d.Volume, out var points) ? points : [], now, settings)).ToList();
+        var snapshot = new ServerDiskSnapshot { CapturedAt = now, Disks = disks, Forecasts = forecasts };
+        _data.Set(snapshot);
+        await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Disks, snapshot, now, cancellationToken).ConfigureAwait(false);
+        return $"{disks.Count} drev";
+    }
+
+    public async Task<string?> CollectServicesAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(session);
+        var services = await session.GetServicesAsync(cancellationToken).ConfigureAwait(false);
+        var snapshot = new ServerServiceSnapshot { CapturedAt = now, Services = services };
+        _data.Set(snapshot);
+        await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Services, snapshot, now, cancellationToken).ConfigureAwait(false);
+        return $"{services.Count} tjenester";
+    }
+
+    /// <summary>The first time, the last day is read, so recent problems show at once; then from where the last read stopped.</summary>
+    public async Task<string?> CollectEventsAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(session);
+        var since = _events?.ReadUntil ?? now - ServerEventAnalysis.RecentWindow;
+        var read = await session.GetEventsAsync(since, Math.Max(10, options.EventLog.MaxEventsPerCollection), cancellationToken).ConfigureAwait(false);
+        var (groups, ignored) = ServerEventAnalysis.Group(read.Entries, ServerEventAnalysis.DefaultIgnore.Concat(options.EventLog.Ignore));
+
+        // Strictly after the newest event read; without events, keep ten minutes' margin for events written late.
+        var readUntil = read.Entries.Count > 0 ? read.Entries.Max(e => e.At) : Max(since, now.AddMinutes(-10));
+        var snapshot = new ServerEventSnapshot
+        {
+            CapturedAt = now,
+            ReadUntil = readUntil,
+            Method = read.Method,
+            Groups = groups,
+            Recent = ServerEventAnalysis.Recent(_events?.Recent ?? [], groups, now),
+            Ignored = ignored,
+            Coverage = read.Truncated
+                ? $"Der var flere end {options.EventLog.MaxEventsPerCollection} hændelser i en log siden sidst; resten læses ved næste indsamling."
+                : read.Note,
+        };
+        _events = snapshot;
+        _data.Set(snapshot);
+        await repository.SaveServerEventsAsync(ServerId, snapshot, cancellationToken).ConfigureAwait(false);
+        await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Events, snapshot with { Groups = [] }, now, cancellationToken).ConfigureAwait(false);
+        return $"{read.Entries.Count} hændelser ({read.Method})";
+
+        static DateTimeOffset Max(DateTimeOffset a, DateTimeOffset b) => a > b ? a : b;
+    }
+
+    public async Task<string?> CollectTasksAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(session);
+        var tasks = options.Tasks;
+        var read = await session.GetTasksAsync(
+            name => (tasks.IncludeMicrosoftTasks || !TaskResults.IsMicrosoftTask(name)) && !WindowsServerRulesAccess.Matches(name, tasks.Ignore),
+            Math.Max(1, tasks.MaxWatched), cancellationToken).ConfigureAwait(false);
+        var snapshot = new ServerTaskSnapshot { CapturedAt = now, Tasks = read.Watched, Visible = read.Visible, Coverage = read.Note };
+        _data.Set(snapshot);
+        await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Tasks, snapshot, now, cancellationToken).ConfigureAwait(false);
+        return $"{read.Watched.Count} af {read.Visible} opgaver overvåges";
+    }
+
+    private async Task EvaluateAsync(DateTimeOffset now, CancellationToken cancellationToken)
+    {
+        await repository.RecordServerAreasAsync(ServerId, ServerArea.All.Where(_areas.ContainsKey).Select(a => _areas[a]).ToList(), cancellationToken).ConfigureAwait(false);
+        var context = new ServerRuleContext(Server.ToIdentity(), now, _data, options) { PerformanceInterval = options.IntervalOf(ServerArea.Performance) };
+        var results = ScopedRuleEngine.Evaluate(WindowsServerRules.Create(), context);
+        var changes = await repository.SaveScopedRuleResultsAsync(FindingScope.Server, ServerId, results, now, cancellationToken).ConfigureAwait(false);
+        await NotifyAsync(ServerNotifications.For(Server.Name, changes, webUrl, now).ToList(), cancellationToken).ConfigureAwait(false);
+        if (logger.IsEnabled(LogLevel.Debug))
+        {
+            var findings = results.Sum(r => r.Findings.Count);
+            var notMeasurable = results.Count(r => r.Outcome is RuleOutcome.NotMeasurable or RuleOutcome.Error);
+            LogCycle(logger, Server.Name, findings, notMeasurable);
+        }
+    }
+
+    /// <summary>What a restart of the agent would lose: the last hour's samples, the boot time and where the event log was read to.</summary>
+    private async Task SeedAsync(DateTimeOffset now, CancellationToken cancellationToken)
+    {
+        var stored = await repository.GetServerPerformanceAsync(ServerId, now - ServerPerformanceMath.Window, cancellationToken).ConfigureAwait(false);
+        _samples = stored.Select(s => new ServerPerformanceSample(new DateTimeOffset(DateTime.SpecifyKind(s.SampledAt, DateTimeKind.Utc)), s.CpuPercent, s.AvailableMb,
+            s.AvailableMb is { } mb && s.AvailablePct is > 0 ? mb * 100 / s.AvailablePct.Value : null, s.PagesPerSec, s.CommitPct)).ToList();
+        var performance = await repository.GetServerSnapshotAsync<ServerPerformanceSnapshot>(ServerId, ServerArea.Performance, cancellationToken).ConfigureAwait(false);
+        _lastBoot = performance?.Os.LastBootUpTime;
+        _restartSeenAt = performance?.RestartSeenAt;
+        _events = await repository.GetServerSnapshotAsync<ServerEventSnapshot>(ServerId, ServerArea.Events, cancellationToken).ConfigureAwait(false);
+    }
+
+    private async Task NotifyAsync(List<Notification> events, CancellationToken cancellationToken)
+    {
+        if (notifications is not null && events.Count > 0)
+        {
+            await notifications.DispatchAsync(events, cancellationToken).ConfigureAwait(false);
+        }
+    }
+
+    [LoggerMessage(Level = LogLevel.Warning, Message = "Windows server {Server} cannot be read: {Reason}")]
+    private static partial void LogNotReachable(ILogger logger, string server, string reason);
+
+    [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: {Area} is not measurable: {Reason}")]
+    private static partial void LogAreaUnavailable(ILogger logger, string server, string area, string reason);
+
+    [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: {Area} can be read again")]
+    private static partial void LogAreaRecovered(ILogger logger, string server, string area);
+
+    [LoggerMessage(Level = LogLevel.Debug, Message = "Windows server {Server}: {Findings} findings, {NotMeasurable} rules not measurable")]
+    private static partial void LogCycle(ILogger logger, string server, int findings, int notMeasurable);
+}
+
+/// <summary>The rules' name matching (names, or prefixes ending in *), for the agent's task filter.</summary>
+internal static class WindowsServerRulesAccess
+{
+    public static bool Matches(string name, IEnumerable<string> patterns) => patterns.Any(p =>
+        p.EndsWith('*') ? name.StartsWith(p[..^1], StringComparison.OrdinalIgnoreCase) : name.Equals(p, StringComparison.OrdinalIgnoreCase));
+}

src/SQLInsight.Agent/Servers/ServerMonitoringService.cs

@@ -0,0 +1,162 @@
+using Microsoft.Extensions.Options;
+using SQLInsight.Core.Servers;
+using SQLInsight.Notifications;
+using SQLInsight.Repository;
+using SQLInsight.Servers;
+
+namespace SQLInsight.Agent.Servers;
+
+/// <summary>
+/// The agent's loop for Windows servers: keeps the SQL Server hosts in step with the monitored instances, reads the
+/// configured servers (changes in SQLInsight Web apply within a minute, without a restart), runs each server's due
+/// areas with a limit on how many at once, and deletes old samples hourly.
+/// </summary>
+public sealed partial class ServerMonitoringService(
+    IOptions<ServerOptions> options,
+    IOptions<SyntheticCheckOptions> checkOptions,
+    IWindowsServerSource source,
+    InsightRepository repository,
+    TimeProvider time,
+    ILoggerFactory loggerFactory,
+    NotificationDispatcher notifications,
+    IOptions<NotificationOptions> notificationOptions,
+    IOptions<AgentOptions> agentOptions,
+    Configuration.RepositoryReadiness readiness) : BackgroundService
+{
+    public static readonly TimeSpan RefreshInterval = TimeSpan.FromMinutes(1);
+
+    private readonly ILogger _logger = loggerFactory.CreateLogger<ServerMonitoringService>();
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        var settings = options.Value;
+        if (!settings.Enabled)
+        {
+            LogDisabled(_logger);
+            return;
+        }
+
+        await readiness.Ready.WaitAsync(stoppingToken).ConfigureAwait(false);
+        var monitors = new Dictionary<int, ServerMonitor>();
+        var running = new Dictionary<int, Task>();
+        using var throttle = new SemaphoreSlim(Math.Max(1, settings.MaxParallelServers));
+        var nextRefresh = DateTimeOffset.MinValue;
+        var nextMaintenance = time.GetUtcNow().AddMinutes(5);
+        using var timer = new PeriodicTimer(agentOptions.Value.Tick, time);
+        do
+        {
+            var now = time.GetUtcNow();
+            try
+            {
+                if (now >= nextRefresh)
+                {
+                    nextRefresh = now + RefreshInterval;
+                    await RefreshAsync(monitors, settings, now, stoppingToken).ConfigureAwait(false);
+                }
+
+                foreach (var (id, monitor) in monitors)
+                {
+                    if (running.TryGetValue(id, out var task) && !task.IsCompleted || !monitor.IsDue(now))
+                    {
+                        continue;
+                    }
+
+                    running[id] = RunThrottledAsync(monitor, now, throttle, stoppingToken);
+                }
+
+                foreach (var done in running.Where(r => r.Value.IsCompleted && !monitors.ContainsKey(r.Key)).Select(r => r.Key).ToList())
+                {
+                    running.Remove(done);
+                }
+
+                if (now >= nextMaintenance)
+                {
+                    nextMaintenance = now.AddHours(1);
+                    await repository.RunServerMaintenanceAsync(new ServerRetention
+                    {
+                        SampleDays = settings.SampleDays,
+                        DiskHistoryDays = Math.Max(settings.DiskHistoryDays, settings.Disks.ForecastHistoryDays + 1),
+                        EventDays = settings.EventDays,
+                        CheckResultDays = checkOptions.Value.ResultDays,
+                        ResolvedFindingDays = agentOptions.Value.Retention.ResolvedFindingDays,
+                    }, now, stoppingToken).ConfigureAwait(false);
+                }
+            }
+            catch (Exception ex) when (ex is not OperationCanceledException)
+            {
+                // Typically the repository is unreachable. Try again next round.
+                LogRoundFailed(_logger, ex);
+            }
+        }
+        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
+
+        await Task.WhenAll(running.Values).ConfigureAwait(false);
+    }
+
+    /// <summary>The SQL Server hosts first, then the servers to look at (not paused).</summary>
+    private async Task RefreshAsync(Dictionary<int, ServerMonitor> monitors, ServerOptions settings, DateTimeOffset now, CancellationToken cancellationToken)
+    {
+        if (settings.IncludeSqlHosts)
+        {
+            var hosts = (await repository.GetConfiguredInstancesAsync(cancellationToken).ConfigureAwait(false))
+                .Select(i => ServerHost.FromDataSource(i.DataSource, Environment.MachineName).Host)
+                .OfType<string>()
+                .Distinct(StringComparer.OrdinalIgnoreCase)
+                .Select(h => (h, ServerHost.ShortName(h)))
+                .ToList();
+            await repository.SyncSqlHostServersAsync(hosts, now, cancellationToken).ConfigureAwait(false);
+        }
+
+        var servers = (await repository.GetWindowsServersAsync(cancellationToken).ConfigureAwait(false)).Where(s => !s.Paused).ToList();
+        foreach (var row in servers)
+        {
+            if (monitors.TryGetValue(row.ServerId, out var monitor) && string.Equals(monitor.Server.Host, row.Host, StringComparison.OrdinalIgnoreCase))
+            {
+                monitor.Update(row);
+                continue;
+            }
+
+            monitors[row.ServerId] = new ServerMonitor(row, source, repository, settings, loggerFactory.CreateLogger($"SQLInsight.Server.{row.Name}"), notifications,
+                notificationOptions.Value.WebUrl);
+            LogStarted(_logger, row.Name, row.Host);
+        }
+
+        foreach (var gone in monitors.Keys.Where(id => servers.All(s => s.ServerId != id)).ToList())
+        {
+            LogStopped(_logger, monitors[gone].Server.Name);
+            monitors.Remove(gone);
+        }
+    }
+
+    private async Task RunThrottledAsync(ServerMonitor monitor, DateTimeOffset now, SemaphoreSlim throttle, CancellationToken cancellationToken)
+    {
+        await throttle.WaitAsync(cancellationToken).ConfigureAwait(false);
+        try
+        {
+            await monitor.RunCycleAsync(now, cancellationToken).ConfigureAwait(false);
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            LogCycleFailed(_logger, ex, monitor.Server.Name);
+        }
+        finally
+        {
+            throttle.Release();
+        }
+    }
+
+    [LoggerMessage(Level = LogLevel.Information, Message = "Windows servers are not monitored (SQLInsight:Servers:Enabled is false).")]
+    private static partial void LogDisabled(ILogger logger);
+
+    [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server} ({Host}): monitoring started")]
+    private static partial void LogStarted(ILogger logger, string server, string host);
+
+    [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: monitoring stopped (paused or removed)")]
+    private static partial void LogStopped(ILogger logger, string server);
+
+    [LoggerMessage(Level = LogLevel.Error, Message = "Windows server {Server}: cycle failed")]
+    private static partial void LogCycleFailed(ILogger logger, Exception exception, string server);
+
+    [LoggerMessage(Level = LogLevel.Error, Message = "Monitoring Windows servers failed this round; retrying.")]
+    private static partial void LogRoundFailed(ILogger logger, Exception exception);
+}

src/SQLInsight.Agent/Servers/ServerOptions.cs

@@ -0,0 +1,86 @@
+using SQLInsight.Core.Servers;
+using SQLInsight.Notifications;
+using SQLInsight.Repository;
+using SQLInsight.Rules.Servers;
+using SQLInsight.Servers;
+
+namespace SQLInsight.Agent.Servers;
+
+/// <summary>Windows servers (SQLInsight:Servers): how and how often to collect, how long to keep it, and the rules' thresholds.</summary>
+public sealed class ServerOptions : ServerRuleSettings
+{
+    public const string SectionName = "SQLInsight:Servers";
+
+    /// <summary>Look at Windows servers at all.</summary>
+    public bool Enabled { get; set; } = true;
+
+    /// <summary>The computers behind the monitored SQL Server instances are Windows servers too, without being added.</summary>
+    public bool IncludeSqlHosts { get; set; } = true;
+
+    /// <summary>How many servers are read at the same time.</summary>
+    public int MaxParallelServers { get; set; } = 4;
+
+    /// <summary>Auto (WS-Man, then DCOM), WSMan or DCOM.</summary>
+    public string Protocol { get; set; } = "Auto";
+
+    /// <summary>How long one CIM operation may take.</summary>
+    public TimeSpan OperationTimeout { get; set; } = TimeSpan.FromSeconds(30);
+
+    /// <summary>Read event logs with the Event Log service's own protocol (filters on the server); off: CIM only.</summary>
+    public bool EventLogRpc { get; set; } = true;
+
+    /// <summary>Overrides of the collection intervals by area: performance, disks, services, events, tasks.</summary>
+    public IDictionary<string, TimeSpan> Intervals { get; } = new Dictionary<string, TimeSpan>(StringComparer.OrdinalIgnoreCase);
+
+    /// <summary>Per-minute CPU and memory samples.</summary>
+    public int SampleDays { get; set; } = 14;
+
+    /// <summary>Free space per volume: at least the forecast's history.</summary>
+    public int DiskHistoryDays { get; set; } = 35;
+
+    /// <summary>Grouped Windows events.</summary>
+    public int EventDays { get; set; } = 90;
+
+    public TimeSpan IntervalOf(string area) =>
+        Intervals.TryGetValue(area, out var interval) && interval >= TimeSpan.FromSeconds(30) ? interval : ServerArea.DefaultInterval(area);
+
+    public ServerConnectionOptions Connection => new() { Protocol = Protocol, OperationTimeout = OperationTimeout, EventLogRpc = EventLogRpc };
+}
+
+/// <summary>Synthetic checks (SQLInsight:SyntheticChecks): how many run at once, how long results are kept, and the rules' thresholds.</summary>
+public sealed class SyntheticCheckOptions : SyntheticCheckRuleSettings
+{
+    public const string SectionName = "SQLInsight:SyntheticChecks";
+
+    public bool Enabled { get; set; } = true;
+
+    public int MaxParallelChecks { get; set; } = 4;
+
+    /// <summary>Every run's result.</summary>
+    public int ResultDays { get; set; } = 14;
+
+    /// <summary>Where the Dynamics AX .NET Business Connector is, when not where the AX installer puts it.</summary>
+    public string? BusinessConnectorPath { get; set; }
+}
+
+/// <summary>Notifications about Windows servers and synthetic checks, with links to their pages in the web UI.</summary>
+internal static class ServerNotifications
+{
+    public static IEnumerable<Notification> For(string name, FindingChanges changes, Uri? webUrl, DateTimeOffset now)
+    {
+        foreach (var (id, finding) in changes.Opened)
+        {
+            yield return NotificationFactory.Opened(name, id, finding, webUrl, now) with { Url = Link(webUrl, id) };
+        }
+
+        foreach (var resolved in changes.Resolved)
+        {
+            yield return NotificationFactory.Resolved(name, resolved.FindingId, resolved.RuleId, resolved.Title, (Core.Findings.Severity)resolved.Severity, webUrl, now)
+                with { Url = Link(webUrl, resolved.FindingId) };
+        }
+    }
+
+    public static Uri? Link(Uri? webUrl, long findingId) => webUrl is null ? null : new Uri(webUrl, $"servere/fund/{findingId}");
+
+    public static Uri? ServerPage(Uri? webUrl, int serverId) => webUrl is null ? null : new Uri(webUrl, $"servere/{serverId}");
+}

src/SQLInsight.Agent/Servers/ServerRequestHandler.cs

@@ -0,0 +1,51 @@
+using SQLInsight.Core.Configuration;
+using SQLInsight.Core.Servers;
+using SQLInsight.Repository;
+using SQLInsight.Servers;
+
+namespace SQLInsight.Agent.Servers;
+
+/// <summary>
+/// Requests from SQLInsight Web about Windows servers and synthetic checks (insight.AgentRequests with a TargetId):
+/// test a server's rights as the agent, or run a check now. Called by the agent's request service.
+/// </summary>
+public sealed class ServerRequestHandler(InsightRepository repository, IWindowsServerSource source, SyntheticCheckService checks, TimeProvider time)
+{
+    public static bool Handles(string kind) => ServerRequestKind.Handles(kind);
+
+    /// <summary>The request's result as JSON. Throws <see cref="KeyNotFoundException"/> for a server or check that does not exist.</summary>
+    public async Task<string> HandleAsync(AgentRequestRow request, CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(request);
+        var target = await repository.GetRequestTargetAsync(request.RequestId, cancellationToken).ConfigureAwait(false)
+            ?? throw new InvalidOperationException("Anmodningen mangler en server eller et tjek.");
+        return request.Kind switch
+        {
+            ServerRequestKind.ServerTest => AgentRequestJson.Serialize(await TestAsync(target, cancellationToken).ConfigureAwait(false)),
+            ServerRequestKind.SyntheticCheckRun => AgentRequestJson.Serialize(await RunAsync(target, cancellationToken).ConfigureAwait(false)),
+            _ => throw new InvalidOperationException($"Ukendt anmodning: {request.Kind}."),
+        };
+    }
+
+    /// <summary>Connects as the agent and tries every area; what could and could not be read is also stored on the server.</summary>
+    public async Task<WindowsServerTestResult> TestAsync(int serverId, CancellationToken cancellationToken)
+    {
+        var server = await repository.GetWindowsServerAsync(serverId, cancellationToken).ConfigureAwait(false)
+            ?? throw new KeyNotFoundException($"Server {serverId} findes ikke.");
+        var account = OperatingSystem.IsWindows() ? CimWindowsServerSource.AgentAccount(server.Host) : null;
+        var result = await WindowsServerTester.TestAsync(source, server.Host, account, time.GetUtcNow(), cancellationToken).ConfigureAwait(false);
+        if (result.Reachable)
+        {
+            await repository.RecordServerAreasAsync(serverId, result.Areas, cancellationToken).ConfigureAwait(false);
+        }
+
+        return result;
+    }
+
+    public async Task<Core.SyntheticChecks.SyntheticCheckResult> RunAsync(int checkId, CancellationToken cancellationToken)
+    {
+        var check = await repository.GetSyntheticCheckAsync(checkId, cancellationToken).ConfigureAwait(false)
+            ?? throw new KeyNotFoundException($"Tjek {checkId} findes ikke.");
+        return await checks.RunCheckAsync(check, cancellationToken).ConfigureAwait(false);
+    }
+}

src/SQLInsight.Agent/Servers/ServerServices.cs

@@ -0,0 +1,29 @@
+using Microsoft.Extensions.Options;
+using SQLInsight.Connectivity;
+using SQLInsight.Servers;
+using SQLInsight.Servers.Synthetic;
+
+namespace SQLInsight.Agent.Servers;
+
+/// <summary>1.5: Windows servers and synthetic checks in the agent.</summary>
+public static class ServerServices
+{
+    public static void Add(IServiceCollection services, IConfiguration configuration)
+    {
+        ArgumentNullException.ThrowIfNull(services);
+        ArgumentNullException.ThrowIfNull(configuration);
+        services.Configure<ServerOptions>(configuration.GetSection(ServerOptions.SectionName));
+        services.Configure<SyntheticCheckOptions>(configuration.GetSection(SyntheticCheckOptions.SectionName));
+        services.AddSingleton<IWindowsServerSource>(sp => OperatingSystem.IsWindows()
+            ? new CimWindowsServerSource(sp.GetRequiredService<IOptions<ServerOptions>>().Value.Connection)
+            : new UnsupportedWindowsServerSource());
+        services.AddSingleton<IBusinessConnectorLocator>(sp => new BusinessConnectorLocator(sp.GetRequiredService<IOptions<SyntheticCheckOptions>>().Value.BusinessConnectorPath));
+        services.AddSingleton<IProcessRunner, ProcessRunner>();
+        services.AddSingleton(sp => new SyntheticCheckRunner(sp.GetRequiredService<ISqlConnectionFactory>(), sp.GetRequiredService<IBusinessConnectorLocator>(),
+            sp.GetRequiredService<IProcessRunner>(), sp.GetRequiredService<TimeProvider>()));
+        services.AddSingleton<SyntheticCheckService>();
+        services.AddHostedService(sp => sp.GetRequiredService<SyntheticCheckService>());
+        services.AddSingleton<ServerRequestHandler>();
+        services.AddHostedService<ServerMonitoringService>();
+    }
+}

src/SQLInsight.Agent/Servers/SyntheticCheckService.cs

@@ -0,0 +1,222 @@
+using Microsoft.Extensions.Options;
+using SQLInsight.Core.Configuration;
+using SQLInsight.Core.SyntheticChecks;
+using SQLInsight.Notifications;
+using SQLInsight.Repository;
+using SQLInsight.Rules.Servers;
+using SQLInsight.Servers;
+using SQLInsight.Servers.Synthetic;
+
+namespace SQLInsight.Agent.Servers;
+
+/// <summary>
+/// Runs the synthetic checks on their schedule (and on request from SQLInsight Web), as the agent's identity: stores
+/// each result, looks up the linked Windows service when a check fails ("svarer ikke, selv om tjenesten kører"),
+/// evaluates the rules and notifies. Never changes anything on the servers it checks.
+/// </summary>
+public sealed partial class SyntheticCheckService(
+    IOptions<SyntheticCheckOptions> options,
+    SyntheticCheckRunner runner,
+    IWindowsServerSource servers,
+    InsightRepository repository,
+    Configuration.AgentCredentialKey key,
+    TimeProvider time,
+    ILogger<SyntheticCheckService> logger,
+    NotificationDispatcher notifications,
+    IOptions<NotificationOptions> notificationOptions,
+    IOptions<AgentOptions> agentOptions,
+    Configuration.RepositoryReadiness readiness) : BackgroundService
+{
+    private readonly Dictionary<int, Task> _running = [];
+    private readonly Dictionary<int, string> _reported = [];
+    private readonly Lock _lock = new();
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        var settings = options.Value;
+        if (!settings.Enabled)
+        {
+            return;
+        }
+
+        await readiness.Ready.WaitAsync(stoppingToken).ConfigureAwait(false);
+        using var throttle = new SemaphoreSlim(Math.Max(1, settings.MaxParallelChecks));
+        using var timer = new PeriodicTimer(agentOptions.Value.Tick, time);
+        do
+        {
+            try
+            {
+                var now = time.GetUtcNow();
+                foreach (var check in await repository.GetSyntheticChecksAsync(stoppingToken).ConfigureAwait(false))
+                {
+                    if (check.Paused || (check.LastRunAt is { } last && last.AddSeconds(check.IntervalSeconds) > now.UtcDateTime))
+                    {
+                        continue;
+                    }
+
+                    lock (_lock)
+                    {
+                        if (_running.TryGetValue(check.CheckId, out var task) && !task.IsCompleted)
+                        {
+                            continue;
+                        }
+
+                        _running[check.CheckId] = RunThrottledAsync(check, throttle, stoppingToken);
+                    }
+                }
+            }
+            catch (Exception ex) when (ex is not OperationCanceledException)
+            {
+                LogRoundFailed(logger, ex);
+            }
+        }
+        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
+
+        Task[] remaining;
+        lock (_lock)
+        {
+            remaining = [.. _running.Values];
+        }
+
+        await Task.WhenAll(remaining).ConfigureAwait(false);
+    }
+
+    private async Task RunThrottledAsync(SyntheticCheckRow check, SemaphoreSlim throttle, CancellationToken cancellationToken)
+    {
+        await throttle.WaitAsync(cancellationToken).ConfigureAwait(false);
+        try
+        {
+            await RunCheckAsync(check, cancellationToken).ConfigureAwait(false);
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            LogCheckFailed(logger, ex, check.Name);
+        }
+        finally
+        {
+            throttle.Release();
+        }
+    }
+
+    /// <summary>Runs a check once, stores the result and its findings. Used on schedule and for "Kør nu" in SQLInsight Web.</summary>
+    public async Task<SyntheticCheckResult> RunCheckAsync(SyntheticCheckRow check, CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(check);
+        var settings = check.Settings;
+        var now = time.GetUtcNow();
+        string? password = null;
+        if (check.Kind == SyntheticCheckKind.Sql && settings.Authentication == "SqlLogin")
+        {
+            (password, var error) = await OpenSecretAsync(check, settings, cancellationToken).ConfigureAwait(false);
+            if (error is not null)
+            {
+                return await StoreAsync(check, new SyntheticCheckResult { At = now, Success = false, Message = error }, cancellationToken).ConfigureAwait(false);
+            }
+        }
+
+        var timeout = TimeSpan.FromSeconds(Math.Clamp(check.TimeoutSeconds, 1, SyntheticCheckLimits.MaxTimeoutSeconds));
+        var result = await runner.RunAsync(new SyntheticCheckRun(check.Kind, settings, timeout, password), cancellationToken).ConfigureAwait(false);
+        if (!result.Success && check.ServerId is { } serverId && settings.ServiceName is { Length: > 0 } service)
+        {
+            result = await WithServiceStateAsync(result, serverId, service, cancellationToken).ConfigureAwait(false);
+        }
+
+        return await StoreAsync(check, result, cancellationToken).ConfigureAwait(false);
+    }
+
+    private async Task<(string? Password, string? Error)> OpenSecretAsync(SyntheticCheckRow check, SyntheticCheckSettings settings, CancellationToken cancellationToken)
+    {
+        string? error = null;
+        string? password = null;
+        if (check.SecretCipher is not { Length: > 0 } cipher)
+        {
+            error = "Der er ikke gemt en adgangskode til SQL-login'et.";
+        }
+        else
+        {
+            try
+            {
+                password = key.OpenContext(cipher, settings.SecretContext());
+            }
+            catch (CredentialException ex)
+            {
+                error = "Adgangskoden kan ikke bruges: " + ex.Message;
+            }
+        }
+
+        if (!string.Equals(error, check.SecretError, StringComparison.Ordinal))
+        {
+            await repository.SetSyntheticCheckSecretErrorAsync(check.CheckId, error, cancellationToken).ConfigureAwait(false);
+        }
+
+        return (password, error);
+    }
+
+    /// <summary>The state of the Windows service behind a failing check: the difference between "nede" and "kører, men svarer ikke".</summary>
+    private async Task<SyntheticCheckResult> WithServiceStateAsync(SyntheticCheckResult result, int serverId, string service, CancellationToken cancellationToken)
+    {
+        var server = await repository.GetWindowsServerAsync(serverId, cancellationToken).ConfigureAwait(false);
+        if (server is null)
+        {
+            return result;
+        }
+
+        try
+        {
+            using var session = await servers.ConnectAsync(server.Host, cancellationToken).ConfigureAwait(false);
+            var state = await session.GetServiceAsync(service, cancellationToken).ConfigureAwait(false);
+            return result with
+            {
+                ServiceState = state?.State ?? "NotFound",
+                Details = [.. result.Details, new SyntheticCheckDetail("Tjeneste", $"{service} på {server.Name}: {state?.State ?? "findes ikke"}")],
+            };
+        }
+        catch (ServerAccessException ex)
+        {
+            return result with { Details = [.. result.Details, new SyntheticCheckDetail("Tjeneste", $"{service} på {server.Name} kunne ikke aflæses: {ex.Message}")] };
+        }
+    }
+
+    private async Task<SyntheticCheckResult> StoreAsync(SyntheticCheckRow check, SyntheticCheckResult result, CancellationToken cancellationToken)
+    {
+        await repository.RecordSyntheticCheckResultAsync(check.CheckId, result, cancellationToken).ConfigureAwait(false);
+        var recent = (await repository.GetSyntheticCheckResultsAsync(check.CheckId, 10, cancellationToken: cancellationToken).ConfigureAwait(false))
+            .Select(r => r.ToResult()).ToList();
+        var settings = check.Settings;
+        var identity = new SyntheticCheckIdentity(check.CheckId, check.Name, check.Kind, settings.Target(check.Kind))
+        {
+            ServerName = check.ServerName,
+            ServiceName = settings.ServiceName,
+            WarnMilliseconds = settings.WarnMilliseconds,
+            Https = settings.Url?.Scheme == Uri.UriSchemeHttps,
+        };
+        var results = ScopedRuleEngine.Evaluate(SyntheticCheckRules.Create(), new SyntheticCheckRuleContext(identity, result.At, recent, options.Value));
+        var changes = await repository.SaveScopedRuleResultsAsync(FindingScope.Check, check.CheckId, results, result.At, cancellationToken).ConfigureAwait(false);
+        var events = ServerNotifications.For(check.Name, changes, notificationOptions.Value.WebUrl, result.At).ToList();
+        if (events.Count > 0)
+        {
+            await notifications.DispatchAsync(events, cancellationToken).ConfigureAwait(false);
+        }
+
+        lock (_lock)
+        {
+            var state = result.Success ? "ok" : result.Message;
+            if (!_reported.TryGetValue(check.CheckId, out var previous) || previous != state)
+            {
+                _reported[check.CheckId] = state;
+                LogResult(logger, check.Name, result.Success, result.Message);
+            }
+        }
+
+        return result;
+    }
+
+    [LoggerMessage(Level = LogLevel.Information, Message = "Synthetic check {Check}: success {Success}: {Message}")]
+    private static partial void LogResult(ILogger logger, string check, bool success, string message);
+
+    [LoggerMessage(Level = LogLevel.Error, Message = "Synthetic check {Check} could not be run")]
+    private static partial void LogCheckFailed(ILogger logger, Exception exception, string check);
+
+    [LoggerMessage(Level = LogLevel.Error, Message = "Running synthetic checks failed this round; retrying.")]
+    private static partial void LogRoundFailed(ILogger logger, Exception exception);
+}

src/SQLInsight.Agent/appsettings.json

@@ -62,6 +62,35 @@
       "RecId32WarnPercent": 50
       // "LocalTimeZone": "Romance Standard Time"   // AOS'ens tidszone for Axapta 3.0/4.0's lokale tider; tom: agentens egen
     },
+    "Servers": {
+      // Windows-servere (se docs/windows-servere.md). Rettigheder: db/permissions/grant-windows-server.ps1 – aldrig lokal administrator.
+      "Enabled": true,
+      "IncludeSqlHosts": true,
+      "Protocol": "Auto",
+      "MaxParallelServers": 4,
+      "Intervals": { "performance": "00:01:00", "disks": "00:15:00", "services": "00:05:00", "events": "00:05:00", "tasks": "00:15:00" },
+      "Cpu": { "HighPercent": 90, "SustainedMinutes": 15 },
+      "Memory": { "AvailablePercent": 5, "AvailableMegabytes": 1024, "PagesPerSecond": 1000, "CommitPercent": 90 },
+      "Disks": { "FreePercent": 10, "CriticalFreePercent": 5, "EnoughFreeGigabytes": 50, "ForecastDays": 30, "ForecastHistoryDays": 14 },
+      "Services": { "Ignore": [], "StartupGraceMinutes": 15 },
+      "EventLog": { "RepeatThreshold": 5, "Ignore": [], "MaxFindings": 15 },
+      "Tasks": { "IncludeMicrosoftTasks": false, "LateMinutes": 60, "Ignore": [] },
+      "UptimeDays": 0,
+      "SampleDays": 14,
+      "DiskHistoryDays": 35,
+      "EventDays": 90
+    },
+    "SyntheticChecks": {
+      // Syntetiske tjek (se docs/syntetiske-tjek.md). Tjekkene selv oprettes i webbrugerfladen eller API'et.
+      "Enabled": true,
+      "MaxParallelChecks": 4,
+      "FailuresBeforeFinding": 2,
+      "SlowRunsBeforeFinding": 3,
+      "CertificateWarningDays": 21,
+      "CertificateCriticalDays": 7,
+      "ResultDays": 14,
+      "BusinessConnectorPath": null
+    },
     "Instances": [
       // Servere tilføjes nemmest i webbrugerfladen (Servere og databaser). Her kan de også stå, fx (se docs/installation.md):
       // { "Name": "prod-sql01", "Server": "sql01.domæne.local", "Authentication": "Windows", "Provider": "Auto", "Encryption": "Mandatory" }

src/SQLInsight.Rules/Servers/ServerRuleSettings.cs

@@ -84,6 +84,9 @@ public sealed class EventLogThresholds
 
     /// <summary>At most this many event findings per server, most important first.</summary>
     public int MaxFindings { get; set; } = 15;
+
+    /// <summary>At most this many events per log are read in one collection; the rest is read next time.</summary>
+    public int MaxEventsPerCollection { get; set; } = 2000;
 }
 
 public sealed class TaskThresholds
@@ -94,6 +97,9 @@ public sealed class TaskThresholds
     /// <summary>A task whose next run time is more than this many minutes in the past has not run as planned (SQLI-WIN-013).</summary>
     public int LateMinutes { get; set; } = 60;
 
+    /// <summary>At most this many tasks per server are watched (their run information is read one at a time).</summary>
+    public int MaxWatched { get; set; } = 50;
+
     /// <summary>Tasks never watched (path\name, or a prefix ending in *).</summary>
     public IList<string> Ignore { get; } = [];
 }

tests/SQLInsight.IntegrationTests/ServerAgentTests.cs

@@ -0,0 +1,318 @@
+using System.Net;
+using System.Net.Sockets;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using SQLInsight.Agent;
+using SQLInsight.Agent.Configuration;
+using SQLInsight.Agent.Servers;
+using SQLInsight.Connectivity;
+using SQLInsight.Core.Configuration;
+using SQLInsight.Core.Findings;
+using SQLInsight.Core.Servers;
+using SQLInsight.Core.SyntheticChecks;
+using SQLInsight.Notifications;
+using SQLInsight.Repository;
+using SQLInsight.Rules.Engine;
+using SQLInsight.Servers;
+using SQLInsight.Servers.Synthetic;
+
+namespace SQLInsight.IntegrationTests;
+
+/// <summary>
+/// The agent's side of Windows servers and synthetic checks against a real repository, with a fake Windows server (the
+/// CIM implementation itself is tested against this computer in SQLInsight.Servers.Tests).
+/// </summary>
+[Collection(SqlServerTestGroup.Name)]
+[Trait("Category", "Integration")]
+public sealed class ServerAgentTests(SqlServerFixture sql) : IAsyncLifetime
+{
+    private readonly string _database = "SQLInsightSrvAgent_" + Guid.NewGuid().ToString("N")[..8];
+    private readonly string _keys = Path.Combine(Path.GetTempPath(), "sqlinsight-keys-" + Guid.NewGuid().ToString("N"));
+    private InsightRepository _repository = null!;
+    private AgentCredentialKey _key = null!;
+    private readonly Recorder _recorder = new();
+    private static readonly Uri WebUrl = new("https://sqlinsight.test/");
+
+    public async Task InitializeAsync()
+    {
+        await sql.ExecuteAsSaAsync($"CREATE DATABASE [{_database}];");
+        _repository = new InsightRepository(sql.DatabaseConnectionString(_database));
+        await _repository.MigrateAsync();
+        _key = AgentCredentialKey.LoadOrCreate(_keys);
+        await _repository.PublishAgentKeyAsync(_key.Describe(DateTimeOffset.UtcNow));
+    }
+
+    public async Task DisposeAsync()
+    {
+        _key.Dispose();
+        Directory.Delete(_keys, recursive: true);
+        await sql.ExecuteAsSaAsync($"ALTER DATABASE [{_database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{_database}];");
+    }
+
+    private sealed class Recorder : INotificationChannel
+    {
+        public List<Notification> Received { get; } = [];
+        public string Name => "recorder";
+        public Severity MinSeverity => Severity.Info;
+
+        public Task SendAsync(IReadOnlyList<Notification> notifications, CancellationToken cancellationToken)
+        {
+            Received.AddRange(notifications);
+            return Task.CompletedTask;
+        }
+    }
+
+    private NotificationDispatcher Dispatcher() => new([_recorder], new NotificationOptions { MaxMessagesPerHour = 1000 }, TimeProvider.System, NullLogger<NotificationDispatcher>.Instance);
+
+    /// <summary>A Windows server as the tests want it: what each area returns, or the error it throws.</summary>
+    private sealed class FakeServer : IWindowsServerSource, IWindowsServerSession
+    {
+        public ServerAccessException? ConnectError { get; set; }
+        public ServerAccessException? ServicesError { get; set; }
+        public ulong Idle { get; set; }
+        public ulong Clock { get; set; } = 1_000_000_000;
+        public double FreeGb { get; set; } = 3;
+        public string AosState { get; set; } = "Running";
+        public int Connects { get; private set; }
+
+        public string Protocol => "WS-Man";
+        public string? ProtocolNote => null;
+
+        public Task<IWindowsServerSession> ConnectAsync(string host, CancellationToken cancellationToken)
+        {
+            Connects++;
+            return ConnectError is { } error ? throw error : Task.FromResult<IWindowsServerSession>(this);
+        }
+
+        public Task<ServerOsInfo> GetOperatingSystemAsync(CancellationToken cancellationToken) => Task.FromResult(new ServerOsInfo
+        {
+            Caption = "Microsoft Windows Server 2019 Standard", Version = "10.0.17763", TotalMemoryMb = 16384, LogicalProcessors = 4,
+            LastBootUpTime = DateTimeOffset.UtcNow.AddDays(-10), ComputerName = "AOS01",
+        });
+
+        public Task<ServerPerformanceRaw> GetPerformanceAsync(DateTimeOffset now, CancellationToken cancellationToken)
+        {
+            // Each call: one minute passed, 95 % of it busy.
+            Clock += 600_000_000;
+            Idle += 30_000_000;
+            return Task.FromResult(new ServerPerformanceRaw
+            {
+                CapturedAt = now, ProcessorIdle100Ns = Idle, ProcessorTimestamp100Ns = Clock, AvailableMb = 6000, CommittedBytes = 4, CommitLimit = 10,
+            });
+        }
+
+        public Task<IReadOnlyList<ServerDisk>> GetDisksAsync(CancellationToken cancellationToken) =>
+            Task.FromResult<IReadOnlyList<ServerDisk>>([new ServerDisk(@"C:\", "System", "NTFS", 100, 40), new ServerDisk(@"D:\", "Data", "NTFS", 100, FreeGb)]);
+
+        public Task<IReadOnlyList<ServerService>> GetServicesAsync(CancellationToken cancellationToken) => ServicesError is { } error
+            ? throw error
+            : Task.FromResult<IReadOnlyList<ServerService>>([new ServerService("AOS60$01", "Microsoft Dynamics AX Object Server 6.0$01", AosState, "Auto")]);
+
+        public Task<ServerService?> GetServiceAsync(string name, CancellationToken cancellationToken) =>
+            Task.FromResult<ServerService?>(new ServerService(name, name, AosState, "Auto"));
+
+        public Task<ServerEventRead> GetEventsAsync(DateTimeOffset since, int max, CancellationToken cancellationToken) => Task.FromResult(new ServerEventRead(
+            [new ServerEventEntry("System", "disk", 153, 2, DateTimeOffset.UtcNow.AddMinutes(-1), @"The IO operation at logical block address 0x1 for Disk 1 was retried.")],
+            "Event Log (RPC)", false));
+
+        public Task<ServerTaskRead> GetTasksAsync(Func<string, bool> watch, int maxWatched, CancellationToken cancellationToken) => Task.FromResult(new ServerTaskRead(
+            [new ServerTask { Path = @"\", Name = "AX export", State = "Ready", LastTaskResult = 1, LastRunTime = DateTimeOffset.UtcNow.AddHours(-1) }], 140));
+
+        public void Dispose()
+        {
+        }
+    }
+
+    private ServerMonitor Monitor(WindowsServerRow row, FakeServer server) =>
+        new(row, server, _repository, new ServerOptions(), NullLogger.Instance, Dispatcher(), WebUrl);
+
+    private async Task<WindowsServerRow> AddServerAsync(string name = "AOS01", IReadOnlyList<string>? watched = null)
+    {
+        var id = await _repository.AddWindowsServerAsync(new NewWindowsServer(name, name.ToLowerInvariant() + ".firma.local", ServerRoles.Application, watched ?? [], []), "anna",
+            DateTimeOffset.UtcNow);
+        return (await _repository.GetWindowsServerAsync(id))!;
+    }
+
+    [Fact]
+    public async Task A_cycle_reads_every_area_stores_it_and_opens_findings_with_links_to_the_server_pages()
+    {
+        var row = await AddServerAsync();
+        var server = new FakeServer();
+        var monitor = Monitor(row, server);
+        var now = DateTimeOffset.UtcNow;
+
+        await monitor.RunCycleAsync(now, CancellationToken.None);
+
+        var findings = await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId);
+        Assert.Contains(findings, f => f.RuleId == "SQLI-WIN-005" && f.Subject == @"disk: D:\");      // 3 GB free
+        Assert.Contains(findings, f => f.RuleId == "SQLI-WIN-009" && f.Subject == "event: System/disk/153");
+        Assert.Contains(findings, f => f.RuleId == "SQLI-WIN-012" && f.Subject == @"task: \AX export");
+        var stored = (await _repository.GetWindowsServerAsync(row.ServerId))!;
+        Assert.Equal(("WS-Man", "Microsoft Windows Server 2019 Standard", 4), (stored.Protocol, stored.OperatingSystem, stored.LogicalProcessors));
+        Assert.All(stored.Areas, a => Assert.True(a.Ok, a.Detail));
+        Assert.Equal(ServerArea.All.Count, stored.Areas.Count);
+        Assert.NotNull(await _repository.GetServerSnapshotAsync<ServerDiskSnapshot>(row.ServerId, ServerArea.Disks));
+        Assert.Single(await _repository.GetServerEventsAsync(row.ServerId, now.AddDays(-1), now));
+        var opened = _recorder.Received.Where(n => n.Kind == NotificationKind.FindingOpened).ToList();
+        Assert.Equal(findings.Count, opened.Count);
+        Assert.All(opened, n => Assert.StartsWith("https://sqlinsight.test/servere/fund/", n.Url!.ToString(), StringComparison.Ordinal));
+        Assert.All(opened, n => Assert.Equal("AOS01", n.Instance));
+
+        // The CPU rule needs a quarter of an hour of minutes: the samples accumulate.
+        for (var minute = 1; minute <= 16; minute++)
+        {
+            await monitor.RunCycleAsync(now.AddMinutes(minute), CancellationToken.None);
+        }
+
+        var cpu = Assert.Single(await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId), f => f.RuleId == "SQLI-WIN-001");
+        Assert.Contains("95 %", cpu.Title, StringComparison.Ordinal);
+        Assert.Equal(17, (await _repository.GetServerPerformanceAsync(row.ServerId, now.AddHours(-1))).Count);
+    }
+
+    [Fact]
+    public async Task A_missing_right_is_not_measurable_with_the_reason_and_logged_findings_stay_open()
+    {
+        var row = await AddServerAsync(watched: ["AOS60$01"]);
+        var server = new FakeServer { AosState = "Stopped" };
+        var monitor = Monitor(row, server);
+        var now = DateTimeOffset.UtcNow;
+        await monitor.RunCycleAsync(now, CancellationToken.None);
+        Assert.Contains(await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId), f => f.RuleId == "SQLI-WIN-008");
+
+        server.ServicesError = ServerErrors.Describe(ServerErrors.AccessDenied, "AccessDenied", "Access is denied.", ServerArea.Services, "aos01", @"FIRMA\AGENT01$", "WS-Man");
+        await monitor.RunCycleAsync(now.AddMinutes(6), CancellationToken.None);
+
+        var states = await _repository.GetServerRuleStatesAsync(FindingScope.Server, row.ServerId);
+        var watched = states.Single(s => s.RuleId == "SQLI-WIN-008");
+        Assert.Equal(RuleOutcome.NotMeasurable, watched.Outcome);
+        Assert.Contains("Service Control Manager", watched.Reason, StringComparison.Ordinal);
+        Assert.Contains(await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId), f => f.RuleId == "SQLI-WIN-008"); // still open
+        var services = (await _repository.GetWindowsServerAsync(row.ServerId))!.Areas.Single(a => a.Area == ServerArea.Services);
+        Assert.False(services.Ok);
+        Assert.Contains("grant-windows-server.ps1", services.Detail, StringComparison.Ordinal);
+    }
+
+    [Fact]
+    public async Task An_unreachable_server_is_notified_once_and_again_when_it_answers()
+    {
+        var row = await AddServerAsync();
+        var server = new FakeServer
+        {
+            ConnectError = new ServerAccessException(ServerAccessProblem.Unreachable, "aos01.firma.local svarer ikke på WinRM (WS-Man, TCP 5985)."),
+        };
+        var monitor = Monitor(row, server);
+        var now = DateTimeOffset.UtcNow;
+
+        await monitor.RunCycleAsync(now, CancellationToken.None);
+        await monitor.RunCycleAsync(now.AddMinutes(16), CancellationToken.None);
+        server.ConnectError = null;
+        await monitor.RunCycleAsync(now.AddMinutes(32), CancellationToken.None);
+
+        Assert.Single(_recorder.Received, n => n.Kind == NotificationKind.InstanceUnreachable);
+        Assert.Single(_recorder.Received, n => n.Kind == NotificationKind.InstanceRecovered);
+        Assert.Equal(new Uri(WebUrl, $"servere/{row.ServerId}"), _recorder.Received.First(n => n.Kind == NotificationKind.InstanceUnreachable).Url);
+        var stored = (await _repository.GetWindowsServerAsync(row.ServerId))!;
+        Assert.True(stored.Reachable);
+    }
+
+    private SyntheticCheckService Checks(FakeServer server, IBusinessConnectorLocator? locator = null) => new(
+        Options.Create(new SyntheticCheckOptions()),
+        new SyntheticCheckRunner(new SqlConnectionFactory(), locator ?? new NoBusinessConnector(), new ProcessRunner(), TimeProvider.System),
+        server, _repository, _key, TimeProvider.System, NullLogger<SyntheticCheckService>.Instance, Dispatcher(),
+        Options.Create(new NotificationOptions { WebUrl = WebUrl }), Options.Create(new AgentOptions()), new RepositoryReadiness());
+
+    private sealed class NoBusinessConnector : IBusinessConnectorLocator
+    {
+        public BusinessConnectorInstallation? Find() => null;
+    }
+
+    private static int ClosedPort()
+    {
+        using var probe = new TcpListener(IPAddress.Loopback, 0);
+        probe.Start();
+        return ((IPEndPoint)probe.LocalEndpoint).Port;
+    }
+
+    [Fact]
+    public async Task A_check_that_fails_while_the_service_runs_says_so_after_two_failures()
+    {
+        var row = await AddServerAsync();
+        var id = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("AOS01 AX", SyntheticCheckKind.AxLogon, row.ServerId,
+            new SyntheticCheckSettings { Host = "127.0.0.1", Port = ClosedPort(), ServiceName = "AOS60$01" }, null, null, 300, 5), "anna", DateTimeOffset.UtcNow);
+        var service = Checks(new FakeServer());
+        var check = (await _repository.GetSyntheticCheckAsync(id))!;
+
+        var first = await service.RunCheckAsync(check, CancellationToken.None);
+        Assert.Empty(await _repository.GetServerFindingsAsync(FindingScope.Check, id));
+        await service.RunCheckAsync(check, CancellationToken.None);
+
+        Assert.False(first.Success);
+        Assert.Equal("Running", first.ServiceState);
+        var finding = Assert.Single(await _repository.GetServerFindingsAsync(FindingScope.Check, id));
+        Assert.Equal("AOS01 AX svarer ikke, selv om tjenesten AOS60$01 kører", finding.Title);
+        Assert.Equal(2, (await _repository.GetSyntheticCheckAsync(id))!.ConsecutiveFailures);
+        Assert.StartsWith("https://sqlinsight.test/servere/fund/", Assert.Single(_recorder.Received).Url!.ToString(), StringComparison.Ordinal);
+    }
+
+    [Fact]
+    public async Task A_sql_check_uses_the_password_sealed_by_the_web_ui_and_only_the_agent_can_open_it()
+    {
+        var settings = new SyntheticCheckSettings
+        {
+            Server = sql.Server, Authentication = "SqlLogin", UserName = SqlServerFixture.AgentLogin, TrustServerCertificate = true, WarnMilliseconds = 60000,
+        };
+        var key = (await _repository.GetCurrentAgentKeyAsync())!;
+        var cipher = CredentialEnvelope.Seal(SqlServerFixture.AgentPassword, key.KeyId, key.PublicKey, settings.SecretContext());
+        var id = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("SQL01 login", SyntheticCheckKind.Sql, null, settings, cipher, key.KeyId, 300, 15), "anna",
+            DateTimeOffset.UtcNow);
+
+        var result = await Checks(new FakeServer()).RunCheckAsync((await _repository.GetSyntheticCheckAsync(id))!, CancellationToken.None);
+
+        Assert.True(result.Success, result.Message);
+        Assert.Equal(RuleOutcome.Passed, (await _repository.GetServerRuleStatesAsync(FindingScope.Check, id)).Single(s => s.RuleId == "SQLI-SYN-001").Outcome);
+
+        // Pointed at another server without typing the password again: it is not sent.
+        await _repository.UpdateSyntheticCheckAsync(id, new SyntheticCheckInput("SQL01 login", SyntheticCheckKind.Sql, null, settings with { Server = "evil.example" }, null, null, 300, 15),
+            "mallory", DateTimeOffset.UtcNow);
+        var refused = await Checks(new FakeServer()).RunCheckAsync((await _repository.GetSyntheticCheckAsync(id))!, CancellationToken.None);
+        Assert.False(refused.Success);
+        Assert.Contains("kan ikke bruges", refused.Message, StringComparison.Ordinal);
+        Assert.NotNull((await _repository.GetSyntheticCheckAsync(id))!.SecretError);
+    }
+
+    [Fact]
+    public async Task Web_requests_test_a_server_and_run_a_check_as_the_agent()
+    {
+        var row = await AddServerAsync();
+        var server = new FakeServer { ServicesError = new ServerAccessException(ServerAccessProblem.AccessDenied, "Adgang nægtet til tjenester.") };
+        var id = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("Port", SyntheticCheckKind.Tcp, null,
+            new SyntheticCheckSettings { Host = "127.0.0.1", Port = ClosedPort() }, null, null, 300, 5), "anna", DateTimeOffset.UtcNow);
+        var handler = new ServerRequestHandler(_repository, server, Checks(server), TimeProvider.System);
+        var requests = new AgentRequestService(_repository, new InstanceConfigurationStore(Options.Create(new AgentOptions()), _repository, _key,
+            NullLogger<InstanceConfigurationStore>.Instance), _key, new SqlConnectionFactory(), new RepositoryReadiness(), TimeProvider.System,
+            NullLogger<AgentRequestService>.Instance, handler);
+
+        var test = await RunAsync(requests, ServerRequestKind.ServerTest, row.ServerId);
+        var run = await RunAsync(requests, ServerRequestKind.SyntheticCheckRun, id);
+        var missing = await RunAsync(requests, ServerRequestKind.ServerTest, 4711);
+
+        Assert.Equal(AgentRequestStatus.Done, test.Status);
+        var result = AgentRequestJson.Deserialize<WindowsServerTestResult>(test.Result)!;
+        Assert.True(result.Reachable);
+        Assert.Equal("Adgang nægtet til tjenester.", Assert.Single(result.Missing).Detail);
+        Assert.Equal(AgentRequestStatus.Done, run.Status);
+        Assert.False(AgentRequestJson.Deserialize<SyntheticCheckResult>(run.Result)!.Success);
+        Assert.Equal(1, (await _repository.GetSyntheticCheckAsync(id))!.ConsecutiveFailures);
+        Assert.Equal(AgentRequestStatus.Failed, missing.Status);
+        Assert.Contains("findes ikke", missing.Error, StringComparison.Ordinal);
+    }
+
+    private async Task<AgentRequestRow> RunAsync(AgentRequestService service, string kind, int target)
+    {
+        var id = await _repository.AddServerRequestAsync(kind, target, "anna", DateTimeOffset.UtcNow);
+        var claimed = await _repository.ClaimAgentRequestAsync(DateTimeOffset.UtcNow);
+        Assert.Equal(id, claimed!.RequestId);
+        await service.HandleAsync(claimed, CancellationToken.None);
+        return (await _repository.GetAgentRequestAsync(id))!;
+    }
+}

tests/SQLInsight.Servers.Tests/LocalCimTests.cs

@@ -87,7 +87,7 @@ public class LocalCimTests
         var source = new CimWindowsServerSource(new ServerConnectionOptions { OperationTimeout = TimeSpan.FromSeconds(5) });
         var error = await Assert.ThrowsAsync<ServerAccessException>(() => source.ConnectAsync("sqlinsight-findes-ikke.invalid", CancellationToken.None));
 
-        Assert.Equal(ServerAccessProblem.Unreachable, error.Problem);
+        Assert.True(error.Problem is ServerAccessProblem.Unreachable or ServerAccessProblem.Timeout, error.Message);
         Assert.Contains("sqlinsight-findes-ikke.invalid", error.Message, StringComparison.Ordinal);
     }
 }