Review 1.5: synthetic checks: "Kør nu" joins the running check, errors are stored once, paused checks are left alone

- "Kør nu" (ServerRequestHandler) goes through SyntheticCheckService.RunNowAsync: it joins a
  run of the check that is going on, or runs it under the same MaxParallelChecks throttle
  and registers it, so it never overlaps a scheduled run. A paused check is not run: the
  answer is "Tjekket er sat på pause." and nothing is stored, evaluated or opened.
- Server requests (test, run now) may take 3 min (the check's max timeout 120 s + the
  30 s service lookup + 30 s margin) instead of AgentRequestService's 2 min, so the result
  is always stored; a server test gets 2.5 min. Stale Running requests are cleaned after
  the longest timeout + 1 min.
- An exception a check run did not foresee becomes a stored failed result ("Tjekket kunne
  ikke køres: …"), so LastRunAt advances and the check waits its interval; it is logged
  once per change. A run that cannot even be stored is not retried before its interval
  (the agent remembers when it started it).
- The service lookup of a failing check has its own 30 s deadline; its error detail
  (naming the agent's account) is Sensitive.
- The rules get max(10, FailuresBeforeFinding, SlowRunsBeforeFinding) + 1 recent results
  (at most 100), so "Fejlet i træk" and FailuresBeforeFinding > 10 work.
- The interval is never under SyntheticCheckLimits.MinIntervalSeconds agent-side; state
  for removed checks is pruned; shutdown waits for running checks before the throttle is
  disposed; a TaskCanceledException that is not shutdown no longer ends the service.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/SQLInsight.Agent/Configuration/AgentRequestService.cs

@@ -61,7 +61,7 @@ public sealed partial class AgentRequestService(
                 {
                     nextHousekeeping = now.AddMinutes(10);
                     await repository.PublishAgentKeyAsync(key.Describe(now), stoppingToken).ConfigureAwait(false);
-                    await repository.CleanUpAgentRequestsAsync(now, RequestTimeout + TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(30), TimeSpan.FromDays(30), stoppingToken)
+                    await repository.CleanUpAgentRequestsAsync(now, StaleAfter, TimeSpan.FromMinutes(30), TimeSpan.FromDays(30), stoppingToken)
                         .ConfigureAwait(false);
                 }
 
@@ -86,8 +86,9 @@ public sealed partial class AgentRequestService(
     public async Task HandleAsync(AgentRequestRow request, CancellationToken cancellationToken)
     {
         ArgumentNullException.ThrowIfNull(request);
+        var limit = TimeoutOf(request.Kind);
         using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
-        timeout.CancelAfter(RequestTimeout);
+        timeout.CancelAfter(limit);
         string? result = null;
         string? error = null;
         try
@@ -104,7 +105,7 @@ public sealed partial class AgentRequestService(
         }
         catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
         {
-            error = $"Agenten fik ikke svar inden for {RequestTimeout.TotalMinutes:0} minutter.";
+            error = $"Agenten fik ikke svar inden for {limit.TotalMinutes:0} minutter.";
         }
         // Anything else (a lock timeout, a repository error while storing a discovery …) is the request's result too, so it
         // is never left as Running for the web UI to wait on.
@@ -117,6 +118,15 @@ public sealed partial class AgentRequestService(
         await repository.CompleteAgentRequestAsync(request.RequestId, error is null, result, error, timeProvider.GetUtcNow(), CancellationToken.None).ConfigureAwait(false);
     }
 
+    /// <summary>How long a request may take: a synthetic check run now may take its own timeout plus the lookup of its service.</summary>
+    public TimeSpan TimeoutOf(string kind) =>
+        serverRequests is not null && Servers.ServerRequestHandler.Handles(kind) ? Longest(RequestTimeout, Servers.ServerRequestHandler.RequestTimeout) : RequestTimeout;
+
+    /// <summary>A request still Running after this was lost (the agent stopped): the longest request plus a minute.</summary>
+    private static TimeSpan StaleAfter => Longest(RequestTimeout, Servers.ServerRequestHandler.RequestTimeout) + TimeSpan.FromMinutes(1);
+
+    private static TimeSpan Longest(TimeSpan a, TimeSpan b) => a > b ? a : b;
+
     private static int Require(AgentRequestRow request) =>
         request.ConfiguredInstanceId ?? throw new InvalidOperationException("Anmodningen mangler en instans.");
 

src/SQLInsight.Agent/Servers/ServerRequestHandler.cs

@@ -1,5 +1,6 @@
 using SQLInsight.Core.Configuration;
 using SQLInsight.Core.Servers;
+using SQLInsight.Core.SyntheticChecks;
 using SQLInsight.Repository;
 using SQLInsight.Servers;
 
@@ -11,6 +12,18 @@ namespace SQLInsight.Agent.Servers;
 /// </summary>
 public sealed class ServerRequestHandler(InsightRepository repository, IWindowsServerSource source, SyntheticCheckService checks, TimeProvider time)
 {
+    /// <summary>Margin for reading the request and storing its result.</summary>
+    public static readonly TimeSpan Margin = TimeSpan.FromSeconds(30);
+
+    /// <summary>
+    /// How long such a request may take: a check's longest timeout, the lookup of its Windows service and storing the
+    /// result, so "Kør nu" always gets its result stored.
+    /// </summary>
+    public static readonly TimeSpan RequestTimeout = TimeSpan.FromSeconds(SyntheticCheckLimits.MaxTimeoutSeconds) + SyntheticCheckService.ServiceLookupTimeout + Margin;
+
+    /// <summary>How long a server test may take: what is not done by then is reported as not answered, in time to be stored.</summary>
+    public static readonly TimeSpan TestTimeout = RequestTimeout - Margin;
+
     public static bool Handles(string kind) => ServerRequestKind.Handles(kind);
 
     /// <summary>The request's result as JSON. Throws <see cref="KeyNotFoundException"/> for a server or check that does not exist.</summary>
@@ -33,7 +46,7 @@ public sealed class ServerRequestHandler(InsightRepository repository, IWindowsS
         var server = await repository.GetWindowsServerAsync(serverId, cancellationToken).ConfigureAwait(false)
             ?? throw new KeyNotFoundException($"Server {serverId} findes ikke.");
         var account = OperatingSystem.IsWindows() ? CimWindowsServerSource.AgentAccount(server.Host) : null;
-        var result = await WindowsServerTester.TestAsync(source, server.Host, account, time.GetUtcNow(), cancellationToken).ConfigureAwait(false);
+        var result = await WindowsServerTester.TestAsync(source, server.Host, account, time.GetUtcNow(), TestTimeout, cancellationToken).ConfigureAwait(false);
         if (result.Reachable)
         {
             await repository.RecordServerAreasAsync(serverId, result.Areas, cancellationToken).ConfigureAwait(false);
@@ -42,10 +55,11 @@ public sealed class ServerRequestHandler(InsightRepository repository, IWindowsS
         return result;
     }
 
-    public async Task<Core.SyntheticChecks.SyntheticCheckResult> RunAsync(int checkId, CancellationToken cancellationToken)
+    /// <summary>"Kør nu": joins a run of the check that is going on, or runs it within the limit on parallel checks; a paused check is not run.</summary>
+    public async Task<SyntheticCheckResult> RunAsync(int checkId, CancellationToken cancellationToken)
     {
         var check = await repository.GetSyntheticCheckAsync(checkId, cancellationToken).ConfigureAwait(false)
             ?? throw new KeyNotFoundException($"Tjek {checkId} findes ikke.");
-        return await checks.RunCheckAsync(check, cancellationToken).ConfigureAwait(false);
+        return await checks.RunNowAsync(check, cancellationToken).ConfigureAwait(false);
     }
 }

src/SQLInsight.Agent/Servers/SyntheticCheckService.cs

@@ -12,7 +12,8 @@ namespace SQLInsight.Agent.Servers;
 /// <summary>
 /// Runs the synthetic checks on their schedule (and on request from SQLInsight Web), as the agent's identity: stores
 /// each result, looks up the linked Windows service when a check fails ("svarer ikke, selv om tjenesten kører"),
-/// evaluates the rules and notifies. Never changes anything on the servers it checks.
+/// evaluates the rules and notifies. A check runs once at a time, and at most MaxParallelChecks run together, also when
+/// asked to run now. Never changes anything on the servers it checks.
 /// </summary>
 public sealed partial class SyntheticCheckService(
     IOptions<SyntheticCheckOptions> options,
@@ -27,9 +28,17 @@ public sealed partial class SyntheticCheckService(
     IOptions<AgentOptions> agentOptions,
     Configuration.RepositoryReadiness readiness) : BackgroundService
 {
-    private readonly Dictionary<int, Task> _running = [];
+    /// <summary>The answer to "Kør nu" for a paused check: it is not run.</summary>
+    public const string PausedMessage = "Tjekket er sat på pause.";
+
+    /// <summary>How long looking up the state of a failing check's Windows service may take.</summary>
+    public static readonly TimeSpan ServiceLookupTimeout = TimeSpan.FromSeconds(30);
+
+    private readonly Dictionary<int, Task<SyntheticCheckResult?>> _running = [];
+    private readonly Dictionary<int, DateTimeOffset> _attempts = [];
     private readonly Dictionary<int, string> _reported = [];
     private readonly Lock _lock = new();
+    private readonly SemaphoreSlim _throttle = new(Math.Clamp(options.Value.MaxParallelChecks, 1, 32));
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
@@ -40,37 +49,37 @@ public sealed partial class SyntheticCheckService(
         }
 
         await readiness.Ready.WaitAsync(stoppingToken).ConfigureAwait(false);
-        using var throttle = new SemaphoreSlim(Math.Max(1, settings.MaxParallelChecks));
         using var timer = new PeriodicTimer(agentOptions.Value.Tick, time);
-        do
+        try
         {
-            try
+            do
             {
-                var now = time.GetUtcNow();
+                try
-                foreach (var check in await repository.GetSyntheticChecksAsync(stoppingToken).ConfigureAwait(false))
                 {
-                    if (check.Paused || (check.LastRunAt is { } last && last.AddSeconds(check.IntervalSeconds) > now.UtcDateTime))
+                    var now = time.GetUtcNow();
+                    var checks = await repository.GetSyntheticChecksAsync(stoppingToken).ConfigureAwait(false);
+                    Prune(checks);
+                    foreach (var check in checks.Where(c => IsDue(c, now)))
                     {
-                        continue;
+                        lock (_lock)
-                    }
-
-                    lock (_lock)
-                    {
-                        if (_running.TryGetValue(check.CheckId, out var task) && !task.IsCompleted)
                         {
-                            continue;
+                            _attempts[check.CheckId] = now;
                         }
 
-                        _running[check.CheckId] = RunThrottledAsync(check, throttle, stoppingToken);
+                        _ = Start(check, stoppingToken);
                     }
                 }
+                catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
+                {
+                    LogRoundFailed(logger, ex);
+                }
             }
-            catch (Exception ex) when (ex is not OperationCanceledException)
+            while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
-            {
+        }
-                LogRoundFailed(logger, ex);
+        catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
-            }
+        {
+            // Stopping: the running checks end below.
         }
-        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
 
         Task[] remaining;
         lock (_lock)
@@ -78,27 +87,118 @@ public sealed partial class SyntheticCheckService(
             remaining = [.. _running.Values];
         }
 
-        await Task.WhenAll(remaining).ConfigureAwait(false);
+        await ServerMonitoringService.WaitQuietlyAsync(remaining).ConfigureAwait(false);
     }
 
-    private async Task RunThrottledAsync(SyntheticCheckRow check, SemaphoreSlim throttle, CancellationToken cancellationToken)
+    public override void Dispose()
     {
-        await throttle.WaitAsync(cancellationToken).ConfigureAwait(false);
+        base.Dispose();
+        _throttle.Dispose();
+    }
+
+    /// <summary>
+    /// Due when its interval (never under a minute) has passed since the last stored result and since the agent last
+    /// started it: a run that could not be stored is not repeated every few seconds.
+    /// </summary>
+    private bool IsDue(SyntheticCheckRow check, DateTimeOffset now)
+    {
+        if (check.Paused)
+        {
+            return false;
+        }
+
+        var interval = SyntheticCheckOptions.IntervalOf(check.IntervalSeconds);
+        if (check.LastRunAt is { } last && new DateTimeOffset(DateTime.SpecifyKind(last, DateTimeKind.Utc)) + interval > now)
+        {
+            return false;
+        }
+
+        lock (_lock)
+        {
+            return !(_attempts.TryGetValue(check.CheckId, out var attempted) && attempted + interval > now)
+                   && !(_running.TryGetValue(check.CheckId, out var running) && !running.IsCompleted);
+        }
+    }
+
+    /// <summary>Forgets checks that were removed.</summary>
+    private void Prune(IReadOnlyList<SyntheticCheckRow> checks)
+    {
+        var ids = checks.Select(c => c.CheckId).ToHashSet();
+        lock (_lock)
+        {
+            foreach (var id in _running.Where(r => !ids.Contains(r.Key) && r.Value.IsCompleted).Select(r => r.Key).ToList())
+            {
+                _running.Remove(id);
+            }
+
+            foreach (var id in _attempts.Keys.Where(id => !ids.Contains(id)).ToList())
+            {
+                _attempts.Remove(id);
+            }
+
+            foreach (var id in _reported.Keys.Where(id => !ids.Contains(id)).ToList())
+            {
+                _reported.Remove(id);
+            }
+        }
+    }
+
+    /// <summary>The check's run that is going on, or a new one under the limit on parallel checks.</summary>
+    private Task<SyntheticCheckResult?> Start(SyntheticCheckRow check, CancellationToken cancellationToken)
+    {
+        lock (_lock)
+        {
+            if (_running.TryGetValue(check.CheckId, out var running) && !running.IsCompleted)
+            {
+                return running;
+            }
+
+            var task = RunThrottledAsync(check, cancellationToken);
+            _running[check.CheckId] = task;
+            return task;
+        }
+    }
+
+    /// <summary>
+    /// "Kør nu" from SQLInsight Web: joins the check's run if one is going on, or runs it under the same limit as the
+    /// scheduled runs. A paused check is not run (and nothing is stored or evaluated).
+    /// </summary>
+    public async Task<SyntheticCheckResult> RunNowAsync(SyntheticCheckRow check, CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(check);
+        if (check.Paused)
+        {
+            return new SyntheticCheckResult { At = time.GetUtcNow(), Success = false, Message = PausedMessage };
+        }
+
+        var result = await Start(check, cancellationToken).WaitAsync(cancellationToken).ConfigureAwait(false);
+        return result ?? throw new InvalidOperationException("Tjekket kunne ikke køres eller resultatet ikke gemmes. Se agentens log.");
+    }
+
+    /// <summary>One run; null when it could not be stored (logged once per change).</summary>
+    private async Task<SyntheticCheckResult?> RunThrottledAsync(SyntheticCheckRow check, CancellationToken cancellationToken)
+    {
+        await _throttle.WaitAsync(cancellationToken).ConfigureAwait(false);
         try
         {
-            await RunCheckAsync(check, cancellationToken).ConfigureAwait(false);
+            return await RunCheckAsync(check, cancellationToken).ConfigureAwait(false);
         }
-        catch (Exception ex) when (ex is not OperationCanceledException)
+        catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
         {
-            LogCheckFailed(logger, ex, check.Name);
+            if (IsNews(check.CheckId, "fejl: " + ex.Message, remember: true))
+            {
+                LogCheckFailed(logger, ex, check.Name);
+            }
+
+            return null;
         }
         finally
         {
-            throttle.Release();
+            _throttle.Release();
         }
     }
 
-    /// <summary>Runs a check once, stores the result and its findings. Used on schedule and for "Kør nu" in SQLInsight Web.</summary>
+    /// <summary>Runs a check once, stores the result and its findings. Used on schedule and for "Kør nu" (through <see cref="RunNowAsync"/>).</summary>
     public async Task<SyntheticCheckResult> RunCheckAsync(SyntheticCheckRow check, CancellationToken cancellationToken)
     {
         ArgumentNullException.ThrowIfNull(check);
@@ -115,7 +215,23 @@ public sealed partial class SyntheticCheckService(
         }
 
         var timeout = TimeSpan.FromSeconds(Math.Clamp(check.TimeoutSeconds, 1, SyntheticCheckLimits.MaxTimeoutSeconds));
-        var result = await runner.RunAsync(new SyntheticCheckRun(check.Kind, settings, timeout, password), cancellationToken).ConfigureAwait(false);
+        SyntheticCheckResult result;
+        try
+        {
+            result = await runner.RunAsync(new SyntheticCheckRun(check.Kind, settings, timeout, password), cancellationToken).ConfigureAwait(false);
+        }
+        catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
+        {
+            // Not a failure the runner foresaw: still the check's result, so it is stored, waits its interval, and is logged once.
+            var message = "Tjekket kunne ikke køres: " + ex.Message;
+            if (IsNews(check.CheckId, message, remember: false))
+            {
+                LogCheckFailed(logger, ex, check.Name);
+            }
+
+            result = new SyntheticCheckResult { At = now, Success = false, DurationMs = (int)Math.Min(int.MaxValue, (time.GetUtcNow() - now).TotalMilliseconds), Message = message };
+        }
+
         if (!result.Success && check.ServerId is { } serverId && settings.ServiceName is { Length: > 0 } service)
         {
             result = await WithServiceStateAsync(result, serverId, service, cancellationToken).ConfigureAwait(false);
@@ -152,7 +268,10 @@ public sealed partial class SyntheticCheckService(
         return (password, error);
     }
 
-    /// <summary>The state of the Windows service behind a failing check: the difference between "nede" and "kører, men svarer ikke".</summary>
+    /// <summary>
+    /// The state of the Windows service behind a failing check: the difference between "nede" and "kører, men svarer
+    /// ikke". Takes at most <see cref="ServiceLookupTimeout"/>.
+    /// </summary>
     private async Task<SyntheticCheckResult> WithServiceStateAsync(SyntheticCheckResult result, int serverId, string service, CancellationToken cancellationToken)
     {
         var server = await repository.GetWindowsServerAsync(serverId, cancellationToken).ConfigureAwait(false);
@@ -161,10 +280,12 @@ public sealed partial class SyntheticCheckService(
             return result;
         }
 
+        using var lookup = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+        lookup.CancelAfter(ServiceLookupTimeout);
         try
         {
-            using var session = await servers.ConnectAsync(server.Host, cancellationToken).ConfigureAwait(false);
+            using var session = await servers.ConnectAsync(server.Host, lookup.Token).ConfigureAwait(false);
-            var state = await session.GetServiceAsync(service, cancellationToken).ConfigureAwait(false);
+            var state = await session.GetServiceAsync(service, lookup.Token).ConfigureAwait(false);
             return result with
             {
                 ServiceState = state?.State ?? "NotFound",
@@ -173,24 +294,34 @@ public sealed partial class SyntheticCheckService(
         }
         catch (ServerAccessException ex)
         {
-            return result with { Details = [.. result.Details, new SyntheticCheckDetail("Tjeneste", $"{service} på {server.Name} kunne ikke aflæses: {ex.Message}")] };
+            // The reason names the agent's account: kept on-prem.
+            return result with { Details = [.. result.Details, new SyntheticCheckDetail("Tjeneste", $"{service} på {server.Name} kunne ikke aflæses: {ex.Message}", Sensitive: true)] };
+        }
+        catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
+        {
+            return result with
+            {
+                Details = [.. result.Details, new SyntheticCheckDetail("Tjeneste",
+                    $"{service} på {server.Name} kunne ikke aflæses: serveren svarede ikke inden for {ServiceLookupTimeout.TotalSeconds:0} sekunder.")],
+            };
         }
     }
 
     private async Task<SyntheticCheckResult> StoreAsync(SyntheticCheckRow check, SyntheticCheckResult result, CancellationToken cancellationToken)
     {
         await repository.RecordSyntheticCheckResultAsync(check.CheckId, result, cancellationToken).ConfigureAwait(false);
-        var recent = (await repository.GetSyntheticCheckResultsAsync(check.CheckId, 10, cancellationToken: cancellationToken).ConfigureAwait(false))
+        var settings = options.Value;
+        var recent = (await repository.GetSyntheticCheckResultsAsync(check.CheckId, settings.RecentResults, cancellationToken: cancellationToken).ConfigureAwait(false))
             .Select(r => r.ToResult()).ToList();
-        var settings = check.Settings;
+        var checkSettings = check.Settings;
-        var identity = new SyntheticCheckIdentity(check.CheckId, check.Name, check.Kind, settings.Target(check.Kind))
+        var identity = new SyntheticCheckIdentity(check.CheckId, check.Name, check.Kind, checkSettings.Target(check.Kind))
         {
             ServerName = check.ServerName,
-            ServiceName = settings.ServiceName,
+            ServiceName = checkSettings.ServiceName,
-            WarnMilliseconds = settings.WarnMilliseconds,
+            WarnMilliseconds = checkSettings.WarnMilliseconds,
-            Https = settings.Url?.Scheme == Uri.UriSchemeHttps,
+            Https = checkSettings.Url?.Scheme == Uri.UriSchemeHttps,
         };
-        var results = ScopedRuleEngine.Evaluate(SyntheticCheckRules.Create(), new SyntheticCheckRuleContext(identity, result.At, recent, options.Value));
+        var results = ScopedRuleEngine.Evaluate(SyntheticCheckRules.Create(), new SyntheticCheckRuleContext(identity, result.At, recent, settings));
         var changes = await repository.SaveScopedRuleResultsAsync(FindingScope.Check, check.CheckId, results, result.At, cancellationToken).ConfigureAwait(false);
         var events = ServerNotifications.For(check.Name, changes, notificationOptions.Value.WebUrl, result.At).ToList();
         if (events.Count > 0)
@@ -198,19 +329,33 @@ public sealed partial class SyntheticCheckService(
             await notifications.DispatchAsync(events, cancellationToken).ConfigureAwait(false);
         }
 
-        lock (_lock)
+        if (IsNews(check.CheckId, result.Success ? "ok" : result.Message, remember: true))
         {
-            var state = result.Success ? "ok" : result.Message;
+            LogResult(logger, check.Name, result.Success, result.Message);
-            if (!_reported.TryGetValue(check.CheckId, out var previous) || previous != state)
-            {
-                _reported[check.CheckId] = state;
-                LogResult(logger, check.Name, result.Success, result.Message);
-            }
         }
 
         return result;
     }
 
+    /// <summary>Whether a check's state differs from the last one logged (the same failure every run is not news).</summary>
+    private bool IsNews(int checkId, string state, bool remember)
+    {
+        lock (_lock)
+        {
+            if (_reported.TryGetValue(checkId, out var previous) && previous == state)
+            {
+                return false;
+            }
+
+            if (remember)
+            {
+                _reported[checkId] = state;
+            }
+
+            return true;
+        }
+    }
+
     [LoggerMessage(Level = LogLevel.Information, Message = "Synthetic check {Check}: success {Success}: {Message}")]
     private static partial void LogResult(ILogger logger, string check, bool success, string message);
 

tests/SQLInsight.IntegrationTests/ServerAgentTests.cs

@@ -236,6 +236,13 @@ public sealed class ServerAgentTests(SqlServerFixture sql) : IAsyncLifetime
         public BusinessConnectorInstallation? Find() => null;
     }
 
+    /// <summary>Fails in a way the check runner does not foresee.</summary>
+    private sealed class BrokenSql : ISqlConnectionFactory
+    {
+        public Task<OpenedConnection> OpenAsync(ConnectionTarget target, CancellationToken cancellationToken = default) =>
+            throw new NotSupportedException("Uventet fejl i driveren.");
+    }
+
     [Fact]
     public async Task A_server_that_hangs_is_cut_off_at_the_cycle_deadline_and_backed_off()
     {
@@ -262,6 +269,28 @@ public sealed class ServerAgentTests(SqlServerFixture sql) : IAsyncLifetime
         Assert.Equal(DateTimeOffset.MinValue, monitor.RetryAt);
     }
 
+    [Fact]
+    public async Task Run_now_leaves_a_paused_check_alone_and_an_unexpected_error_is_a_stored_failure()
+    {
+        var paused = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("Pause", SyntheticCheckKind.Tcp, null,
+            new SyntheticCheckSettings { Host = "127.0.0.1", Port = ClosedPort() }, null, null, 300, 5, Paused: true), "anna", DateTimeOffset.UtcNow);
+        var sql = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("SQL", SyntheticCheckKind.Sql, null, new SyntheticCheckSettings { Server = "sql01" },
+            null, null, 300, 5), "anna", DateTimeOffset.UtcNow);
+        var service = Checks(new FakeServer(), sql: new BrokenSql());
+
+        var declined = await service.RunNowAsync((await _repository.GetSyntheticCheckAsync(paused))!, CancellationToken.None);
+        var failed = await service.RunNowAsync((await _repository.GetSyntheticCheckAsync(sql))!, CancellationToken.None);
+
+        Assert.False(declined.Success);
+        Assert.Equal(SyntheticCheckService.PausedMessage, declined.Message);
+        Assert.Empty(await _repository.GetSyntheticCheckResultsAsync(paused));
+        Assert.Null((await _repository.GetSyntheticCheckAsync(paused))!.LastRunAt);
+        Assert.Empty(await _repository.GetServerRuleStatesAsync(FindingScope.Check, paused));
+        Assert.False(failed.Success);
+        Assert.StartsWith("Tjekket kunne ikke køres: Uventet fejl i driveren.", failed.Message, StringComparison.Ordinal);
+        Assert.NotNull((await _repository.GetSyntheticCheckAsync(sql))!.LastRunAt); // stored: waits its interval instead of running every 15 seconds
+    }
+
     private static int ClosedPort()
     {
         using var probe = new TcpListener(IPAddress.Loopback, 0);