Deadlines
- Event log over RPC (MS-EVEN6): the whole read has a deadline (OperationTimeout per log);
the token cancels the EventLogSession's operations and is checked before each
FormatDescription, and the agent stops waiting even if the native call ignores it.
Reported as ServerAccessProblem.Timeout ("svarede ikke i tide over hændelsesloggens
protokol (RPC) inden for N sekunder").
- Every CIM call runs on the thread pool and is awaited with the caller's token; a call
that ignores cancellation is abandoned and the CIM session is closed once it returns.
- Each server cycle has a deadline (new SQLInsight:Servers:CycleTimeout, default 5 min,
1-30 min, at least 2 x OperationTimeout): the areas not read by then are "ikke målbar"
("Serveren svarede ikke inden for N minutter, så indsamlingen blev afbrudt; …"),
logged once, and the slot is freed.
- Scheduled tasks: run information is read within a time budget (CycleTimeout / 3); the
rest keep their state without the last run, and the coverage note says so.
- "Test server" has an overall deadline (WindowsServerTester, default 2 min); areas not
done in time are reported as not answered, so the result is always stored.
Backoff and memory
- A server that does not answer (Unreachable/Timeout, or a timed-out cycle) is backed
off 1, 2, 5, 10, then 15 minutes, reset when it answers.
- Protocol Auto: the protocol that worked (e.g. DCOM after WS-Man was refused, also for
access denied) is tried first for an hour, then WS-Man is tried again; a server whose
event log RPC failed is read over CIM for an hour.
- Over WS-Man with read-only WinRM rights Task Scheduler's GetInfoByName is denied: with
Auto the tasks are read over DCOM instead (remembered for the server); with WSMan forced
the reason says to choose Auto or DCOM. The advice "winrm configSDDL default" (broader
WinRM rights) is gone; texts follow the grant script (-WinRM read right,
-EnableFirewall -AgentAddress), not Remote Management Users.
Events
- Each log continues from its own mark (ServerEventSnapshot.LogReadUntil, a small additive
field in Core): a truncated System log no longer loses the events between its last
read event and the newest Application event.
- CIM fallback: Win32_NTLogEvent is streamed and only the oldest max per log are kept
(memory bounded, deterministic order), consistent with the RPC read and the coverage
text, which now names the logs that had more.
Other
- CimInstances (and method results) are mapped to SQLInsight's types and disposed at once.
- Options are clamped when bound (ServerOptions/SyntheticCheckOptions.Normalize):
OperationTimeout 5 s-5 min, MaxEventsPerCollection 10-10000, MaxWatched 1-200,
MaxParallelServers/Checks 1-32, retention days >= 1, ForecastHistoryDays 1-365,
FailuresBeforeFinding/SlowRunsBeforeFinding 1-99.
- Shutdown: the timer's cancellation is caught, running cycles are awaited before the
throttle is disposed; a cycle's time is taken when it gets its slot; a TaskCanceledException
that is not shutdown no longer ends the service.
- Retention maintenance also runs (hourly) when SQLInsight:Servers:Enabled is false but
synthetic checks are on.
- LocalCimTests run alone (collection without parallelization), share one warmed-up
session, use the same 2 min operation timeout and a 5 min deadline per test, a smaller
workload, and report timings in assertion messages.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
124 lines
5.4 KiB
C#
124 lines
5.4 KiB
C#
using System.Diagnostics;
|
|
using SQLInsight.Core.Servers;
|
|
|
|
namespace SQLInsight.Servers.Tests;
|
|
|
|
/// <summary>What the agent remembers per server, and that a server that does not answer never holds the agent longer than its deadline.</summary>
|
|
public class ServerDeadlineTests
|
|
{
|
|
private sealed class Clock(DateTimeOffset now) : TimeProvider
|
|
{
|
|
public DateTimeOffset Now { get; set; } = now;
|
|
|
|
public override DateTimeOffset GetUtcNow() => Now;
|
|
}
|
|
|
|
[Fact]
|
|
public void A_fact_about_a_server_is_remembered_for_a_while_then_tried_again()
|
|
{
|
|
var clock = new Clock(new DateTimeOffset(2026, 10, 2, 12, 0, 0, TimeSpan.Zero));
|
|
var memory = new ServerHostMemory(clock, TimeSpan.FromHours(1));
|
|
|
|
memory.Remember("aos01.firma.local", "protocol", "DCOM");
|
|
clock.Now = clock.Now.AddMinutes(59);
|
|
Assert.Equal("DCOM", memory.Recall<string>("AOS01.firma.local", "protocol"));
|
|
Assert.Null(memory.Recall<string>("aos01.firma.local", "eventlog-rpc"));
|
|
|
|
clock.Now = clock.Now.AddMinutes(1);
|
|
Assert.Null(memory.Recall<string>("aos01.firma.local", "protocol"));
|
|
|
|
memory.Remember("aos01", "protocol", "DCOM");
|
|
memory.Forget("aos01", "protocol");
|
|
Assert.Null(memory.Recall<string>("aos01", "protocol"));
|
|
}
|
|
|
|
[Fact]
|
|
public void Operation_timeouts_are_bounded()
|
|
{
|
|
Assert.Equal(TimeSpan.FromSeconds(5), ServerConnectionOptions.BoundTimeout(TimeSpan.Zero));
|
|
Assert.Equal(TimeSpan.FromSeconds(5), ServerConnectionOptions.BoundTimeout(TimeSpan.FromSeconds(-1)));
|
|
Assert.Equal(TimeSpan.FromMinutes(5), ServerConnectionOptions.BoundTimeout(TimeSpan.FromHours(3)));
|
|
Assert.Equal(TimeSpan.FromSeconds(45), ServerConnectionOptions.BoundTimeout(TimeSpan.FromSeconds(45)));
|
|
}
|
|
|
|
/// <summary>A server that accepts the connection, then never answers any area (it honours cancellation, as the CIM source makes sure it is).</summary>
|
|
private sealed class HungServer(bool hangOnConnect = false) : IWindowsServerSource, IWindowsServerSession
|
|
{
|
|
public string Protocol => "DCOM";
|
|
|
|
public string? ProtocolNote => null;
|
|
|
|
public async Task<IWindowsServerSession> ConnectAsync(string host, CancellationToken cancellationToken)
|
|
{
|
|
if (hangOnConnect)
|
|
{
|
|
await Hang(cancellationToken);
|
|
}
|
|
|
|
return this;
|
|
}
|
|
|
|
private static Task Hang(CancellationToken cancellationToken) => Task.Delay(Timeout.Infinite, cancellationToken);
|
|
|
|
public async Task<ServerOsInfo> GetOperatingSystemAsync(CancellationToken cancellationToken)
|
|
{
|
|
await Hang(cancellationToken);
|
|
throw new UnreachableException();
|
|
}
|
|
|
|
public Task<ServerPerformanceRaw> GetPerformanceAsync(DateTimeOffset now, CancellationToken cancellationToken) => throw new UnreachableException();
|
|
|
|
public Task<IReadOnlyList<ServerDisk>> GetDisksAsync(CancellationToken cancellationToken) =>
|
|
Task.FromResult<IReadOnlyList<ServerDisk>>([new ServerDisk(@"C:\", null, "NTFS", 100, 50)]);
|
|
|
|
public async Task<IReadOnlyList<ServerService>> GetServicesAsync(CancellationToken cancellationToken)
|
|
{
|
|
await Hang(cancellationToken);
|
|
throw new UnreachableException();
|
|
}
|
|
|
|
public Task<ServerService?> GetServiceAsync(string name, CancellationToken cancellationToken) => throw new UnreachableException();
|
|
|
|
public async Task<ServerEventRead> GetEventsAsync(IReadOnlyDictionary<string, DateTimeOffset> since, int max, CancellationToken cancellationToken)
|
|
{
|
|
await Hang(cancellationToken);
|
|
throw new UnreachableException();
|
|
}
|
|
|
|
public async Task<ServerTaskRead> GetTasksAsync(Func<string, bool> watch, int maxWatched, TimeSpan budget, CancellationToken cancellationToken)
|
|
{
|
|
await Hang(cancellationToken);
|
|
throw new UnreachableException();
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_test_of_a_server_that_hangs_ends_at_its_deadline_with_every_area_said()
|
|
{
|
|
var watch = Stopwatch.StartNew();
|
|
var result = await WindowsServerTester.TestAsync(new HungServer(), "aos01", null, DateTimeOffset.UtcNow, TimeSpan.FromMilliseconds(300), CancellationToken.None);
|
|
var hungAtConnect = await WindowsServerTester.TestAsync(new HungServer(hangOnConnect: true), "aos01", null, DateTimeOffset.UtcNow, TimeSpan.FromMilliseconds(300),
|
|
CancellationToken.None);
|
|
|
|
Assert.True(watch.Elapsed < TimeSpan.FromSeconds(30), $"took {watch.Elapsed}");
|
|
Assert.True(result.Reachable);
|
|
Assert.Equal(ServerArea.All, result.Areas.Select(a => a.Area));
|
|
Assert.All(result.Areas, a => Assert.False(a.Ok));
|
|
Assert.All(result.Areas, a => Assert.Contains("svarede ikke inden for", a.Detail, StringComparison.Ordinal));
|
|
Assert.False(hungAtConnect.Reachable);
|
|
Assert.Contains("svarede ikke inden for", hungAtConnect.Error, StringComparison.Ordinal);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Stopping_the_agent_still_cancels_a_test()
|
|
{
|
|
using var stop = new CancellationTokenSource(TimeSpan.FromMilliseconds(200));
|
|
|
|
await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
|
|
WindowsServerTester.TestAsync(new HungServer(hangOnConnect: true), "aos01", null, DateTimeOffset.UtcNow, TimeSpan.FromMinutes(5), stop.Token));
|
|
}
|
|
}
|