Deadlines
- Event log over RPC (MS-EVEN6): the whole read has a deadline (OperationTimeout per log);
the token cancels the EventLogSession's operations and is checked before each
FormatDescription, and the agent stops waiting even if the native call ignores it.
Reported as ServerAccessProblem.Timeout ("svarede ikke i tide over hændelsesloggens
protokol (RPC) inden for N sekunder").
- Every CIM call runs on the thread pool and is awaited with the caller's token; a call
that ignores cancellation is abandoned and the CIM session is closed once it returns.
- Each server cycle has a deadline (new SQLInsight:Servers:CycleTimeout, default 5 min,
1-30 min, at least 2 x OperationTimeout): the areas not read by then are "ikke målbar"
("Serveren svarede ikke inden for N minutter, så indsamlingen blev afbrudt; …"),
logged once, and the slot is freed.
- Scheduled tasks: run information is read within a time budget (CycleTimeout / 3); the
rest keep their state without the last run, and the coverage note says so.
- "Test server" has an overall deadline (WindowsServerTester, default 2 min); areas not
done in time are reported as not answered, so the result is always stored.
Backoff and memory
- A server that does not answer (Unreachable/Timeout, or a timed-out cycle) is backed
off 1, 2, 5, 10, then 15 minutes, reset when it answers.
- Protocol Auto: the protocol that worked (e.g. DCOM after WS-Man was refused, also for
access denied) is tried first for an hour, then WS-Man is tried again; a server whose
event log RPC failed is read over CIM for an hour.
- Over WS-Man with read-only WinRM rights Task Scheduler's GetInfoByName is denied: with
Auto the tasks are read over DCOM instead (remembered for the server); with WSMan forced
the reason says to choose Auto or DCOM. The advice "winrm configSDDL default" (broader
WinRM rights) is gone; texts follow the grant script (-WinRM read right,
-EnableFirewall -AgentAddress), not Remote Management Users.
Events
- Each log continues from its own mark (ServerEventSnapshot.LogReadUntil, a small additive
field in Core): a truncated System log no longer loses the events between its last
read event and the newest Application event.
- CIM fallback: Win32_NTLogEvent is streamed and only the oldest max per log are kept
(memory bounded, deterministic order), consistent with the RPC read and the coverage
text, which now names the logs that had more.
Other
- CimInstances (and method results) are mapped to SQLInsight's types and disposed at once.
- Options are clamped when bound (ServerOptions/SyntheticCheckOptions.Normalize):
OperationTimeout 5 s-5 min, MaxEventsPerCollection 10-10000, MaxWatched 1-200,
MaxParallelServers/Checks 1-32, retention days >= 1, ForecastHistoryDays 1-365,
FailuresBeforeFinding/SlowRunsBeforeFinding 1-99.
- Shutdown: the timer's cancellation is caught, running cycles are awaited before the
throttle is disposed; a cycle's time is taken when it gets its slot; a TaskCanceledException
that is not shutdown no longer ends the service.
- Retention maintenance also runs (hourly) when SQLInsight:Servers:Enabled is false but
synthetic checks are on.
- LocalCimTests run alone (collection without parallelization), share one warmed-up
session, use the same 2 min operation timeout and a 5 min deadline per test, a smaller
workload, and report timings in assertion messages.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
114 lines
5.8 KiB
C#
114 lines
5.8 KiB
C#
using SQLInsight.Agent.Servers;
|
|
using SQLInsight.Core.Servers;
|
|
using SQLInsight.Core.SyntheticChecks;
|
|
using SQLInsight.Servers;
|
|
|
|
namespace SQLInsight.Agent.Tests;
|
|
|
|
public class ServerAgentOptionsTests
|
|
{
|
|
[Fact]
|
|
public void Server_options_are_kept_within_what_works()
|
|
{
|
|
var wild = new ServerOptions
|
|
{
|
|
MaxParallelServers = 0, OperationTimeout = TimeSpan.Zero, CycleTimeout = TimeSpan.FromHours(5), SampleDays = 0, DiskHistoryDays = -3, EventDays = 0,
|
|
};
|
|
wild.EventLog.MaxEventsPerCollection = 1_000_000;
|
|
wild.Tasks.MaxWatched = 0;
|
|
wild.Disks.ForecastHistoryDays = 0;
|
|
|
|
wild.Normalize();
|
|
|
|
Assert.Equal(1, wild.MaxParallelServers);
|
|
Assert.Equal(TimeSpan.FromSeconds(5), wild.OperationTimeout);
|
|
Assert.Equal(TimeSpan.FromMinutes(30), wild.CycleTimeout);
|
|
Assert.Equal((1, 1, 1), (wild.SampleDays, wild.DiskHistoryDays, wild.EventDays)); // 0 days would make maintenance delete everything
|
|
Assert.Equal(10000, wild.EventLog.MaxEventsPerCollection);
|
|
Assert.Equal(1, wild.Tasks.MaxWatched);
|
|
Assert.Equal(1, wild.Disks.ForecastHistoryDays);
|
|
|
|
var other = new ServerOptions { MaxParallelServers = 500, OperationTimeout = TimeSpan.FromHours(2), CycleTimeout = TimeSpan.FromSeconds(1) };
|
|
other.EventLog.MaxEventsPerCollection = 1;
|
|
other.Tasks.MaxWatched = 10_000;
|
|
other.Normalize();
|
|
|
|
Assert.Equal(32, other.MaxParallelServers);
|
|
Assert.Equal(TimeSpan.FromMinutes(5), other.OperationTimeout);
|
|
Assert.Equal(TimeSpan.FromMinutes(10), other.CycleTimeout); // at least two operations
|
|
Assert.Equal(10, other.EventLog.MaxEventsPerCollection);
|
|
Assert.Equal(200, other.Tasks.MaxWatched);
|
|
Assert.Equal(other.CycleTimeout / 3, other.TaskBudget);
|
|
}
|
|
|
|
[Fact]
|
|
public void Defaults_are_unchanged_by_normalizing()
|
|
{
|
|
var defaults = new ServerOptions().Normalize();
|
|
|
|
Assert.Equal((4, TimeSpan.FromSeconds(30), TimeSpan.FromMinutes(5)), (defaults.MaxParallelServers, defaults.OperationTimeout, defaults.CycleTimeout));
|
|
Assert.Equal((14, 35, 90), (defaults.SampleDays, defaults.DiskHistoryDays, defaults.EventDays));
|
|
Assert.Equal((2000, 50), (defaults.EventLog.MaxEventsPerCollection, defaults.Tasks.MaxWatched));
|
|
Assert.Equal(TimeSpan.FromSeconds(30), ServerConnectionOptions.BoundTimeout(TimeSpan.FromSeconds(30)));
|
|
}
|
|
|
|
[Fact]
|
|
public void Synthetic_check_options_are_kept_within_what_works()
|
|
{
|
|
var wild = new SyntheticCheckOptions { MaxParallelChecks = -1, ResultDays = 0, FailuresBeforeFinding = 30, SlowRunsBeforeFinding = 0 }.Normalize();
|
|
|
|
Assert.Equal(1, wild.MaxParallelChecks);
|
|
Assert.Equal(1, wild.ResultDays);
|
|
Assert.Equal(1, wild.SlowRunsBeforeFinding);
|
|
Assert.Equal(31, wild.RecentResults); // enough to count 30 failures in a row
|
|
Assert.Equal(11, new SyntheticCheckOptions().RecentResults);
|
|
Assert.Equal(100, new SyntheticCheckOptions { FailuresBeforeFinding = 5000 }.Normalize().RecentResults);
|
|
Assert.Equal(32, new SyntheticCheckOptions { MaxParallelChecks = 1000 }.Normalize().MaxParallelChecks);
|
|
}
|
|
|
|
[Fact]
|
|
public void A_check_never_runs_more_often_than_the_minimum_interval()
|
|
{
|
|
Assert.Equal(TimeSpan.FromSeconds(SyntheticCheckLimits.MinIntervalSeconds), SyntheticCheckOptions.IntervalOf(0));
|
|
Assert.Equal(TimeSpan.FromSeconds(SyntheticCheckLimits.MinIntervalSeconds), SyntheticCheckOptions.IntervalOf(5));
|
|
Assert.Equal(TimeSpan.FromMinutes(5), SyntheticCheckOptions.IntervalOf(300));
|
|
Assert.Equal(TimeSpan.FromSeconds(SyntheticCheckLimits.MaxIntervalSeconds), SyntheticCheckOptions.IntervalOf(int.MaxValue));
|
|
}
|
|
|
|
[Fact]
|
|
public void An_unreachable_server_is_backed_off_1_2_5_10_then_15_minutes()
|
|
{
|
|
var waits = Enumerable.Range(1, 7).Select(ServerMonitor.Backoff).Select(t => (int)t.TotalMinutes);
|
|
|
|
Assert.Equal([1, 2, 5, 10, 15, 15, 15], waits);
|
|
}
|
|
|
|
[Fact]
|
|
public void Each_event_log_continues_from_its_own_mark()
|
|
{
|
|
var now = new DateTimeOffset(2026, 10, 2, 12, 0, 0, TimeSpan.Zero);
|
|
var since = new Dictionary<string, DateTimeOffset> { ["System"] = now.AddHours(-24), ["Application"] = now.AddHours(-24) };
|
|
|
|
// System had more than could be read (up to 08:00); Application was read to 11:59.
|
|
var read = new ServerEventRead(
|
|
[
|
|
new ServerEventEntry("System", "disk", 153, 2, now.AddHours(-6)),
|
|
new ServerEventEntry("System", "disk", 153, 2, now.AddHours(-4)),
|
|
new ServerEventEntry("Application", "app", 1000, 2, now.AddMinutes(-1)),
|
|
], "Event Log (RPC)", true, null, ["System"]);
|
|
|
|
var marks = ServerMonitor.NextReadUntil(read, since, now);
|
|
|
|
Assert.Equal(now.AddHours(-4), marks["System"]); // the rest of System is read next time, not skipped
|
|
Assert.Equal(now.AddMinutes(-1), marks["Application"]);
|
|
var snapshot = new ServerEventSnapshot { CapturedAt = now, ReadUntil = marks.Values.Min(), LogReadUntil = marks };
|
|
Assert.Equal(now.AddHours(-4), ServerMonitor.EventsReadUntil(snapshot, "System", now));
|
|
Assert.Equal(now.AddMinutes(-1), ServerMonitor.EventsReadUntil(snapshot, "application", now));
|
|
|
|
// A log without events keeps ten minutes' margin; a snapshot from before per-log marks continues from its single mark.
|
|
var quiet = ServerMonitor.NextReadUntil(new ServerEventRead([], "Event Log (RPC)", false), since, now);
|
|
Assert.Equal(now.AddMinutes(-10), quiet["System"]);
|
|
Assert.Equal(now.AddHours(-2), ServerMonitor.EventsReadUntil(new ServerEventSnapshot { CapturedAt = now, ReadUntil = now.AddHours(-2) }, "System", now));
|
|
Assert.Equal(now - ServerEventAnalysis.RecentWindow, ServerMonitor.EventsReadUntil(null, "System", now));
|
|
}
|
|
}
|