Søren Eggert Lundsteen Olsen
ba7c751b5c
Scoped rules with their own context (a Windows server, a synthetic check) that produce the same findings and outcomes as the instance rules; missing data is "ikke målbar" with the collector's reason. Thresholds are settings (SQLInsight:Servers, SQLInsight:SyntheticChecks). - Sustained CPU, low memory (share and MB), paging with low memory, commit charge. - Free disk space and "fuld om ca. N dage" forecasts. - Stopped automatic services (minus those that stop by design, and a grace period after a restart) and watched services that must run. - Windows event log groups: important kinds at once, ordinary errors when they repeat, capped per server; sample messages are sensitive. - Unexpected restart (Kernel-Power 41, EventLog 6008, BugCheck), optional uptime. - Scheduled tasks that failed or did not run as planned. - Synthetic checks failing (twice in a row), "svarer ikke, selv om tjenesten kører", slow answers, TLS certificates about to expire. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
334 lines
17 KiB
C#
334 lines
17 KiB
C#
using SQLInsight.Core.Collection;
|
|
using SQLInsight.Core.Findings;
|
|
using SQLInsight.Core.Servers;
|
|
using SQLInsight.Rules.Engine;
|
|
using SQLInsight.Rules.Servers;
|
|
|
|
namespace SQLInsight.Rules.Tests;
|
|
|
|
public class WindowsServerRuleTests
|
|
{
|
|
private static readonly DateTimeOffset Now = new(2026, 10, 2, 12, 0, 0, TimeSpan.Zero);
|
|
|
|
private static ServerIdentity Server(IReadOnlyList<string>? watched = null, IReadOnlyList<string>? ignored = null) =>
|
|
new(1, "AOS01", "aos01.firma.local", ServerRoles.Application) { WatchedServices = watched ?? [], IgnoredServices = ignored ?? [] };
|
|
|
|
private static ServerRuleContext Context(CollectedData data, ServerIdentity? server = null, ServerRuleSettings? settings = null) =>
|
|
new(server ?? Server(), Now, data, settings ?? new ServerRuleSettings());
|
|
|
|
private static ServerPerformanceSnapshot Performance(Func<int, ServerPerformanceSample> sample, int minutes = 60, DateTimeOffset? boot = null) => new()
|
|
{
|
|
CapturedAt = Now,
|
|
Os = new ServerOsInfo { Caption = "Windows Server 2022", TotalMemoryMb = 16384, LastBootUpTime = boot ?? Now.AddDays(-30), LogicalProcessors = 8 },
|
|
Samples = Enumerable.Range(0, minutes).Select(m => sample(m) with { At = Now.AddMinutes(-m) }).Reverse().ToList(),
|
|
};
|
|
|
|
private static ServerPerformanceSample Sample(double? cpu = 20, double? availableMb = 8000, double? pages = 10, double? commit = 40) =>
|
|
new(Now, cpu, availableMb, 16384, pages, commit);
|
|
|
|
private static CollectedData With<T>(T value)
|
|
where T : class
|
|
{
|
|
var data = new CollectedData();
|
|
data.Set(value);
|
|
return data;
|
|
}
|
|
|
|
private static RuleResult Run(IScopedRule<ServerRuleContext> rule, ServerRuleContext context) => Assert.Single(ScopedRuleEngine.Evaluate([rule], context));
|
|
|
|
[Fact]
|
|
public void Every_rule_has_a_unique_well_formed_id_and_a_title()
|
|
{
|
|
var all = ServerRuleCatalog.All;
|
|
|
|
Assert.Equal(all.Count, all.Select(r => r.Id).Distinct().Count());
|
|
Assert.All(all, r => Assert.Matches("^SQLI-(WIN|SYN)-[0-9]{3}$", r.Id));
|
|
Assert.All(all, r => Assert.False(string.IsNullOrWhiteSpace(r.Title)));
|
|
Assert.DoesNotContain(all, r => BuiltInRules.Create().Any(b => b.Id == r.Id));
|
|
}
|
|
|
|
[Fact]
|
|
public void Missing_data_is_not_measurable_with_the_reason()
|
|
{
|
|
var data = new CollectedData();
|
|
data.MarkUnavailable<ServerDiskSnapshot>("Adgang nægtet til root\\cimv2.");
|
|
|
|
var result = Run(new DiskFreeSpaceRule(), Context(data));
|
|
|
|
Assert.Equal(RuleOutcome.NotMeasurable, result.Outcome);
|
|
Assert.Equal("Adgang nægtet til root\\cimv2.", result.Reason);
|
|
}
|
|
|
|
[Fact]
|
|
public void Sustained_high_cpu_is_a_finding_a_burst_is_not()
|
|
{
|
|
var sustained = Run(new ServerCpuRule(), Context(With(Performance(m => Sample(cpu: 95)))));
|
|
var burst = Run(new ServerCpuRule(), Context(With(Performance(m => Sample(cpu: m < 3 ? 100 : 30)))));
|
|
var first = Run(new ServerCpuRule(), Context(With(Performance(_ => Sample(cpu: null)))));
|
|
|
|
Assert.Equal(Severity.High, Assert.Single(sustained.Findings).Severity);
|
|
Assert.Contains("95 %", sustained.Findings[0].Title, StringComparison.Ordinal);
|
|
Assert.Equal(RuleOutcome.Passed, burst.Outcome);
|
|
Assert.Equal(RuleOutcome.NotMeasurable, first.Outcome);
|
|
}
|
|
|
|
[Fact]
|
|
public void Low_memory_needs_both_little_share_and_few_megabytes()
|
|
{
|
|
var low = Run(new ServerMemoryRule(), Context(With(Performance(_ => Sample(availableMb: 300)))));
|
|
var bigServer = Run(new ServerMemoryRule(), Context(With(Performance(_ => new ServerPerformanceSample(Now, 10, 4000, 262144, 0, 40)))));
|
|
|
|
Assert.Equal(Severity.Medium, Assert.Single(low.Findings).Severity);
|
|
Assert.Equal(RuleOutcome.Passed, bigServer.Outcome); // 1,5 % of 256 GB, but 4 GB free
|
|
}
|
|
|
|
[Fact]
|
|
public void Paging_only_counts_with_little_free_memory()
|
|
{
|
|
Assert.Single(Run(new ServerPagingRule(), Context(With(Performance(_ => Sample(availableMb: 900, pages: 5000))))).Findings);
|
|
Assert.Equal(RuleOutcome.Passed, Run(new ServerPagingRule(), Context(With(Performance(_ => Sample(availableMb: 8000, pages: 5000))))).Outcome);
|
|
Assert.Equal(Severity.High, Assert.Single(Run(new ServerCommitRule(), Context(With(Performance(_ => Sample(commit: 98))))).Findings).Severity);
|
|
}
|
|
|
|
[Fact]
|
|
public void Disk_free_space_by_share_unless_there_is_plenty()
|
|
{
|
|
var disks = new ServerDiskSnapshot
|
|
{
|
|
CapturedAt = Now,
|
|
Disks =
|
|
[
|
|
new ServerDisk(@"C:\", "System", "NTFS", 100, 30),
|
|
new ServerDisk(@"D:\", "Data", "NTFS", 500, 30), // 6 %: Medium
|
|
new ServerDisk(@"E:\", "Log", "NTFS", 100, 3), // 3 %: High
|
|
new ServerDisk(@"F:\", "Backup", "NTFS", 4000, 200), // 5 % but 200 GB: fine
|
|
new ServerDisk(@"G:\", "Temp", "NTFS", 50, 0.5), // under 1 GB: Critical
|
|
],
|
|
};
|
|
|
|
var findings = Run(new DiskFreeSpaceRule(), Context(With(disks))).Findings.ToDictionary(f => f.Subject!, f => f.Severity);
|
|
|
|
Assert.Equal(new Dictionary<string, Severity> { [@"disk: D:\"] = Severity.Medium, [@"disk: E:\"] = Severity.High, [@"disk: G:\"] = Severity.Critical }, findings);
|
|
}
|
|
|
|
[Fact]
|
|
public void Forecast_findings_by_days_until_full()
|
|
{
|
|
var disks = new ServerDiskSnapshot
|
|
{
|
|
CapturedAt = Now,
|
|
Disks = [new ServerDisk(@"D:\", null, "NTFS", 500, 100), new ServerDisk(@"E:\", null, "NTFS", 500, 100)],
|
|
Forecasts =
|
|
[
|
|
new DiskForecast(@"D:\", 12, 8, 300, 14, 0.95),
|
|
new DiskForecast(@"E:\", 1.5, 60, 300, 14, 0.95),
|
|
new DiskForecast(@"F:\", null, 0, 300, 14, 0.1, "ingen tydelig tendens"),
|
|
new DiskForecast(@"G:\", 200, 0.5, 300, 14, 0.9),
|
|
],
|
|
};
|
|
|
|
var findings = Run(new DiskForecastRule(), Context(With(disks))).Findings;
|
|
|
|
Assert.Equal(2, findings.Count);
|
|
Assert.Contains(findings, f => f.Title == @"Disken D:\ er fuld om ca. 12 dage" && f.Severity == Severity.Medium);
|
|
Assert.Contains(findings, f => f.Subject == @"forecast: E:\" && f.Severity == Severity.Critical);
|
|
Assert.Equal(RuleOutcome.NotMeasurable, Run(new DiskForecastRule(), Context(With(disks with { Forecasts = [] }))).Outcome);
|
|
}
|
|
|
|
private static ServerServiceSnapshot Services(params ServerService[] services) => new() { CapturedAt = Now, Services = services };
|
|
|
|
[Fact]
|
|
public void Stopped_automatic_services_except_the_ones_that_stop_by_design()
|
|
{
|
|
var data = With(Services(
|
|
new ServerService("AOS60$01", "Microsoft Dynamics AX Object Server 6.0$01", "Stopped", "Auto", ExitCode: 1067),
|
|
new ServerService("Spooler", "Print Spooler", "Running", "Auto"),
|
|
new ServerService("sppsvc", "Software Protection", "Stopped", "Auto", DelayedAutoStart: true),
|
|
new ServerService("clr_optimization_v4.0.30319_64", ".NET Optimization", "Stopped", "Auto"),
|
|
new ServerService("MyBatch", "Batch", "Stopped", "Auto"),
|
|
new ServerService("Manual1", "Manual", "Stopped", "Manual")));
|
|
data.Set(Performance(_ => Sample()));
|
|
|
|
var result = Run(new StoppedAutomaticServiceRule(), Context(data, Server(ignored: ["MyBatch"])));
|
|
|
|
var finding = Assert.Single(result.Findings);
|
|
Assert.Equal("service: AOS60$01", finding.Subject);
|
|
Assert.Contains("1067", finding.Explanation, StringComparison.Ordinal);
|
|
}
|
|
|
|
[Fact]
|
|
public void Right_after_a_restart_stopped_services_are_still_starting()
|
|
{
|
|
var data = With(Services(new ServerService("AOS60$01", "AOS", "Stopped", "Auto")));
|
|
data.Set(Performance(_ => Sample(), boot: Now.AddMinutes(-5)));
|
|
|
|
Assert.Equal(RuleOutcome.Passed, Run(new StoppedAutomaticServiceRule(), Context(data)).Outcome);
|
|
}
|
|
|
|
[Fact]
|
|
public void Watched_services_must_run_and_exist()
|
|
{
|
|
var data = With(Services(new ServerService("AOS60$01", "AOS", "Stopped", "Manual"), new ServerService("W3SVC", "IIS", "Running", "Auto")));
|
|
|
|
var result = Run(new WatchedServiceRule(), Context(data, Server(watched: ["AOS60$01", "W3SVC", "ReportServer"])));
|
|
var none = Run(new WatchedServiceRule(), Context(data));
|
|
var automatic = Run(new StoppedAutomaticServiceRule(), Context(data, Server(watched: ["AOS60$01"])));
|
|
|
|
Assert.Equal(2, result.Findings.Count);
|
|
Assert.Contains(result.Findings, f => f.Subject == "watched: AOS60$01" && f.Severity == Severity.High);
|
|
Assert.Contains(result.Findings, f => f.Subject == "watched: ReportServer" && f.Severity == Severity.Medium);
|
|
Assert.Equal(RuleOutcome.NotApplicable, none.Outcome);
|
|
Assert.Equal(RuleOutcome.Passed, automatic.Outcome); // not reported twice
|
|
}
|
|
|
|
private static ServerEventGroup Group(string source, long id, ServerEventCategory category, int count, int level = 2) =>
|
|
new("System", source, id, level, category, count, Now.AddHours(-2), Now.AddMinutes(-5), "besked med \\\\fil01\\hemmeligt");
|
|
|
|
[Fact]
|
|
public void Events_important_kinds_at_once_ordinary_errors_when_they_repeat()
|
|
{
|
|
var events = new ServerEventSnapshot
|
|
{
|
|
CapturedAt = Now,
|
|
ReadUntil = Now,
|
|
Recent =
|
|
[
|
|
Group("disk", 153, ServerEventCategory.Disk, 1),
|
|
Group("Service Control Manager", 7031, ServerEventCategory.ServiceFailure, 4),
|
|
Group("Some-App", 9, ServerEventCategory.Error, 2),
|
|
Group("Other-App", 3, ServerEventCategory.Error, 12),
|
|
Group("EventLog", 6008, ServerEventCategory.UnexpectedShutdown, 1),
|
|
],
|
|
};
|
|
|
|
var findings = Run(new WindowsEventLogRule(), Context(With(events))).Findings;
|
|
|
|
Assert.Equal(["event: System/disk/153", "event: System/Service Control Manager/7031", "event: System/Other-App/3"], findings.Select(f => f.Subject).ToList());
|
|
Assert.Equal([Severity.High, Severity.High, Severity.Low], findings.Select(f => f.Severity).ToList());
|
|
Assert.True(findings[0].Evidence.Single(e => e.Name == "Eksempel").Sensitive);
|
|
}
|
|
|
|
[Fact]
|
|
public void Event_findings_are_capped()
|
|
{
|
|
var events = new ServerEventSnapshot
|
|
{
|
|
CapturedAt = Now,
|
|
ReadUntil = Now,
|
|
Recent = Enumerable.Range(1, 40).Select(i => Group("App" + i, i, ServerEventCategory.ApplicationCrash, 1)).ToList(),
|
|
};
|
|
|
|
Assert.Equal(15, Run(new WindowsEventLogRule(), Context(With(events))).Findings.Count);
|
|
}
|
|
|
|
[Fact]
|
|
public void Restart_unexpected_with_a_crash_event_otherwise_info()
|
|
{
|
|
var data = With(Performance(_ => Sample(), boot: Now.AddHours(-3)));
|
|
var planned = Run(new ServerRestartRule(), Context(data));
|
|
data.Set(new ServerEventSnapshot
|
|
{
|
|
CapturedAt = Now, ReadUntil = Now,
|
|
Recent = [Group("Microsoft-Windows-Kernel-Power", 41, ServerEventCategory.UnexpectedShutdown, 1, level: 1) with { LastAt = Now.AddHours(-3) }],
|
|
});
|
|
var crash = Run(new ServerRestartRule(), Context(data));
|
|
var old = Run(new ServerRestartRule(), Context(With(Performance(_ => Sample(), boot: Now.AddDays(-3)))));
|
|
|
|
Assert.Equal(Severity.Info, Assert.Single(planned.Findings).Severity);
|
|
Assert.Equal(Severity.High, Assert.Single(crash.Findings).Severity);
|
|
Assert.Contains("uventet", crash.Findings[0].Title, StringComparison.Ordinal);
|
|
Assert.Equal(RuleOutcome.Passed, old.Outcome);
|
|
}
|
|
|
|
[Fact]
|
|
public void Uptime_rule_is_off_by_default()
|
|
{
|
|
var data = With(Performance(_ => Sample(), boot: Now.AddDays(-200)));
|
|
|
|
Assert.Equal(RuleOutcome.NotApplicable, Run(new ServerUptimeRule(), Context(data)).Outcome);
|
|
Assert.Equal(Severity.Low, Assert.Single(Run(new ServerUptimeRule(), Context(data, settings: new ServerRuleSettings { UptimeDays = 120 })).Findings).Severity);
|
|
}
|
|
|
|
[Fact]
|
|
public void Scheduled_tasks_failed_and_missed()
|
|
{
|
|
var tasks = new ServerTaskSnapshot
|
|
{
|
|
CapturedAt = Now,
|
|
Visible = 120,
|
|
Tasks =
|
|
[
|
|
new ServerTask { Path = @"\", Name = "AX export", State = "Ready", LastTaskResult = 1, LastRunTime = Now.AddHours(-1), NextRunTime = Now.AddHours(23) },
|
|
new ServerTask { Path = @"\", Name = "OK", State = "Ready", LastTaskResult = 0, LastRunTime = Now.AddHours(-1), NextRunTime = Now.AddHours(1) },
|
|
new ServerTask { Path = @"\Firma\", Name = "Late", State = "Ready", LastTaskResult = 0, LastRunTime = Now.AddDays(-2), NextRunTime = Now.AddHours(-5) },
|
|
new ServerTask { Path = @"\", Name = "Disabled", State = "Disabled", LastTaskResult = 1, NextRunTime = Now.AddHours(-5) },
|
|
new ServerTask { Path = @"\", Name = "Missed", State = "Ready", LastTaskResult = 0, MissedRuns = 2 },
|
|
new ServerTask { Path = @"\", Name = "Running", State = "Running", LastTaskResult = TaskResults.Running, NextRunTime = Now.AddHours(-5) },
|
|
],
|
|
};
|
|
var data = With(tasks);
|
|
|
|
var failed = Run(new FailedScheduledTaskRule(), Context(data));
|
|
var missed = Run(new MissedScheduledTaskRule(), Context(data));
|
|
|
|
Assert.Equal(@"task: \AX export", Assert.Single(failed.Findings).Subject);
|
|
Assert.Equal([@"task-missed: \Firma\Late", @"task-missed: \Missed"], missed.Findings.Select(f => f.Subject).ToList());
|
|
}
|
|
}
|
|
|
|
public class SyntheticCheckRuleTests
|
|
{
|
|
private static readonly DateTimeOffset Now = new(2026, 10, 2, 12, 0, 0, TimeSpan.Zero);
|
|
|
|
private static SyntheticCheckIdentity Check(string kind = Core.SyntheticChecks.SyntheticCheckKind.Tcp, string? service = null, int? warn = null, bool https = false) =>
|
|
new(7, "AOS01 port", kind, "aos01:2712") { ServerName = "AOS01", ServiceName = service, WarnMilliseconds = warn, Https = https };
|
|
|
|
private static Core.SyntheticChecks.SyntheticCheckResult Result(bool success, int minutesAgo, int ms = 20, string? service = null, DateTimeOffset? certificate = null) =>
|
|
new() { At = Now.AddMinutes(-minutesAgo), Success = success, DurationMs = ms, Message = success ? "ok" : "Forbindelsen blev afvist.", ServiceState = service, CertificateExpiresAt = certificate };
|
|
|
|
private static RuleResult Run(IScopedRule<SyntheticCheckRuleContext> rule, SyntheticCheckIdentity check, params Core.SyntheticChecks.SyntheticCheckResult[] newestFirst) =>
|
|
Assert.Single(ScopedRuleEngine.Evaluate([rule], new SyntheticCheckRuleContext(check, Now, newestFirst, new SyntheticCheckRuleSettings())));
|
|
|
|
[Fact]
|
|
public void One_failure_is_not_an_alarm_two_in_a_row_are()
|
|
{
|
|
Assert.Equal(RuleOutcome.Passed, Run(new SyntheticCheckFailingRule(), Check(), Result(false, 0), Result(true, 5)).Outcome);
|
|
var failing = Run(new SyntheticCheckFailingRule(), Check(), Result(false, 0), Result(false, 5), Result(true, 10));
|
|
Assert.Equal(Severity.High, Assert.Single(failing.Findings).Severity);
|
|
Assert.Equal("AOS01 port fejler: Forbindelsen blev afvist.", failing.Findings[0].Title);
|
|
Assert.Equal(RuleOutcome.NotMeasurable, Run(new SyntheticCheckFailingRule(), Check()).Outcome);
|
|
}
|
|
|
|
[Fact]
|
|
public void Service_running_but_not_answering_is_said_plainly()
|
|
{
|
|
var running = Run(new SyntheticCheckFailingRule(), Check(service: "AOS60$01"), Result(false, 0, service: "Running"), Result(false, 5, service: "Running"));
|
|
var stopped = Run(new SyntheticCheckFailingRule(), Check(service: "AOS60$01"), Result(false, 0, service: "Stopped"), Result(false, 5, service: "Stopped"));
|
|
|
|
Assert.Equal("AOS01 port svarer ikke, selv om tjenesten AOS60$01 kører", running.Findings[0].Title);
|
|
Assert.Contains("en tjenesteovervågning alene ikke ser", running.Findings[0].Explanation, StringComparison.Ordinal);
|
|
Assert.Equal("AOS01 port svarer ikke: tjenesten AOS60$01 er stoppet", stopped.Findings[0].Title);
|
|
}
|
|
|
|
[Fact]
|
|
public void Slow_answers_three_times_in_a_row()
|
|
{
|
|
Assert.Equal(RuleOutcome.NotApplicable, Run(new SyntheticCheckSlowRule(), Check(), Result(true, 0, 5000)).Outcome);
|
|
var slow = Run(new SyntheticCheckSlowRule(), Check(warn: 1000), Result(true, 0, 1500), Result(true, 5, 2500), Result(true, 10, 1200));
|
|
Assert.Equal(Severity.Medium, Assert.Single(slow.Findings).Severity);
|
|
Assert.Equal(RuleOutcome.Passed, Run(new SyntheticCheckSlowRule(), Check(warn: 1000), Result(true, 0, 1500), Result(true, 5, 500), Result(true, 10, 1200)).Outcome);
|
|
}
|
|
|
|
[Fact]
|
|
public void Certificate_expiry()
|
|
{
|
|
var check = Check(Core.SyntheticChecks.SyntheticCheckKind.Http, https: true);
|
|
|
|
Assert.Equal(RuleOutcome.Passed, Run(new CertificateExpiryRule(), check, Result(true, 0, certificate: Now.AddDays(60))).Outcome);
|
|
Assert.Equal(Severity.Medium, Run(new CertificateExpiryRule(), check, Result(true, 0, certificate: Now.AddDays(20))).Findings[0].Severity);
|
|
Assert.Equal(Severity.High, Run(new CertificateExpiryRule(), check, Result(true, 0, certificate: Now.AddDays(3))).Findings[0].Severity);
|
|
Assert.Equal(Severity.Critical, Run(new CertificateExpiryRule(), check, Result(false, 0, certificate: Now.AddDays(-1))).Findings[0].Severity);
|
|
Assert.Equal(RuleOutcome.NotApplicable, Run(new CertificateExpiryRule(), Check(), Result(true, 0)).Outcome);
|
|
Assert.Equal(RuleOutcome.NotMeasurable, Run(new CertificateExpiryRule(), check, Result(false, 0)).Outcome);
|
|
}
|
|
}
|