Rules: Windows servers (SQLI-WIN-001..013) and synthetic checks (SQLI-SYN-001..003)
Scoped rules with their own context (a Windows server, a synthetic check) that produce the same findings and outcomes as the instance rules; missing data is "ikke målbar" with the collector's reason. Thresholds are settings (SQLInsight:Servers, SQLInsight:SyntheticChecks). - Sustained CPU, low memory (share and MB), paging with low memory, commit charge. - Free disk space and "fuld om ca. N dage" forecasts. - Stopped automatic services (minus those that stop by design, and a grace period after a restart) and watched services that must run. - Windows event log groups: important kinds at once, ordinary errors when they repeat, capped per server; sample messages are sensitive. - Unexpected restart (Kernel-Power 41, EventLog 6008, BugCheck), optional uptime. - Scheduled tasks that failed or did not run as planned. - Synthetic checks failing (twice in a row), "svarer ikke, selv om tjenesten kører", slow answers, TLS certificates about to expire. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
ae20954a91
commit
ba7c751b5c
5 changed files with 1331 additions and 0 deletions
90
src/SQLInsight.Rules/Servers/ScopedRules.cs
Normal file
90
src/SQLInsight.Rules/Servers/ScopedRules.cs
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
using SQLInsight.Core.Collection;
|
||||
using SQLInsight.Core.Findings;
|
||||
using SQLInsight.Rules.Engine;
|
||||
|
||||
namespace SQLInsight.Rules.Servers;
|
||||
|
||||
/// <summary>
|
||||
/// A rule about something that is not a SQL Server instance (a Windows server, a synthetic check). It produces the
|
||||
/// same findings, explanations and outcomes as the instance rules, against its own kind of context.
|
||||
/// </summary>
|
||||
public interface IScopedRule<in TContext>
|
||||
{
|
||||
/// <summary>Stable identifier, e.g. <c>SQLI-WIN-001</c>. Never reused for a different check.</summary>
|
||||
string Id { get; }
|
||||
|
||||
string Title { get; }
|
||||
|
||||
RuleEvaluation Evaluate(TContext context);
|
||||
}
|
||||
|
||||
public abstract class ScopedRule<TContext> : IScopedRule<TContext>
|
||||
{
|
||||
public abstract string Id { get; }
|
||||
public abstract string Title { get; }
|
||||
|
||||
public abstract RuleEvaluation Evaluate(TContext context);
|
||||
|
||||
protected Finding CreateFinding(Severity severity, string title, string explanation, string recommendation, IEnumerable<Evidence>? evidence = null,
|
||||
string? subject = null, params Reference[] references) => new()
|
||||
{
|
||||
RuleId = Id,
|
||||
Severity = severity,
|
||||
Title = title,
|
||||
Subject = subject,
|
||||
Explanation = explanation,
|
||||
Recommendation = recommendation,
|
||||
Evidence = evidence?.ToList() ?? [],
|
||||
References = references,
|
||||
};
|
||||
|
||||
protected static RuleEvaluation Result(IEnumerable<Finding> findings)
|
||||
{
|
||||
var list = findings.ToArray();
|
||||
return list.Length == 0 ? RuleEvaluation.Passed() : RuleEvaluation.Found(list);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Runs scoped rules. Like <see cref="RuleEngine"/>: a rule that cannot run is not measurable (with the reason) or an error, never silently passed.</summary>
|
||||
public static class ScopedRuleEngine
|
||||
{
|
||||
public static IReadOnlyList<RuleResult> Evaluate<TContext>(IEnumerable<IScopedRule<TContext>> rules, TContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(rules);
|
||||
return rules.Select(rule => Evaluate(rule, context)).ToList();
|
||||
}
|
||||
|
||||
private static RuleResult Evaluate<TContext>(IScopedRule<TContext> rule, TContext context)
|
||||
{
|
||||
try
|
||||
{
|
||||
var evaluation = rule.Evaluate(context);
|
||||
return new RuleResult { RuleId = rule.Id, RuleTitle = rule.Title, Outcome = evaluation.Outcome, Findings = evaluation.Findings, Reason = evaluation.Reason };
|
||||
}
|
||||
catch (DataUnavailableException ex)
|
||||
{
|
||||
return new RuleResult
|
||||
{
|
||||
RuleId = rule.Id,
|
||||
RuleTitle = rule.Title,
|
||||
Outcome = ex.NotApplicable ? RuleOutcome.NotApplicable : RuleOutcome.NotMeasurable,
|
||||
Reason = ex.Message,
|
||||
};
|
||||
}
|
||||
#pragma warning disable CA1031 // One faulty rule must not stop the others; the failure is reported, not hidden.
|
||||
catch (Exception ex)
|
||||
#pragma warning restore CA1031
|
||||
{
|
||||
return new RuleResult { RuleId = rule.Id, RuleTitle = rule.Title, Outcome = RuleOutcome.Error, Reason = $"{ex.GetType().Name}: {ex.Message}" };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Every rule about Windows servers and synthetic checks, for rule listings.</summary>
|
||||
public static class ServerRuleCatalog
|
||||
{
|
||||
public static IReadOnlyList<(string Id, string Title)> All =>
|
||||
WindowsServerRules.Create().Select(r => (r.Id, r.Title))
|
||||
.Concat(SyntheticCheckRules.Create().Select(r => (r.Id, r.Title)))
|
||||
.ToList();
|
||||
}
|
||||
106
src/SQLInsight.Rules/Servers/ServerRuleSettings.cs
Normal file
106
src/SQLInsight.Rules/Servers/ServerRuleSettings.cs
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
using SQLInsight.Core.Collection;
|
||||
using SQLInsight.Core.Servers;
|
||||
|
||||
namespace SQLInsight.Rules.Servers;
|
||||
|
||||
/// <summary>
|
||||
/// Thresholds of the Windows server rules (SQLInsight:Servers in the agent's appsettings.json). Not sealed: the agent's
|
||||
/// options add how and how often to collect.
|
||||
/// </summary>
|
||||
public class ServerRuleSettings
|
||||
{
|
||||
public CpuThresholds Cpu { get; set; } = new();
|
||||
public MemoryThresholds Memory { get; set; } = new();
|
||||
public DiskThresholds Disks { get; set; } = new();
|
||||
public ServiceThresholds Services { get; set; } = new();
|
||||
public EventLogThresholds EventLog { get; set; } = new();
|
||||
public TaskThresholds Tasks { get; set; } = new();
|
||||
|
||||
/// <summary>A server not restarted for more than this many days is a (low) finding: it most likely lacks Windows updates. 0: off.</summary>
|
||||
public int UptimeDays { get; set; }
|
||||
}
|
||||
|
||||
public sealed class CpuThresholds
|
||||
{
|
||||
/// <summary>Average CPU at or above this (percent) …</summary>
|
||||
public double HighPercent { get; set; } = 90;
|
||||
|
||||
/// <summary>… for this many minutes is a finding (SQLI-WIN-001).</summary>
|
||||
public int SustainedMinutes { get; set; } = 15;
|
||||
}
|
||||
|
||||
public sealed class MemoryThresholds
|
||||
{
|
||||
/// <summary>Available memory below this share (percent) …</summary>
|
||||
public double AvailablePercent { get; set; } = 5;
|
||||
|
||||
/// <summary>… and below this many MB is a finding (SQLI-WIN-002). Both, so big SQL Server hosts with little left by design are not flagged.</summary>
|
||||
public double AvailableMegabytes { get; set; } = 1024;
|
||||
|
||||
/// <summary>Hard page faults per second above this, while memory is low, is paging (SQLI-WIN-003).</summary>
|
||||
public double PagesPerSecond { get; set; } = 1000;
|
||||
|
||||
/// <summary>Committed memory at or above this share of the commit limit (percent) is a finding (SQLI-WIN-004).</summary>
|
||||
public double CommitPercent { get; set; } = 90;
|
||||
}
|
||||
|
||||
public sealed class DiskThresholds
|
||||
{
|
||||
/// <summary>Free space below this (percent) is a finding (SQLI-WIN-005) …</summary>
|
||||
public double FreePercent { get; set; } = 10;
|
||||
|
||||
/// <summary>… High below this …</summary>
|
||||
public double CriticalFreePercent { get; set; } = 5;
|
||||
|
||||
/// <summary>… unless there is at least this much free (GB): a big volume with a small share free can be fine.</summary>
|
||||
public double EnoughFreeGigabytes { get; set; } = 50;
|
||||
|
||||
/// <summary>A volume forecast to be full within this many days is a finding (SQLI-WIN-006); within 7 days High, 2 days Critical.</summary>
|
||||
public int ForecastDays { get; set; } = 30;
|
||||
|
||||
/// <summary>How many days of SQLInsight's own samples the forecast is computed from.</summary>
|
||||
public int ForecastHistoryDays { get; set; } = 14;
|
||||
}
|
||||
|
||||
public sealed class ServiceThresholds
|
||||
{
|
||||
/// <summary>
|
||||
/// Services that may be stopped although they start automatically (they stop by design), in addition to the
|
||||
/// built-in list. Name, or a prefix ending in *.
|
||||
/// </summary>
|
||||
public IList<string> Ignore { get; } = [];
|
||||
|
||||
/// <summary>Right after a restart services are still starting: no finding for this many minutes.</summary>
|
||||
public int StartupGraceMinutes { get; set; } = 15;
|
||||
}
|
||||
|
||||
public sealed class EventLogThresholds
|
||||
{
|
||||
/// <summary>An ordinary error must repeat this many times in 24 hours before it is a finding; the important kinds count at once.</summary>
|
||||
public int RepeatThreshold { get; set; } = 5;
|
||||
|
||||
/// <summary>Events left out as noise, in addition to the built-in list: "Kilde:id", "Kilde" or "Kilde*".</summary>
|
||||
public IList<string> Ignore { get; } = [];
|
||||
|
||||
/// <summary>At most this many event findings per server, most important first.</summary>
|
||||
public int MaxFindings { get; set; } = 15;
|
||||
}
|
||||
|
||||
public sealed class TaskThresholds
|
||||
{
|
||||
/// <summary>Also watch Microsoft's own tasks (\Microsoft\…). Off: there are hundreds, and their failures are rarely yours.</summary>
|
||||
public bool IncludeMicrosoftTasks { get; set; }
|
||||
|
||||
/// <summary>A task whose next run time is more than this many minutes in the past has not run as planned (SQLI-WIN-013).</summary>
|
||||
public int LateMinutes { get; set; } = 60;
|
||||
|
||||
/// <summary>Tasks never watched (path\name, or a prefix ending in *).</summary>
|
||||
public IList<string> Ignore { get; } = [];
|
||||
}
|
||||
|
||||
/// <summary>What the Windows server rules see: the server, the latest data per area, and the thresholds.</summary>
|
||||
public sealed record ServerRuleContext(ServerIdentity Server, DateTimeOffset Now, CollectedData Data, ServerRuleSettings Settings)
|
||||
{
|
||||
/// <summary>The interval performance samples are taken at, for "sustained" averages.</summary>
|
||||
public TimeSpan PerformanceInterval { get; init; } = ServerArea.DefaultInterval(ServerArea.Performance);
|
||||
}
|
||||
207
src/SQLInsight.Rules/Servers/SyntheticCheckRules.cs
Normal file
207
src/SQLInsight.Rules/Servers/SyntheticCheckRules.cs
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
using System.Globalization;
|
||||
using SQLInsight.Core.Findings;
|
||||
using SQLInsight.Core.SyntheticChecks;
|
||||
using SQLInsight.Rules.Engine;
|
||||
|
||||
namespace SQLInsight.Rules.Servers;
|
||||
|
||||
/// <summary>Thresholds of the synthetic check rules (SQLInsight:SyntheticChecks in the agent's appsettings.json).</summary>
|
||||
public class SyntheticCheckRuleSettings
|
||||
{
|
||||
/// <summary>A check must fail this many times in a row before it is a finding, so one lost packet is not an alarm.</summary>
|
||||
public int FailuresBeforeFinding { get; set; } = 2;
|
||||
|
||||
/// <summary>A slow answer (above the check's limit) this many times in a row is a finding.</summary>
|
||||
public int SlowRunsBeforeFinding { get; set; } = 3;
|
||||
|
||||
/// <summary>A TLS certificate expiring within this many days is a finding …</summary>
|
||||
public int CertificateWarningDays { get; set; } = 21;
|
||||
|
||||
/// <summary>… High within this many days.</summary>
|
||||
public int CertificateCriticalDays { get; set; } = 7;
|
||||
}
|
||||
|
||||
/// <summary>A configured check as the rules see it.</summary>
|
||||
public sealed record SyntheticCheckIdentity(int CheckId, string Name, string Kind, string Target)
|
||||
{
|
||||
public string? ServerName { get; init; }
|
||||
public string? ServiceName { get; init; }
|
||||
public int? WarnMilliseconds { get; init; }
|
||||
public bool Https { get; init; }
|
||||
}
|
||||
|
||||
/// <param name="Recent">The latest results, newest first.</param>
|
||||
public sealed record SyntheticCheckRuleContext(SyntheticCheckIdentity Check, DateTimeOffset Now, IReadOnlyList<SyntheticCheckResult> Recent, SyntheticCheckRuleSettings Settings);
|
||||
|
||||
public static class SyntheticCheckRules
|
||||
{
|
||||
public static IReadOnlyList<IScopedRule<SyntheticCheckRuleContext>> Create() => [new SyntheticCheckFailingRule(), new SyntheticCheckSlowRule(), new CertificateExpiryRule()];
|
||||
|
||||
internal static string Time(DateTimeOffset value) => value.UtcDateTime.ToString("yyyy-MM-dd HH:mm 'UTC'", CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
public sealed class SyntheticCheckFailingRule : ScopedRule<SyntheticCheckRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-SYN-001";
|
||||
public override string Title => "Et syntetisk tjek fejler";
|
||||
|
||||
public override RuleEvaluation Evaluate(SyntheticCheckRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
if (context.Recent.Count == 0)
|
||||
{
|
||||
return RuleEvaluation.NotMeasurable("Tjekket har ikke kørt endnu.");
|
||||
}
|
||||
|
||||
var needed = Math.Max(1, context.Settings.FailuresBeforeFinding);
|
||||
var failures = context.Recent.TakeWhile(r => !r.Success).ToList();
|
||||
if (failures.Count < needed)
|
||||
{
|
||||
return RuleEvaluation.Passed();
|
||||
}
|
||||
|
||||
var check = context.Check;
|
||||
var latest = failures[0];
|
||||
var serviceRunning = string.Equals(latest.ServiceState, "Running", StringComparison.OrdinalIgnoreCase);
|
||||
var serviceKnown = check.ServiceName is not null && latest.ServiceState is not null;
|
||||
var title = serviceKnown && serviceRunning
|
||||
? $"{check.Name} svarer ikke, selv om tjenesten {check.ServiceName} kører"
|
||||
: serviceKnown
|
||||
? $"{check.Name} svarer ikke: tjenesten {check.ServiceName} er {StateText(latest.ServiceState!)}"
|
||||
: $"{check.Name} fejler: {latest.Message}";
|
||||
var explanation = What(check.Kind)
|
||||
+ (serviceKnown && serviceRunning
|
||||
? $" Tjenesten {check.ServiceName} på {check.ServerName} kører, så Windows ser alt som i orden, men den svarer ikke: den kan hænge, være overbelastet, "
|
||||
+ "vente på databasen eller være blokeret af en firewall. Det er netop den slags fejl, en tjenesteovervågning alene ikke ser."
|
||||
: serviceKnown ? $" Tjenesten {check.ServiceName} på {check.ServerName} kører ikke." : string.Empty);
|
||||
var evidence = new List<Evidence>
|
||||
{
|
||||
new("Fejl", latest.Message, Sensitive: true),
|
||||
new("Mål", check.Target),
|
||||
new("Fejlet i træk", failures.Count.ToString(CultureInfo.InvariantCulture)),
|
||||
new("Fejler siden", SyntheticCheckRules.Time(failures[^1].At)),
|
||||
new("Seneste forsøg", SyntheticCheckRules.Time(latest.At)),
|
||||
};
|
||||
if (check.ServiceName is not null)
|
||||
{
|
||||
evidence.Add(new Evidence("Tjeneste", $"{check.ServiceName}: {(latest.ServiceState is { } state ? StateText(state) : "kunne ikke aflæses")}"));
|
||||
}
|
||||
|
||||
evidence.AddRange(latest.Details.Select(d => new Evidence(d.Name, d.Value, d.Sensitive)));
|
||||
return RuleEvaluation.Found(CreateFinding(
|
||||
Severity.High,
|
||||
title,
|
||||
explanation,
|
||||
Recommendation(check.Kind, serviceKnown && serviceRunning),
|
||||
evidence,
|
||||
subject: "check"));
|
||||
}
|
||||
|
||||
private static string StateText(string state) => state switch
|
||||
{
|
||||
"Running" => "kører",
|
||||
"Stopped" => "stoppet",
|
||||
"Start Pending" => "ved at starte",
|
||||
"Stop Pending" => "ved at stoppe",
|
||||
"Paused" => "sat på pause",
|
||||
_ => state,
|
||||
};
|
||||
|
||||
private static string What(string kind) => kind switch
|
||||
{
|
||||
SyntheticCheckKind.Tcp => "Porten svarer ikke: de programmer og brugere, der forbinder til den, kan heller ikke.",
|
||||
SyntheticCheckKind.Http => "Siden svarer ikke som forventet: brugere og integrationer, der bruger den, får fejl eller venter.",
|
||||
SyntheticCheckKind.Sql => "Login eller forespørgsel mod SQL Server fejler: applikationer, der bruger samme vej ind, kan heller ikke arbejde.",
|
||||
SyntheticCheckKind.Dns => "Navnet kan ikke slås op (eller peger forkert hen): alt, der forbinder via navnet, fejler eller rammer den forkerte server.",
|
||||
SyntheticCheckKind.AxLogon => "Dynamics AX kan ikke nås: AOS-porten svarer ikke, eller et logon gennem Business Connector fejler. Brugere og batchjob kan ikke arbejde.",
|
||||
_ => "Tjekket fejler.",
|
||||
};
|
||||
|
||||
private static string Recommendation(string kind, bool serviceRunning) =>
|
||||
(serviceRunning ? "Se tjenestens egen log og Windows' hændelseslog på serveren; en tjeneste, der kører men ikke svarer, skal ofte genstartes af en driftsansvarlig. " : string.Empty)
|
||||
+ kind switch
|
||||
{
|
||||
SyntheticCheckKind.Tcp => "Tjek at serveren og tjenesten kører, og at firewallen tillader porten fra agentens server.",
|
||||
SyntheticCheckKind.Http => "Åbn adressen i en browser fra agentens server, og se IIS-/webserverens log og applikationspuljen.",
|
||||
SyntheticCheckKind.Sql => "Prøv at logge ind med samme login fra agentens server, og se SQL Server-fundene og fejlloggen (fx fejl 18456).",
|
||||
SyntheticCheckKind.Dns => "Slå navnet op med nslookup fra agentens server, og tjek DNS-serverne og posterne.",
|
||||
SyntheticCheckKind.AxLogon => "Tjek AOS-tjenesten og dens hændelseslog, at AX-databasen svarer, og at agentens konto er AX-bruger, hvis logon testes.",
|
||||
_ => string.Empty,
|
||||
} + " SQLInsight genstarter aldrig selv noget.";
|
||||
}
|
||||
|
||||
public sealed class SyntheticCheckSlowRule : ScopedRule<SyntheticCheckRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-SYN-002";
|
||||
public override string Title => "Et syntetisk tjek svarer langsomt";
|
||||
|
||||
public override RuleEvaluation Evaluate(SyntheticCheckRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
if (context.Check.WarnMilliseconds is not { } limit)
|
||||
{
|
||||
return RuleEvaluation.NotApplicable("Der er ikke sat en grænse for svartid på tjekket.");
|
||||
}
|
||||
|
||||
var needed = Math.Max(1, context.Settings.SlowRunsBeforeFinding);
|
||||
var latest = context.Recent.Take(needed).ToList();
|
||||
if (latest.Count < needed || latest.Any(r => !r.Success || r.DurationMs <= limit))
|
||||
{
|
||||
return context.Recent.Count == 0 ? RuleEvaluation.NotMeasurable("Tjekket har ikke kørt endnu.") : RuleEvaluation.Passed();
|
||||
}
|
||||
|
||||
return RuleEvaluation.Found(CreateFinding(
|
||||
Severity.Medium,
|
||||
$"{context.Check.Name} svarer langsomt ({latest.Min(r => r.DurationMs).ToString("N0", CultureInfo.GetCultureInfo("da-DK"))}–{latest.Max(r => r.DurationMs).ToString("N0", CultureInfo.GetCultureInfo("da-DK"))} ms)",
|
||||
"Tjekket lykkes, men svaret har de seneste gange taget længere end grænsen. Brugerne oplever det som en langsom applikation, længe før den holder op med at svare.",
|
||||
"Se belastningen på serveren (CPU, hukommelse, disk) og på databasen bag i samme tidsrum, fx med Analyse af tidsrum.",
|
||||
[
|
||||
new Evidence("Svartider", string.Join(", ", latest.Select(r => r.DurationMs.ToString(CultureInfo.InvariantCulture) + " ms"))),
|
||||
new Evidence("Grænse", limit.ToString(CultureInfo.InvariantCulture) + " ms"),
|
||||
new Evidence("Mål", context.Check.Target),
|
||||
],
|
||||
subject: "slow"));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class CertificateExpiryRule : ScopedRule<SyntheticCheckRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-SYN-003";
|
||||
public override string Title => "TLS-certifikatet udløber snart";
|
||||
|
||||
public override RuleEvaluation Evaluate(SyntheticCheckRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
if (context.Check.Kind != SyntheticCheckKind.Http || !context.Check.Https)
|
||||
{
|
||||
return RuleEvaluation.NotApplicable("Kun for HTTPS-tjek.");
|
||||
}
|
||||
|
||||
var latest = context.Recent.FirstOrDefault(r => r.CertificateExpiresAt is not null);
|
||||
if (latest?.CertificateExpiresAt is not { } expires)
|
||||
{
|
||||
return RuleEvaluation.NotMeasurable("Certifikatet er ikke set endnu (tjekket har ikke fået forbindelse).");
|
||||
}
|
||||
|
||||
var days = (expires - context.Now).TotalDays;
|
||||
if (days > context.Settings.CertificateWarningDays)
|
||||
{
|
||||
return RuleEvaluation.Passed();
|
||||
}
|
||||
|
||||
var severity = days <= 0 ? Severity.Critical : days <= context.Settings.CertificateCriticalDays ? Severity.High : Severity.Medium;
|
||||
return RuleEvaluation.Found(CreateFinding(
|
||||
severity,
|
||||
days <= 0
|
||||
? $"Certifikatet for {context.Check.Name} er udløbet"
|
||||
: $"Certifikatet for {context.Check.Name} udløber om {Math.Floor(days).ToString(CultureInfo.InvariantCulture)} dage",
|
||||
"Når certifikatet udløber, afviser browsere og integrationer forbindelsen (eller viser en sikkerhedsadvarsel), og siden er i praksis nede.",
|
||||
"Forny certifikatet og bind det nye i webserveren (IIS: Bindinger på webstedet). Tjek også, om andre servere bruger samme certifikat.",
|
||||
[
|
||||
new Evidence("Udløber", SyntheticCheckRules.Time(expires)),
|
||||
new Evidence("Certifikat", latest.CertificateSubject ?? "–"),
|
||||
new Evidence("Adresse", context.Check.Target),
|
||||
],
|
||||
subject: "certificate"));
|
||||
}
|
||||
}
|
||||
594
src/SQLInsight.Rules/Servers/WindowsServerRules.cs
Normal file
594
src/SQLInsight.Rules/Servers/WindowsServerRules.cs
Normal file
|
|
@ -0,0 +1,594 @@
|
|||
using System.Globalization;
|
||||
using SQLInsight.Core.Findings;
|
||||
using SQLInsight.Core.Servers;
|
||||
using SQLInsight.Rules.Engine;
|
||||
|
||||
namespace SQLInsight.Rules.Servers;
|
||||
|
||||
/// <summary>The rules about Windows servers (SQL Server hosts, AOS and other application servers, file servers).</summary>
|
||||
public static class WindowsServerRules
|
||||
{
|
||||
public static IReadOnlyList<IScopedRule<ServerRuleContext>> Create() =>
|
||||
[
|
||||
new ServerCpuRule(),
|
||||
new ServerMemoryRule(),
|
||||
new ServerPagingRule(),
|
||||
new ServerCommitRule(),
|
||||
new DiskFreeSpaceRule(),
|
||||
new DiskForecastRule(),
|
||||
new StoppedAutomaticServiceRule(),
|
||||
new WatchedServiceRule(),
|
||||
new WindowsEventLogRule(),
|
||||
new ServerRestartRule(),
|
||||
new ServerUptimeRule(),
|
||||
new FailedScheduledTaskRule(),
|
||||
new MissedScheduledTaskRule(),
|
||||
];
|
||||
|
||||
internal static string Number(double value, int decimals = 0) => value.ToString("N" + decimals.ToString(CultureInfo.InvariantCulture), CultureInfo.GetCultureInfo("da-DK"));
|
||||
|
||||
internal static string Time(DateTimeOffset? value) => value is { } v ? v.UtcDateTime.ToString("yyyy-MM-dd HH:mm 'UTC'", CultureInfo.InvariantCulture) : "–";
|
||||
|
||||
internal static Reference Learn(string title, string path) => new(title, new Uri("https://learn.microsoft.com/" + path));
|
||||
|
||||
/// <summary>A name in a list where entries may end in * (prefix).</summary>
|
||||
internal static bool Matches(string name, IEnumerable<string> patterns) => patterns.Any(p =>
|
||||
p.EndsWith('*') ? name.StartsWith(p[..^1], StringComparison.OrdinalIgnoreCase) : name.Equals(p, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
public sealed class ServerCpuRule : ScopedRule<ServerRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-WIN-001";
|
||||
public override string Title => "CPU-forbruget er højt i længere tid";
|
||||
|
||||
public override RuleEvaluation Evaluate(ServerRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var performance = context.Data.Get<ServerPerformanceSnapshot>();
|
||||
var settings = context.Settings.Cpu;
|
||||
var period = TimeSpan.FromMinutes(Math.Max(1, settings.SustainedMinutes));
|
||||
var average = ServerPerformanceMath.SustainedAverage(performance.Samples, s => s.CpuPercent, period, context.Now, context.PerformanceInterval);
|
||||
if (average is null)
|
||||
{
|
||||
return !performance.Samples.Any(s => s.CpuPercent is not null)
|
||||
? RuleEvaluation.NotMeasurable("CPU måles som forskellen mellem to målinger; der er endnu ikke to.")
|
||||
: RuleEvaluation.Passed();
|
||||
}
|
||||
|
||||
if (average < settings.HighPercent)
|
||||
{
|
||||
return RuleEvaluation.Passed();
|
||||
}
|
||||
|
||||
var max = performance.Samples.Where(s => s.At > context.Now - period).Max(s => s.CpuPercent ?? 0);
|
||||
return RuleEvaluation.Found(CreateFinding(
|
||||
average >= 98 ? Severity.Critical : Severity.High,
|
||||
$"CPU har været {WindowsServerRules.Number(average.Value)} % i gennemsnit de seneste {settings.SustainedMinutes} minutter",
|
||||
"Serverens processorer har været næsten fuldt belastet i længere tid. Alt på serveren bliver langsommere: forespørgsler, AOS-kald, "
|
||||
+ "fildeling og logon. Ved kortvarige toppe er det normalt; vedvarende høj CPU er det ikke.",
|
||||
"Find processen med Jobliste/Ressourceovervågning eller Performance Monitor (Process(*)\\% Processor Time). På en SQL Server-host: se Top-SQL for "
|
||||
+ "instansen i samme tidsrum. På en AOS: se batchjob og brugere i AX. Tjek også, om antivirus scanner database- eller AX-filer.",
|
||||
[
|
||||
new Evidence("Gennemsnit", WindowsServerRules.Number(average.Value, 1) + " %"),
|
||||
new Evidence("Højeste minut", WindowsServerRules.Number(max, 1) + " %"),
|
||||
new Evidence("Periode", $"{settings.SustainedMinutes} minutter"),
|
||||
new Evidence("Grænse", WindowsServerRules.Number(settings.HighPercent) + " %"),
|
||||
new Evidence("Logiske processorer", performance.Os.LogicalProcessors?.ToString(CultureInfo.InvariantCulture) ?? "–"),
|
||||
],
|
||||
subject: "cpu"));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ServerMemoryRule : ScopedRule<ServerRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-WIN-002";
|
||||
public override string Title => "Serveren har næsten ingen ledig hukommelse";
|
||||
|
||||
public override RuleEvaluation Evaluate(ServerRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var performance = context.Data.Get<ServerPerformanceSnapshot>();
|
||||
var settings = context.Settings.Memory;
|
||||
var period = TimeSpan.FromMinutes(5);
|
||||
var availableMb = ServerPerformanceMath.SustainedAverage(performance.Samples, s => s.AvailableMb, period, context.Now, context.PerformanceInterval);
|
||||
var availablePercent = ServerPerformanceMath.SustainedAverage(performance.Samples, s => s.AvailablePercent, period, context.Now, context.PerformanceInterval);
|
||||
if (availableMb is null || availablePercent is null)
|
||||
{
|
||||
return performance.Samples.Any(s => s.AvailableMb is not null) ? RuleEvaluation.Passed() : RuleEvaluation.NotMeasurable("Ledig hukommelse kunne ikke aflæses.");
|
||||
}
|
||||
|
||||
if (availablePercent >= settings.AvailablePercent || availableMb >= settings.AvailableMegabytes)
|
||||
{
|
||||
return RuleEvaluation.Passed();
|
||||
}
|
||||
|
||||
var severe = availablePercent < 2 && availableMb < 256;
|
||||
return RuleEvaluation.Found(CreateFinding(
|
||||
severe ? Severity.High : Severity.Medium,
|
||||
$"Kun {WindowsServerRules.Number(availableMb.Value)} MB ledig hukommelse ({WindowsServerRules.Number(availablePercent.Value, 1)} %)",
|
||||
"Når Windows løber tør for ledig hukommelse, flyttes data til sidefilen på disken, og alt bliver langsomt. Processer kan fejle med "
|
||||
+ "\"out of memory\", og SQL Server kan blive tvunget til at give hukommelse tilbage.",
|
||||
"Find den proces, der bruger hukommelsen (Jobliste, kolonnen Hukommelse/Commit). På en SQL Server-host: sæt max server memory, så Windows har "
|
||||
+ "mindst 2–4 GB (mere på store servere). På en AOS eller terminalserver: se antal sessioner og brugere. Overvej mere RAM.",
|
||||
[
|
||||
new Evidence("Ledig hukommelse (5 min.)", WindowsServerRules.Number(availableMb.Value) + " MB"),
|
||||
new Evidence("Andel ledig", WindowsServerRules.Number(availablePercent.Value, 1) + " %"),
|
||||
new Evidence("Hukommelse i alt", performance.Os.TotalMemoryMb is { } total ? WindowsServerRules.Number(total) + " MB" : "–"),
|
||||
new Evidence("Grænse", $"under {WindowsServerRules.Number(settings.AvailablePercent)} % og {WindowsServerRules.Number(settings.AvailableMegabytes)} MB"),
|
||||
],
|
||||
subject: "memory",
|
||||
WindowsServerRules.Learn("Server memory configuration options", "sql/database-engine/configure-windows/server-memory-server-configuration-options")));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ServerPagingRule : ScopedRule<ServerRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-WIN-003";
|
||||
public override string Title => "Serveren pager hukommelse til disk";
|
||||
|
||||
public override RuleEvaluation Evaluate(ServerRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var performance = context.Data.Get<ServerPerformanceSnapshot>();
|
||||
var settings = context.Settings.Memory;
|
||||
var period = TimeSpan.FromMinutes(15);
|
||||
var pages = ServerPerformanceMath.SustainedAverage(performance.Samples, s => s.PagesPerSecond, period, context.Now, context.PerformanceInterval);
|
||||
var available = ServerPerformanceMath.SustainedAverage(performance.Samples, s => s.AvailablePercent, period, context.Now, context.PerformanceInterval);
|
||||
if (pages is null)
|
||||
{
|
||||
return performance.Samples.Any(s => s.PagesPerSecond is not null)
|
||||
? RuleEvaluation.Passed()
|
||||
: RuleEvaluation.NotMeasurable("Paging måles som forskellen mellem to målinger af Memory\\Pages/sec; der er endnu ikke to.");
|
||||
}
|
||||
|
||||
// Pages/sec alone is also file I/O through memory-mapped files; together with little free memory it is paging.
|
||||
if (pages < settings.PagesPerSecond || available is null or >= 10)
|
||||
{
|
||||
return RuleEvaluation.Passed();
|
||||
}
|
||||
|
||||
return RuleEvaluation.Found(CreateFinding(
|
||||
Severity.Medium,
|
||||
$"Serveren pager {WindowsServerRules.Number(pages.Value)} sider pr. sekund med lidt ledig hukommelse",
|
||||
"Windows læser og skriver hukommelsessider til sidefilen, fordi der ikke er plads i RAM. Det er mange gange langsommere end hukommelse og "
|
||||
+ "belaster disken, som andre også bruger.",
|
||||
"Se SQLI-WIN-002 for hvem der bruger hukommelsen. Reducér forbruget (max server memory, færre sessioner) eller tilføj RAM.",
|
||||
[
|
||||
new Evidence("Pages/sec (15 min.)", WindowsServerRules.Number(pages.Value)),
|
||||
new Evidence("Andel ledig hukommelse", WindowsServerRules.Number(available.Value, 1) + " %"),
|
||||
new Evidence("Grænse", WindowsServerRules.Number(settings.PagesPerSecond) + " pages/sec"),
|
||||
],
|
||||
subject: "paging"));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ServerCommitRule : ScopedRule<ServerRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-WIN-004";
|
||||
public override string Title => "Den reserverede hukommelse (commit) er tæt på grænsen";
|
||||
|
||||
public override RuleEvaluation Evaluate(ServerRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var performance = context.Data.Get<ServerPerformanceSnapshot>();
|
||||
var commit = ServerPerformanceMath.SustainedAverage(performance.Samples, s => s.CommitPercent, TimeSpan.FromMinutes(5), context.Now, context.PerformanceInterval);
|
||||
if (commit is null)
|
||||
{
|
||||
return performance.Samples.Any(s => s.CommitPercent is not null) ? RuleEvaluation.Passed() : RuleEvaluation.NotMeasurable("Commit-grænsen kunne ikke aflæses.");
|
||||
}
|
||||
|
||||
if (commit < context.Settings.Memory.CommitPercent)
|
||||
{
|
||||
return RuleEvaluation.Passed();
|
||||
}
|
||||
|
||||
return RuleEvaluation.Found(CreateFinding(
|
||||
commit >= 97 ? Severity.High : Severity.Medium,
|
||||
$"{WindowsServerRules.Number(commit.Value)} % af commit-grænsen er brugt",
|
||||
"Commit charge er den hukommelse, programmerne har reserveret (RAM plus sidefil). Når grænsen nås, kan Windows ikke give mere hukommelse ud: "
|
||||
+ "programmer og tjenester fejler eller går ned, selv om der måske er ledig RAM.",
|
||||
"Find processen med størst commit (Jobliste, kolonnen Commit-størrelse). Et program, der vokser støt, lækker hukommelse. Øg sidefilen eller RAM, "
|
||||
+ "hvis forbruget er forventet.",
|
||||
[new Evidence("Commit (5 min.)", WindowsServerRules.Number(commit.Value, 1) + " %"), new Evidence("Grænse", WindowsServerRules.Number(context.Settings.Memory.CommitPercent) + " %")],
|
||||
subject: "commit"));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class DiskFreeSpaceRule : ScopedRule<ServerRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-WIN-005";
|
||||
public override string Title => "Lidt ledig plads på et drev";
|
||||
|
||||
public override RuleEvaluation Evaluate(ServerRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var disks = context.Data.Get<ServerDiskSnapshot>();
|
||||
var settings = context.Settings.Disks;
|
||||
return Result(disks.Disks
|
||||
.Where(d => d.SizeGb > 0 && d.FreePercent < settings.FreePercent && d.FreeGb < settings.EnoughFreeGigabytes)
|
||||
.Select(d =>
|
||||
{
|
||||
var severity = d.FreeGb < 1 || d.FreePercent < 1 ? Severity.Critical : d.FreePercent < settings.CriticalFreePercent ? Severity.High : Severity.Medium;
|
||||
var forecast = disks.Forecasts.FirstOrDefault(f => string.Equals(f.Volume, d.Volume, StringComparison.OrdinalIgnoreCase));
|
||||
return CreateFinding(
|
||||
severity,
|
||||
$"Drev {d.Volume} har kun {WindowsServerRules.Number(d.FreeGb, 1)} GB ledig ({WindowsServerRules.Number(d.FreePercent, 1)} %)",
|
||||
"Når et drev løber fuldt, stopper alt, der skriver til det: SQL Server kan ikke vokse data- og logfiler (databasen bliver skrivebeskyttet "
|
||||
+ "i praksis), backup fejler, AOS og Windows kan ikke skrive logfiler og midlertidige filer, og brugere kan ikke gemme.",
|
||||
"Find hvad der fylder (fx med WinDirStat eller TreeSize), ryd op i gamle backupfiler, logfiler og midlertidige filer, eller udvid drevet. "
|
||||
+ "Flyt ikke database- eller logfiler uden at planlægge det.",
|
||||
[
|
||||
new Evidence("Ledig", WindowsServerRules.Number(d.FreeGb, 1) + " GB"),
|
||||
new Evidence("Størrelse", WindowsServerRules.Number(d.SizeGb, 1) + " GB"),
|
||||
new Evidence("Andel ledig", WindowsServerRules.Number(d.FreePercent, 1) + " %"),
|
||||
new Evidence("Prognose", forecast?.Describe() ?? "–"),
|
||||
new Evidence("Navn", d.Label ?? "–"),
|
||||
],
|
||||
subject: "disk: " + d.Volume);
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class DiskForecastRule : ScopedRule<ServerRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-WIN-006";
|
||||
public override string Title => "Et drev bliver fuldt om få dage";
|
||||
|
||||
public override RuleEvaluation Evaluate(ServerRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var disks = context.Data.Get<ServerDiskSnapshot>();
|
||||
var settings = context.Settings.Disks;
|
||||
if (disks.Disks.Count > 0 && disks.Forecasts.Count == 0)
|
||||
{
|
||||
return RuleEvaluation.NotMeasurable("Prognosen beregnes ud fra SQLInsights egne målinger af ledig plads; der er endnu ingen historik.");
|
||||
}
|
||||
|
||||
return Result(disks.Forecasts
|
||||
.Where(f => f.DaysUntilFull is { } days && days <= settings.ForecastDays)
|
||||
.Select(f =>
|
||||
{
|
||||
var days = f.DaysUntilFull!.Value;
|
||||
var disk = disks.Disks.FirstOrDefault(d => string.Equals(d.Volume, f.Volume, StringComparison.OrdinalIgnoreCase));
|
||||
return CreateFinding(
|
||||
days <= 2 ? Severity.Critical : days <= 7 ? Severity.High : Severity.Medium,
|
||||
$"Disken {f.Volume} er {f.Describe()}",
|
||||
$"Pladsen på drevet er de seneste {WindowsServerRules.Number(f.HistoryDays)} dage brugt med ca. {WindowsServerRules.Number(f.GrowthGbPerDay, 1)} GB "
|
||||
+ "om dagen. Fortsætter det, er drevet fuldt, og så stopper alt, der skriver til det (databaser, backup, logfiler). Prognosen er en ret linje "
|
||||
+ "gennem SQLInsights egne målinger og tager ikke højde for oprydning, der ikke er sket endnu.",
|
||||
"Find hvad der vokser (databasefiler, backup, logfiler, brugerdata), og ryd op, flyt eller udvid drevet i god tid. Tjek at oprydningsjob og "
|
||||
+ "backup-rotation faktisk kører.",
|
||||
[
|
||||
new Evidence("Fuldt om ca.", WindowsServerRules.Number(days, days < 10 ? 1 : 0) + " dage"),
|
||||
new Evidence("Forventet fuld", f.FullAt(context.Now) is { } at ? at.UtcDateTime.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture) : "–"),
|
||||
new Evidence("Vækst pr. dag", WindowsServerRules.Number(f.GrowthGbPerDay, 2) + " GB"),
|
||||
new Evidence("Ledig nu", disk is null ? "–" : WindowsServerRules.Number(disk.FreeGb, 1) + " GB af " + WindowsServerRules.Number(disk.SizeGb, 1) + " GB"),
|
||||
new Evidence("Historik", $"{WindowsServerRules.Number(f.HistoryDays, 1)} dage, {f.Samples} målinger, R² {WindowsServerRules.Number(f.Fit, 2)}"),
|
||||
],
|
||||
subject: "forecast: " + f.Volume);
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Services set to start automatically that are not running, except those that stop by design.</summary>
|
||||
public sealed class StoppedAutomaticServiceRule : ScopedRule<ServerRuleContext>
|
||||
{
|
||||
/// <summary>Automatic services that stop by themselves when they have nothing to do.</summary>
|
||||
public static readonly IReadOnlyList<string> StopByDesign =
|
||||
[
|
||||
"sppsvc", "gupdate", "gupdatem", "edgeupdate", "edgeupdatem", "MapsBroker", "RemoteRegistry", "tiledatamodelsvc", "WbioSrvc", "DoSvc", "BITS",
|
||||
"wuauserv", "TrustedInstaller", "ShellHWDetection", "clr_optimization_*", "OneSyncSvc_*", "CDPUserSvc_*", "GoogleUpdater*", "MozillaMaintenance",
|
||||
"WaaSMedicSvc", "UsoSvc", "sedsvc", "AGMService", "AdobeARMservice", "Intel(R) TPM Provisioning Service", "SysmonLog", "TabletInputService",
|
||||
];
|
||||
|
||||
public override string Id => "SQLI-WIN-007";
|
||||
public override string Title => "En tjeneste, der skal starte automatisk, kører ikke";
|
||||
|
||||
public override RuleEvaluation Evaluate(ServerRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var services = context.Data.Get<ServerServiceSnapshot>();
|
||||
if (InGracePeriod(context))
|
||||
{
|
||||
return RuleEvaluation.Passed();
|
||||
}
|
||||
|
||||
var ignore = StopByDesign.Concat(context.Settings.Services.Ignore).Concat(context.Server.IgnoredServices).ToList();
|
||||
return Result(services.Services
|
||||
.Where(s => s.IsAutomatic && !s.IsRunning && !WindowsServerRules.Matches(s.Name, ignore) && !WindowsServerRules.Matches(s.Name, context.Server.WatchedServices))
|
||||
.Select(s => CreateFinding(
|
||||
Severity.Medium,
|
||||
$"Tjenesten {s.DisplayName} ({s.Name}) kører ikke",
|
||||
$"Tjenesten er sat til at starte automatisk, men står som \"{s.State}\"" + (s.ExitCode is { } code and not 0 and not 1077 ? $" (afslutningskode {code})" : string.Empty)
|
||||
+ ". Det, tjenesten gør, sker ikke, og en automatisk tjeneste, der er stoppet, er oftest gået ned.",
|
||||
"Se Windows' hændelseslog (System, Service Control Manager) for hvorfor tjenesten stoppede, og start den. SQLInsight starter aldrig selv tjenester. "
|
||||
+ "Skal tjenesten ikke køre, så sæt den til Manuel eller Deaktiveret, eller tilføj den til listen over tjenester, der ignoreres for serveren.",
|
||||
[
|
||||
new Evidence("Tjeneste", s.Name),
|
||||
new Evidence("Visningsnavn", s.DisplayName),
|
||||
new Evidence("Status", s.State),
|
||||
new Evidence("Start", s.StartMode + (s.DelayedAutoStart ? " (forsinket)" : string.Empty)),
|
||||
new Evidence("Afslutningskode", s.ExitCode?.ToString(CultureInfo.InvariantCulture) ?? "–"),
|
||||
],
|
||||
subject: "service: " + s.Name)));
|
||||
}
|
||||
|
||||
internal static bool InGracePeriod(ServerRuleContext context) =>
|
||||
context.Data.TryGet<ServerPerformanceSnapshot>(out var performance) && performance.Os.LastBootUpTime is { } boot
|
||||
&& context.Now - boot < TimeSpan.FromMinutes(context.Settings.Services.StartupGraceMinutes);
|
||||
}
|
||||
|
||||
/// <summary>Services chosen for the server (the AOS, IIS, SQL Server …) that must run, whatever their start mode.</summary>
|
||||
public sealed class WatchedServiceRule : ScopedRule<ServerRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-WIN-008";
|
||||
public override string Title => "En overvåget tjeneste kører ikke";
|
||||
|
||||
public override RuleEvaluation Evaluate(ServerRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
if (context.Server.WatchedServices.Count == 0)
|
||||
{
|
||||
return RuleEvaluation.NotApplicable("Der er ikke valgt tjenester at overvåge på serveren.");
|
||||
}
|
||||
|
||||
var services = context.Data.Get<ServerServiceSnapshot>();
|
||||
if (StoppedAutomaticServiceRule.InGracePeriod(context))
|
||||
{
|
||||
return RuleEvaluation.Passed();
|
||||
}
|
||||
|
||||
var findings = new List<Finding>();
|
||||
foreach (var name in context.Server.WatchedServices)
|
||||
{
|
||||
var service = services.Services.FirstOrDefault(s => s.Name.Equals(name, StringComparison.OrdinalIgnoreCase) || s.DisplayName.Equals(name, StringComparison.OrdinalIgnoreCase));
|
||||
if (service is null)
|
||||
{
|
||||
findings.Add(CreateFinding(
|
||||
Severity.Medium,
|
||||
$"Den overvågede tjeneste {name} findes ikke eller kan ikke ses",
|
||||
"Tjenesten står på listen over tjenester, der skal køre på serveren, men Windows viser den ikke. Den kan være afinstalleret eller omdøbt, "
|
||||
+ "eller agenten må ikke se den (tjenestens egne sikkerhedsindstillinger).",
|
||||
"Tjek tjenestens navn (ikke visningsnavnet) i services.msc, og ret listen. Kør tildelingsscriptet igen, hvis agenten mangler rettigheder.",
|
||||
[new Evidence("Tjeneste", name), new Evidence("Tjenester agenten kan se", services.Services.Count.ToString(CultureInfo.InvariantCulture))],
|
||||
subject: "watched: " + name));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!service.IsRunning)
|
||||
{
|
||||
findings.Add(CreateFinding(
|
||||
Severity.High,
|
||||
$"Tjenesten {service.DisplayName} ({service.Name}) kører ikke",
|
||||
$"Tjenesten er valgt som en, der skal køre på serveren, men står som \"{service.State}\". Det den leverer (fx AOS'en for AX-brugerne, IIS, "
|
||||
+ "SQL Server) er ikke tilgængeligt.",
|
||||
"Se Windows' hændelseslog (System og Application) for hvorfor den stoppede, og start den igen. SQLInsight starter aldrig selv tjenester.",
|
||||
[
|
||||
new Evidence("Tjeneste", service.Name),
|
||||
new Evidence("Status", service.State),
|
||||
new Evidence("Start", service.StartMode),
|
||||
new Evidence("Afslutningskode", service.ExitCode?.ToString(CultureInfo.InvariantCulture) ?? "–"),
|
||||
],
|
||||
subject: "watched: " + service.Name));
|
||||
}
|
||||
}
|
||||
|
||||
return Result(findings);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class WindowsEventLogRule : ScopedRule<ServerRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-WIN-009";
|
||||
public override string Title => "Fejl i Windows' hændelseslog";
|
||||
|
||||
public override RuleEvaluation Evaluate(ServerRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var events = context.Data.Get<ServerEventSnapshot>();
|
||||
var settings = context.Settings.EventLog;
|
||||
return Result(events.Recent
|
||||
.Where(g => g.Category != ServerEventCategory.UnexpectedShutdown)
|
||||
.Where(g => g.Category != ServerEventCategory.Error || g.Count >= Math.Max(1, settings.RepeatThreshold))
|
||||
.OrderByDescending(g => g.Category).ThenByDescending(g => g.Count)
|
||||
.Take(Math.Max(1, settings.MaxFindings))
|
||||
.Select(g => CreateFinding(
|
||||
SeverityOf(g),
|
||||
$"{ServerEventAnalysis.Display(g.Category)}: {g.Source} {g.EventId} ({g.Count} {(g.Count == 1 ? "gang" : "gange")} i {g.Log}-loggen)",
|
||||
Explanation(g.Category),
|
||||
Recommendation(g.Category),
|
||||
[
|
||||
new Evidence("Log", g.Log),
|
||||
new Evidence("Kilde", g.Source),
|
||||
new Evidence("Hændelses-id", g.EventId.ToString(CultureInfo.InvariantCulture)),
|
||||
new Evidence("Niveau", g.Level == 1 ? "Kritisk" : "Fejl"),
|
||||
new Evidence("Antal (24 timer)", g.Count.ToString(CultureInfo.InvariantCulture)),
|
||||
new Evidence("Første", WindowsServerRules.Time(g.FirstAt)),
|
||||
new Evidence("Seneste", WindowsServerRules.Time(g.LastAt)),
|
||||
new Evidence("Eksempel", g.Sample ?? "(ingen tekst)", Sensitive: true),
|
||||
],
|
||||
subject: "event: " + g.Key)));
|
||||
}
|
||||
|
||||
private static Severity SeverityOf(ServerEventGroup group) => group.Category switch
|
||||
{
|
||||
ServerEventCategory.Disk or ServerEventCategory.Resources or ServerEventCategory.Critical => Severity.High,
|
||||
ServerEventCategory.ServiceFailure => group.Count >= 3 ? Severity.High : Severity.Medium,
|
||||
ServerEventCategory.DynamicsAx or ServerEventCategory.ApplicationCrash => Severity.Medium,
|
||||
_ => Severity.Low,
|
||||
};
|
||||
|
||||
private static string Explanation(ServerEventCategory category) => category switch
|
||||
{
|
||||
ServerEventCategory.Disk =>
|
||||
"Windows har haft fejl mod en disk, controller eller filsystemet: timeouts, nulstillinger eller sider, der ikke kunne læses eller skrives. "
|
||||
+ "Det går forud for datatab og meget langsom I/O, også i SQL Server (fejl 823/824/833).",
|
||||
ServerEventCategory.Resources =>
|
||||
"Windows er løbet tør for en ressource (hukommelse, nonpaged pool, virtuel hukommelse). Programmer og tjenester fejler, når de ikke kan få mere.",
|
||||
ServerEventCategory.Critical => "Windows har logget en kritisk hændelse: den højeste alvorsgrad, som bruges, når noget vigtigt ikke virker.",
|
||||
ServerEventCategory.ServiceFailure =>
|
||||
"En tjeneste stoppede uventet eller kunne ikke starte. Det, tjenesten leverer, er væk, indtil den startes igen (eller Windows genstarter den).",
|
||||
ServerEventCategory.DynamicsAx =>
|
||||
"Dynamics AX' Application Object Server (AOS) har logget fejl. De kan betyde, at brugere eller batchjob får fejl, eller at AOS'en har problemer "
|
||||
+ "med databasen.",
|
||||
ServerEventCategory.ApplicationCrash => "Et program eller en tjeneste er gået ned eller holdt op med at svare. Det, programmet lavede, blev afbrudt.",
|
||||
_ => "Den samme fejl er logget gentagne gange det seneste døgn. Gentagne fejl er sjældent tilfældige.",
|
||||
};
|
||||
|
||||
private static string Recommendation(ServerEventCategory category) => category switch
|
||||
{
|
||||
ServerEventCategory.Disk =>
|
||||
"Se hændelsen i Logbog (Event Viewer) på serveren, og tjek disk, controller, drivere og SAN/hypervisor. Kør CHECKDB på databaser på drevet, "
|
||||
+ "og sørg for gyldige backups.",
|
||||
ServerEventCategory.Resources => "Find processen, der bruger ressourcen (Jobliste, poolmon for nonpaged pool), og se SQLI-WIN-002 og -004.",
|
||||
ServerEventCategory.ServiceFailure => "Se den næste hændelse fra tjenesten selv for årsagen, og tjek tjenestekontoens adgangskode og afhængigheder.",
|
||||
ServerEventCategory.DynamicsAx => "Se hele hændelsen i Logbog på AOS-serveren og AOS'ens egen log. Gentagne databasefejl: se også SQL Server-fundene for AX-databasen.",
|
||||
ServerEventCategory.ApplicationCrash => "Hændelsen nævner programmet og modulet, der fejlede. Søg på dem, installér opdateringer, og se om nedbruddet gentager sig.",
|
||||
_ => "Se hændelsen i Logbog (Event Viewer) og slå kilden og hændelses-id'et op. Er den harmløs, kan den ignoreres under SQLInsight:Servers:EventLog:Ignore.",
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>A restart in the last day: unexpected (crash, power, blue screen) or planned.</summary>
|
||||
public sealed class ServerRestartRule : ScopedRule<ServerRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-WIN-010";
|
||||
public override string Title => "Serveren er genstartet";
|
||||
|
||||
public override RuleEvaluation Evaluate(ServerRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var performance = context.Data.Get<ServerPerformanceSnapshot>();
|
||||
if (performance.Os.LastBootUpTime is not { } boot)
|
||||
{
|
||||
return RuleEvaluation.NotMeasurable("Starttidspunktet kunne ikke aflæses (Win32_OperatingSystem.LastBootUpTime).");
|
||||
}
|
||||
|
||||
if (context.Now - boot >= ServerEventAnalysis.RecentWindow)
|
||||
{
|
||||
return RuleEvaluation.Passed();
|
||||
}
|
||||
|
||||
context.Data.TryGet<ServerEventSnapshot>(out var events);
|
||||
var crash = events?.Recent.FirstOrDefault(g => g.Category == ServerEventCategory.UnexpectedShutdown && g.LastAt >= boot.AddHours(-1));
|
||||
var evidence = new List<Evidence>
|
||||
{
|
||||
new("Startet", WindowsServerRules.Time(boot)),
|
||||
new("Oppe i", Engine.Format.Age(context.Now - boot)),
|
||||
};
|
||||
if (crash is not null)
|
||||
{
|
||||
evidence.Add(new Evidence("Hændelse", $"{crash.Source} {crash.EventId}"));
|
||||
evidence.Add(new Evidence("Tekst", crash.Sample ?? "–", Sensitive: true));
|
||||
return RuleEvaluation.Found(CreateFinding(
|
||||
Severity.High,
|
||||
$"Serveren genstartede uventet {WindowsServerRules.Time(boot)}",
|
||||
"Windows blev ikke lukket ned ordentligt: strømmen forsvandt, serveren gik ned (blå skærm) eller blev nulstillet hårdt, fx af hypervisoren. "
|
||||
+ "Alt, der kørte, blev afbrudt midt i det, og databaser og AX skal gendanne sig selv ved start.",
|
||||
"Se Logbog (System) omkring tidspunktet: Kernel-Power 41, EventLog 6008 og BugCheck 1001 fortæller hvorfor. Ved blå skærm: se dumpfilen i "
|
||||
+ "C:\\Windows\\Minidump. Tjek hypervisor- og strømlogs, og at SQL Server-databaser og AOS kom korrekt op.",
|
||||
evidence,
|
||||
subject: "restart"));
|
||||
}
|
||||
|
||||
return RuleEvaluation.Found(CreateFinding(
|
||||
Severity.Info,
|
||||
$"Serveren er genstartet {WindowsServerRules.Time(boot)}",
|
||||
"Serveren er genstartet det seneste døgn, uden tegn på at det skete uventet (fx planlagte Windows-opdateringer). Fundet er til orientering og "
|
||||
+ "forsvinder af sig selv efter et døgn.",
|
||||
"Ingen handling, hvis genstarten var planlagt. Var den ikke, så se Logbog (System, User32 1074) for hvem eller hvad der genstartede serveren.",
|
||||
evidence,
|
||||
subject: "restart"));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ServerUptimeRule : ScopedRule<ServerRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-WIN-011";
|
||||
public override string Title => "Serveren er ikke genstartet i lang tid";
|
||||
|
||||
public override RuleEvaluation Evaluate(ServerRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
if (context.Settings.UptimeDays <= 0)
|
||||
{
|
||||
return RuleEvaluation.NotApplicable("Slået fra (SQLInsight:Servers:UptimeDays er 0).");
|
||||
}
|
||||
|
||||
var performance = context.Data.Get<ServerPerformanceSnapshot>();
|
||||
if (performance.Os.LastBootUpTime is not { } boot)
|
||||
{
|
||||
return RuleEvaluation.NotMeasurable("Starttidspunktet kunne ikke aflæses.");
|
||||
}
|
||||
|
||||
var uptime = context.Now - boot;
|
||||
return uptime.TotalDays < context.Settings.UptimeDays
|
||||
? RuleEvaluation.Passed()
|
||||
: RuleEvaluation.Found(CreateFinding(
|
||||
Severity.Low,
|
||||
$"Serveren har kørt i {WindowsServerRules.Number(uptime.TotalDays)} dage uden genstart",
|
||||
"De fleste Windows-opdateringer kræver en genstart for at virke. En server, der ikke er genstartet i flere måneder, mangler derfor næsten "
|
||||
+ "sikkert sikkerhedsopdateringer.",
|
||||
"Planlæg et servicevindue og installér Windows-opdateringer. Tjek at opdateringsværktøjet (WSUS, Intune, …) faktisk når serveren.",
|
||||
[new Evidence("Startet", WindowsServerRules.Time(boot)), new Evidence("Grænse", $"{context.Settings.UptimeDays} dage")],
|
||||
subject: "uptime"));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class FailedScheduledTaskRule : ScopedRule<ServerRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-WIN-012";
|
||||
public override string Title => "En planlagt opgave fejlede sidste gang, den kørte";
|
||||
|
||||
public override RuleEvaluation Evaluate(ServerRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var tasks = context.Data.Get<ServerTaskSnapshot>();
|
||||
return Result(tasks.Tasks
|
||||
.Where(t => t.Enabled && TaskResults.IsFailure(t.LastTaskResult) && !WindowsServerRules.Matches(t.FullName, context.Settings.Tasks.Ignore))
|
||||
.Select(t => CreateFinding(
|
||||
Severity.Medium,
|
||||
$"Opgaven {t.FullName} fejlede: {TaskResults.Describe(t.LastTaskResult!.Value)}",
|
||||
"Opgaven i Opgavestyring (Task Scheduler) sluttede med en fejlkode sidste gang, den kørte. Planlagte opgaver er ofte oprydning, eksport, "
|
||||
+ "integrationer eller backup af filer: når de fejler, opdager ingen det, før noget mangler.",
|
||||
"Åbn opgaven i Opgavestyring, se fanen Historik og opgavens egen log. Tjek kontoen, opgaven kører som (adgangskode, rettigheder), og at "
|
||||
+ "programmet eller scriptet findes.",
|
||||
[
|
||||
new Evidence("Opgave", t.FullName),
|
||||
new Evidence("Resultat", TaskResults.Describe(t.LastTaskResult.Value)),
|
||||
new Evidence("Seneste kørsel", WindowsServerRules.Time(t.LastRunTime)),
|
||||
new Evidence("Næste kørsel", WindowsServerRules.Time(t.NextRunTime)),
|
||||
],
|
||||
subject: "task: " + t.FullName)));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class MissedScheduledTaskRule : ScopedRule<ServerRuleContext>
|
||||
{
|
||||
public override string Id => "SQLI-WIN-013";
|
||||
public override string Title => "En planlagt opgave kørte ikke, da den skulle";
|
||||
|
||||
public override RuleEvaluation Evaluate(ServerRuleContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var tasks = context.Data.Get<ServerTaskSnapshot>();
|
||||
var late = TimeSpan.FromMinutes(Math.Max(1, context.Settings.Tasks.LateMinutes));
|
||||
return Result(tasks.Tasks
|
||||
.Where(t => t.Enabled && !string.Equals(t.State, "Running", StringComparison.OrdinalIgnoreCase)
|
||||
&& !WindowsServerRules.Matches(t.FullName, context.Settings.Tasks.Ignore)
|
||||
&& (t.MissedRuns > 0 || (t.NextRunTime is { } next && next < context.Now - late && (t.LastRunTime is null || t.LastRunTime < next))))
|
||||
.Select(t => CreateFinding(
|
||||
Severity.Medium,
|
||||
t.MissedRuns > 0 ? $"Opgaven {t.FullName} har sprunget {t.MissedRuns} kørsler over" : $"Opgaven {t.FullName} kørte ikke {WindowsServerRules.Time(t.NextRunTime)}",
|
||||
"Opgaven skulle have kørt, men gjorde det ikke. Det sker, når Opgavestyring ikke kører, når opgavens betingelser ikke er opfyldt (fx \"kun når "
|
||||
+ "brugeren er logget på\"), eller når serveren var slukket. Det, opgaven gør, er ikke sket.",
|
||||
"Se opgavens Historik i Opgavestyring og dens betingelser og indstillinger (\"Kør opgaven hurtigst muligt efter en mistet start\").",
|
||||
[
|
||||
new Evidence("Opgave", t.FullName),
|
||||
new Evidence("Planlagt", WindowsServerRules.Time(t.NextRunTime)),
|
||||
new Evidence("Seneste kørsel", WindowsServerRules.Time(t.LastRunTime)),
|
||||
new Evidence("Mistede kørsler", t.MissedRuns.ToString(CultureInfo.InvariantCulture)),
|
||||
new Evidence("Status", t.State),
|
||||
],
|
||||
subject: "task-missed: " + t.FullName)));
|
||||
}
|
||||
}
|
||||
334
tests/SQLInsight.Rules.Tests/WindowsServerRuleTests.cs
Normal file
334
tests/SQLInsight.Rules.Tests/WindowsServerRuleTests.cs
Normal file
|
|
@ -0,0 +1,334 @@
|
|||
using SQLInsight.Core.Collection;
|
||||
using SQLInsight.Core.Findings;
|
||||
using SQLInsight.Core.Servers;
|
||||
using SQLInsight.Rules.Engine;
|
||||
using SQLInsight.Rules.Servers;
|
||||
|
||||
namespace SQLInsight.Rules.Tests;
|
||||
|
||||
public class WindowsServerRuleTests
|
||||
{
|
||||
private static readonly DateTimeOffset Now = new(2026, 10, 2, 12, 0, 0, TimeSpan.Zero);
|
||||
|
||||
private static ServerIdentity Server(IReadOnlyList<string>? watched = null, IReadOnlyList<string>? ignored = null) =>
|
||||
new(1, "AOS01", "aos01.firma.local", ServerRoles.Application) { WatchedServices = watched ?? [], IgnoredServices = ignored ?? [] };
|
||||
|
||||
private static ServerRuleContext Context(CollectedData data, ServerIdentity? server = null, ServerRuleSettings? settings = null) =>
|
||||
new(server ?? Server(), Now, data, settings ?? new ServerRuleSettings());
|
||||
|
||||
private static ServerPerformanceSnapshot Performance(Func<int, ServerPerformanceSample> sample, int minutes = 60, DateTimeOffset? boot = null) => new()
|
||||
{
|
||||
CapturedAt = Now,
|
||||
Os = new ServerOsInfo { Caption = "Windows Server 2022", TotalMemoryMb = 16384, LastBootUpTime = boot ?? Now.AddDays(-30), LogicalProcessors = 8 },
|
||||
Samples = Enumerable.Range(0, minutes).Select(m => sample(m) with { At = Now.AddMinutes(-m) }).Reverse().ToList(),
|
||||
};
|
||||
|
||||
private static ServerPerformanceSample Sample(double? cpu = 20, double? availableMb = 8000, double? pages = 10, double? commit = 40) =>
|
||||
new(Now, cpu, availableMb, 16384, pages, commit);
|
||||
|
||||
private static CollectedData With<T>(T value)
|
||||
where T : class
|
||||
{
|
||||
var data = new CollectedData();
|
||||
data.Set(value);
|
||||
return data;
|
||||
}
|
||||
|
||||
private static RuleResult Run(IScopedRule<ServerRuleContext> rule, ServerRuleContext context) => Assert.Single(ScopedRuleEngine.Evaluate([rule], context));
|
||||
|
||||
[Fact]
|
||||
public void Every_rule_has_a_unique_well_formed_id_and_a_title()
|
||||
{
|
||||
var all = ServerRuleCatalog.All;
|
||||
|
||||
Assert.Equal(all.Count, all.Select(r => r.Id).Distinct().Count());
|
||||
Assert.All(all, r => Assert.Matches("^SQLI-(WIN|SYN)-[0-9]{3}$", r.Id));
|
||||
Assert.All(all, r => Assert.False(string.IsNullOrWhiteSpace(r.Title)));
|
||||
Assert.DoesNotContain(all, r => BuiltInRules.Create().Any(b => b.Id == r.Id));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Missing_data_is_not_measurable_with_the_reason()
|
||||
{
|
||||
var data = new CollectedData();
|
||||
data.MarkUnavailable<ServerDiskSnapshot>("Adgang nægtet til root\\cimv2.");
|
||||
|
||||
var result = Run(new DiskFreeSpaceRule(), Context(data));
|
||||
|
||||
Assert.Equal(RuleOutcome.NotMeasurable, result.Outcome);
|
||||
Assert.Equal("Adgang nægtet til root\\cimv2.", result.Reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sustained_high_cpu_is_a_finding_a_burst_is_not()
|
||||
{
|
||||
var sustained = Run(new ServerCpuRule(), Context(With(Performance(m => Sample(cpu: 95)))));
|
||||
var burst = Run(new ServerCpuRule(), Context(With(Performance(m => Sample(cpu: m < 3 ? 100 : 30)))));
|
||||
var first = Run(new ServerCpuRule(), Context(With(Performance(_ => Sample(cpu: null)))));
|
||||
|
||||
Assert.Equal(Severity.High, Assert.Single(sustained.Findings).Severity);
|
||||
Assert.Contains("95 %", sustained.Findings[0].Title, StringComparison.Ordinal);
|
||||
Assert.Equal(RuleOutcome.Passed, burst.Outcome);
|
||||
Assert.Equal(RuleOutcome.NotMeasurable, first.Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Low_memory_needs_both_little_share_and_few_megabytes()
|
||||
{
|
||||
var low = Run(new ServerMemoryRule(), Context(With(Performance(_ => Sample(availableMb: 300)))));
|
||||
var bigServer = Run(new ServerMemoryRule(), Context(With(Performance(_ => new ServerPerformanceSample(Now, 10, 4000, 262144, 0, 40)))));
|
||||
|
||||
Assert.Equal(Severity.Medium, Assert.Single(low.Findings).Severity);
|
||||
Assert.Equal(RuleOutcome.Passed, bigServer.Outcome); // 1,5 % of 256 GB, but 4 GB free
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Paging_only_counts_with_little_free_memory()
|
||||
{
|
||||
Assert.Single(Run(new ServerPagingRule(), Context(With(Performance(_ => Sample(availableMb: 900, pages: 5000))))).Findings);
|
||||
Assert.Equal(RuleOutcome.Passed, Run(new ServerPagingRule(), Context(With(Performance(_ => Sample(availableMb: 8000, pages: 5000))))).Outcome);
|
||||
Assert.Equal(Severity.High, Assert.Single(Run(new ServerCommitRule(), Context(With(Performance(_ => Sample(commit: 98))))).Findings).Severity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Disk_free_space_by_share_unless_there_is_plenty()
|
||||
{
|
||||
var disks = new ServerDiskSnapshot
|
||||
{
|
||||
CapturedAt = Now,
|
||||
Disks =
|
||||
[
|
||||
new ServerDisk(@"C:\", "System", "NTFS", 100, 30),
|
||||
new ServerDisk(@"D:\", "Data", "NTFS", 500, 30), // 6 %: Medium
|
||||
new ServerDisk(@"E:\", "Log", "NTFS", 100, 3), // 3 %: High
|
||||
new ServerDisk(@"F:\", "Backup", "NTFS", 4000, 200), // 5 % but 200 GB: fine
|
||||
new ServerDisk(@"G:\", "Temp", "NTFS", 50, 0.5), // under 1 GB: Critical
|
||||
],
|
||||
};
|
||||
|
||||
var findings = Run(new DiskFreeSpaceRule(), Context(With(disks))).Findings.ToDictionary(f => f.Subject!, f => f.Severity);
|
||||
|
||||
Assert.Equal(new Dictionary<string, Severity> { [@"disk: D:\"] = Severity.Medium, [@"disk: E:\"] = Severity.High, [@"disk: G:\"] = Severity.Critical }, findings);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Forecast_findings_by_days_until_full()
|
||||
{
|
||||
var disks = new ServerDiskSnapshot
|
||||
{
|
||||
CapturedAt = Now,
|
||||
Disks = [new ServerDisk(@"D:\", null, "NTFS", 500, 100), new ServerDisk(@"E:\", null, "NTFS", 500, 100)],
|
||||
Forecasts =
|
||||
[
|
||||
new DiskForecast(@"D:\", 12, 8, 300, 14, 0.95),
|
||||
new DiskForecast(@"E:\", 1.5, 60, 300, 14, 0.95),
|
||||
new DiskForecast(@"F:\", null, 0, 300, 14, 0.1, "ingen tydelig tendens"),
|
||||
new DiskForecast(@"G:\", 200, 0.5, 300, 14, 0.9),
|
||||
],
|
||||
};
|
||||
|
||||
var findings = Run(new DiskForecastRule(), Context(With(disks))).Findings;
|
||||
|
||||
Assert.Equal(2, findings.Count);
|
||||
Assert.Contains(findings, f => f.Title == @"Disken D:\ er fuld om ca. 12 dage" && f.Severity == Severity.Medium);
|
||||
Assert.Contains(findings, f => f.Subject == @"forecast: E:\" && f.Severity == Severity.Critical);
|
||||
Assert.Equal(RuleOutcome.NotMeasurable, Run(new DiskForecastRule(), Context(With(disks with { Forecasts = [] }))).Outcome);
|
||||
}
|
||||
|
||||
private static ServerServiceSnapshot Services(params ServerService[] services) => new() { CapturedAt = Now, Services = services };
|
||||
|
||||
[Fact]
|
||||
public void Stopped_automatic_services_except_the_ones_that_stop_by_design()
|
||||
{
|
||||
var data = With(Services(
|
||||
new ServerService("AOS60$01", "Microsoft Dynamics AX Object Server 6.0$01", "Stopped", "Auto", ExitCode: 1067),
|
||||
new ServerService("Spooler", "Print Spooler", "Running", "Auto"),
|
||||
new ServerService("sppsvc", "Software Protection", "Stopped", "Auto", DelayedAutoStart: true),
|
||||
new ServerService("clr_optimization_v4.0.30319_64", ".NET Optimization", "Stopped", "Auto"),
|
||||
new ServerService("MyBatch", "Batch", "Stopped", "Auto"),
|
||||
new ServerService("Manual1", "Manual", "Stopped", "Manual")));
|
||||
data.Set(Performance(_ => Sample()));
|
||||
|
||||
var result = Run(new StoppedAutomaticServiceRule(), Context(data, Server(ignored: ["MyBatch"])));
|
||||
|
||||
var finding = Assert.Single(result.Findings);
|
||||
Assert.Equal("service: AOS60$01", finding.Subject);
|
||||
Assert.Contains("1067", finding.Explanation, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Right_after_a_restart_stopped_services_are_still_starting()
|
||||
{
|
||||
var data = With(Services(new ServerService("AOS60$01", "AOS", "Stopped", "Auto")));
|
||||
data.Set(Performance(_ => Sample(), boot: Now.AddMinutes(-5)));
|
||||
|
||||
Assert.Equal(RuleOutcome.Passed, Run(new StoppedAutomaticServiceRule(), Context(data)).Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Watched_services_must_run_and_exist()
|
||||
{
|
||||
var data = With(Services(new ServerService("AOS60$01", "AOS", "Stopped", "Manual"), new ServerService("W3SVC", "IIS", "Running", "Auto")));
|
||||
|
||||
var result = Run(new WatchedServiceRule(), Context(data, Server(watched: ["AOS60$01", "W3SVC", "ReportServer"])));
|
||||
var none = Run(new WatchedServiceRule(), Context(data));
|
||||
var automatic = Run(new StoppedAutomaticServiceRule(), Context(data, Server(watched: ["AOS60$01"])));
|
||||
|
||||
Assert.Equal(2, result.Findings.Count);
|
||||
Assert.Contains(result.Findings, f => f.Subject == "watched: AOS60$01" && f.Severity == Severity.High);
|
||||
Assert.Contains(result.Findings, f => f.Subject == "watched: ReportServer" && f.Severity == Severity.Medium);
|
||||
Assert.Equal(RuleOutcome.NotApplicable, none.Outcome);
|
||||
Assert.Equal(RuleOutcome.Passed, automatic.Outcome); // not reported twice
|
||||
}
|
||||
|
||||
private static ServerEventGroup Group(string source, long id, ServerEventCategory category, int count, int level = 2) =>
|
||||
new("System", source, id, level, category, count, Now.AddHours(-2), Now.AddMinutes(-5), "besked med \\\\fil01\\hemmeligt");
|
||||
|
||||
[Fact]
|
||||
public void Events_important_kinds_at_once_ordinary_errors_when_they_repeat()
|
||||
{
|
||||
var events = new ServerEventSnapshot
|
||||
{
|
||||
CapturedAt = Now,
|
||||
ReadUntil = Now,
|
||||
Recent =
|
||||
[
|
||||
Group("disk", 153, ServerEventCategory.Disk, 1),
|
||||
Group("Service Control Manager", 7031, ServerEventCategory.ServiceFailure, 4),
|
||||
Group("Some-App", 9, ServerEventCategory.Error, 2),
|
||||
Group("Other-App", 3, ServerEventCategory.Error, 12),
|
||||
Group("EventLog", 6008, ServerEventCategory.UnexpectedShutdown, 1),
|
||||
],
|
||||
};
|
||||
|
||||
var findings = Run(new WindowsEventLogRule(), Context(With(events))).Findings;
|
||||
|
||||
Assert.Equal(["event: System/disk/153", "event: System/Service Control Manager/7031", "event: System/Other-App/3"], findings.Select(f => f.Subject).ToList());
|
||||
Assert.Equal([Severity.High, Severity.High, Severity.Low], findings.Select(f => f.Severity).ToList());
|
||||
Assert.True(findings[0].Evidence.Single(e => e.Name == "Eksempel").Sensitive);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Event_findings_are_capped()
|
||||
{
|
||||
var events = new ServerEventSnapshot
|
||||
{
|
||||
CapturedAt = Now,
|
||||
ReadUntil = Now,
|
||||
Recent = Enumerable.Range(1, 40).Select(i => Group("App" + i, i, ServerEventCategory.ApplicationCrash, 1)).ToList(),
|
||||
};
|
||||
|
||||
Assert.Equal(15, Run(new WindowsEventLogRule(), Context(With(events))).Findings.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Restart_unexpected_with_a_crash_event_otherwise_info()
|
||||
{
|
||||
var data = With(Performance(_ => Sample(), boot: Now.AddHours(-3)));
|
||||
var planned = Run(new ServerRestartRule(), Context(data));
|
||||
data.Set(new ServerEventSnapshot
|
||||
{
|
||||
CapturedAt = Now, ReadUntil = Now,
|
||||
Recent = [Group("Microsoft-Windows-Kernel-Power", 41, ServerEventCategory.UnexpectedShutdown, 1, level: 1) with { LastAt = Now.AddHours(-3) }],
|
||||
});
|
||||
var crash = Run(new ServerRestartRule(), Context(data));
|
||||
var old = Run(new ServerRestartRule(), Context(With(Performance(_ => Sample(), boot: Now.AddDays(-3)))));
|
||||
|
||||
Assert.Equal(Severity.Info, Assert.Single(planned.Findings).Severity);
|
||||
Assert.Equal(Severity.High, Assert.Single(crash.Findings).Severity);
|
||||
Assert.Contains("uventet", crash.Findings[0].Title, StringComparison.Ordinal);
|
||||
Assert.Equal(RuleOutcome.Passed, old.Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Uptime_rule_is_off_by_default()
|
||||
{
|
||||
var data = With(Performance(_ => Sample(), boot: Now.AddDays(-200)));
|
||||
|
||||
Assert.Equal(RuleOutcome.NotApplicable, Run(new ServerUptimeRule(), Context(data)).Outcome);
|
||||
Assert.Equal(Severity.Low, Assert.Single(Run(new ServerUptimeRule(), Context(data, settings: new ServerRuleSettings { UptimeDays = 120 })).Findings).Severity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Scheduled_tasks_failed_and_missed()
|
||||
{
|
||||
var tasks = new ServerTaskSnapshot
|
||||
{
|
||||
CapturedAt = Now,
|
||||
Visible = 120,
|
||||
Tasks =
|
||||
[
|
||||
new ServerTask { Path = @"\", Name = "AX export", State = "Ready", LastTaskResult = 1, LastRunTime = Now.AddHours(-1), NextRunTime = Now.AddHours(23) },
|
||||
new ServerTask { Path = @"\", Name = "OK", State = "Ready", LastTaskResult = 0, LastRunTime = Now.AddHours(-1), NextRunTime = Now.AddHours(1) },
|
||||
new ServerTask { Path = @"\Firma\", Name = "Late", State = "Ready", LastTaskResult = 0, LastRunTime = Now.AddDays(-2), NextRunTime = Now.AddHours(-5) },
|
||||
new ServerTask { Path = @"\", Name = "Disabled", State = "Disabled", LastTaskResult = 1, NextRunTime = Now.AddHours(-5) },
|
||||
new ServerTask { Path = @"\", Name = "Missed", State = "Ready", LastTaskResult = 0, MissedRuns = 2 },
|
||||
new ServerTask { Path = @"\", Name = "Running", State = "Running", LastTaskResult = TaskResults.Running, NextRunTime = Now.AddHours(-5) },
|
||||
],
|
||||
};
|
||||
var data = With(tasks);
|
||||
|
||||
var failed = Run(new FailedScheduledTaskRule(), Context(data));
|
||||
var missed = Run(new MissedScheduledTaskRule(), Context(data));
|
||||
|
||||
Assert.Equal(@"task: \AX export", Assert.Single(failed.Findings).Subject);
|
||||
Assert.Equal([@"task-missed: \Firma\Late", @"task-missed: \Missed"], missed.Findings.Select(f => f.Subject).ToList());
|
||||
}
|
||||
}
|
||||
|
||||
public class SyntheticCheckRuleTests
|
||||
{
|
||||
private static readonly DateTimeOffset Now = new(2026, 10, 2, 12, 0, 0, TimeSpan.Zero);
|
||||
|
||||
private static SyntheticCheckIdentity Check(string kind = Core.SyntheticChecks.SyntheticCheckKind.Tcp, string? service = null, int? warn = null, bool https = false) =>
|
||||
new(7, "AOS01 port", kind, "aos01:2712") { ServerName = "AOS01", ServiceName = service, WarnMilliseconds = warn, Https = https };
|
||||
|
||||
private static Core.SyntheticChecks.SyntheticCheckResult Result(bool success, int minutesAgo, int ms = 20, string? service = null, DateTimeOffset? certificate = null) =>
|
||||
new() { At = Now.AddMinutes(-minutesAgo), Success = success, DurationMs = ms, Message = success ? "ok" : "Forbindelsen blev afvist.", ServiceState = service, CertificateExpiresAt = certificate };
|
||||
|
||||
private static RuleResult Run(IScopedRule<SyntheticCheckRuleContext> rule, SyntheticCheckIdentity check, params Core.SyntheticChecks.SyntheticCheckResult[] newestFirst) =>
|
||||
Assert.Single(ScopedRuleEngine.Evaluate([rule], new SyntheticCheckRuleContext(check, Now, newestFirst, new SyntheticCheckRuleSettings())));
|
||||
|
||||
[Fact]
|
||||
public void One_failure_is_not_an_alarm_two_in_a_row_are()
|
||||
{
|
||||
Assert.Equal(RuleOutcome.Passed, Run(new SyntheticCheckFailingRule(), Check(), Result(false, 0), Result(true, 5)).Outcome);
|
||||
var failing = Run(new SyntheticCheckFailingRule(), Check(), Result(false, 0), Result(false, 5), Result(true, 10));
|
||||
Assert.Equal(Severity.High, Assert.Single(failing.Findings).Severity);
|
||||
Assert.Equal("AOS01 port fejler: Forbindelsen blev afvist.", failing.Findings[0].Title);
|
||||
Assert.Equal(RuleOutcome.NotMeasurable, Run(new SyntheticCheckFailingRule(), Check()).Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Service_running_but_not_answering_is_said_plainly()
|
||||
{
|
||||
var running = Run(new SyntheticCheckFailingRule(), Check(service: "AOS60$01"), Result(false, 0, service: "Running"), Result(false, 5, service: "Running"));
|
||||
var stopped = Run(new SyntheticCheckFailingRule(), Check(service: "AOS60$01"), Result(false, 0, service: "Stopped"), Result(false, 5, service: "Stopped"));
|
||||
|
||||
Assert.Equal("AOS01 port svarer ikke, selv om tjenesten AOS60$01 kører", running.Findings[0].Title);
|
||||
Assert.Contains("en tjenesteovervågning alene ikke ser", running.Findings[0].Explanation, StringComparison.Ordinal);
|
||||
Assert.Equal("AOS01 port svarer ikke: tjenesten AOS60$01 er stoppet", stopped.Findings[0].Title);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Slow_answers_three_times_in_a_row()
|
||||
{
|
||||
Assert.Equal(RuleOutcome.NotApplicable, Run(new SyntheticCheckSlowRule(), Check(), Result(true, 0, 5000)).Outcome);
|
||||
var slow = Run(new SyntheticCheckSlowRule(), Check(warn: 1000), Result(true, 0, 1500), Result(true, 5, 2500), Result(true, 10, 1200));
|
||||
Assert.Equal(Severity.Medium, Assert.Single(slow.Findings).Severity);
|
||||
Assert.Equal(RuleOutcome.Passed, Run(new SyntheticCheckSlowRule(), Check(warn: 1000), Result(true, 0, 1500), Result(true, 5, 500), Result(true, 10, 1200)).Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Certificate_expiry()
|
||||
{
|
||||
var check = Check(Core.SyntheticChecks.SyntheticCheckKind.Http, https: true);
|
||||
|
||||
Assert.Equal(RuleOutcome.Passed, Run(new CertificateExpiryRule(), check, Result(true, 0, certificate: Now.AddDays(60))).Outcome);
|
||||
Assert.Equal(Severity.Medium, Run(new CertificateExpiryRule(), check, Result(true, 0, certificate: Now.AddDays(20))).Findings[0].Severity);
|
||||
Assert.Equal(Severity.High, Run(new CertificateExpiryRule(), check, Result(true, 0, certificate: Now.AddDays(3))).Findings[0].Severity);
|
||||
Assert.Equal(Severity.Critical, Run(new CertificateExpiryRule(), check, Result(false, 0, certificate: Now.AddDays(-1))).Findings[0].Severity);
|
||||
Assert.Equal(RuleOutcome.NotApplicable, Run(new CertificateExpiryRule(), Check(), Result(true, 0)).Outcome);
|
||||
Assert.Equal(RuleOutcome.NotMeasurable, Run(new CertificateExpiryRule(), check, Result(false, 0)).Outcome);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue