diff --git a/src/SQLInsight.Rules/Servers/ScopedRules.cs b/src/SQLInsight.Rules/Servers/ScopedRules.cs new file mode 100644 index 0000000..06b25f5 --- /dev/null +++ b/src/SQLInsight.Rules/Servers/ScopedRules.cs @@ -0,0 +1,90 @@ +using SQLInsight.Core.Collection; +using SQLInsight.Core.Findings; +using SQLInsight.Rules.Engine; + +namespace SQLInsight.Rules.Servers; + +/// +/// A rule about something that is not a SQL Server instance (a Windows server, a synthetic check). It produces the +/// same findings, explanations and outcomes as the instance rules, against its own kind of context. +/// +public interface IScopedRule +{ + /// Stable identifier, e.g. SQLI-WIN-001. Never reused for a different check. + string Id { get; } + + string Title { get; } + + RuleEvaluation Evaluate(TContext context); +} + +public abstract class ScopedRule : IScopedRule +{ + public abstract string Id { get; } + public abstract string Title { get; } + + public abstract RuleEvaluation Evaluate(TContext context); + + protected Finding CreateFinding(Severity severity, string title, string explanation, string recommendation, IEnumerable? evidence = null, + string? subject = null, params Reference[] references) => new() + { + RuleId = Id, + Severity = severity, + Title = title, + Subject = subject, + Explanation = explanation, + Recommendation = recommendation, + Evidence = evidence?.ToList() ?? [], + References = references, + }; + + protected static RuleEvaluation Result(IEnumerable findings) + { + var list = findings.ToArray(); + return list.Length == 0 ? RuleEvaluation.Passed() : RuleEvaluation.Found(list); + } +} + +/// Runs scoped rules. Like : a rule that cannot run is not measurable (with the reason) or an error, never silently passed. +public static class ScopedRuleEngine +{ + public static IReadOnlyList Evaluate(IEnumerable> rules, TContext context) + { + ArgumentNullException.ThrowIfNull(rules); + return rules.Select(rule => Evaluate(rule, context)).ToList(); + } + + private static RuleResult Evaluate(IScopedRule rule, TContext context) + { + try + { + var evaluation = rule.Evaluate(context); + return new RuleResult { RuleId = rule.Id, RuleTitle = rule.Title, Outcome = evaluation.Outcome, Findings = evaluation.Findings, Reason = evaluation.Reason }; + } + catch (DataUnavailableException ex) + { + return new RuleResult + { + RuleId = rule.Id, + RuleTitle = rule.Title, + Outcome = ex.NotApplicable ? RuleOutcome.NotApplicable : RuleOutcome.NotMeasurable, + Reason = ex.Message, + }; + } +#pragma warning disable CA1031 // One faulty rule must not stop the others; the failure is reported, not hidden. + catch (Exception ex) +#pragma warning restore CA1031 + { + return new RuleResult { RuleId = rule.Id, RuleTitle = rule.Title, Outcome = RuleOutcome.Error, Reason = $"{ex.GetType().Name}: {ex.Message}" }; + } + } +} + +/// Every rule about Windows servers and synthetic checks, for rule listings. +public static class ServerRuleCatalog +{ + public static IReadOnlyList<(string Id, string Title)> All => + WindowsServerRules.Create().Select(r => (r.Id, r.Title)) + .Concat(SyntheticCheckRules.Create().Select(r => (r.Id, r.Title))) + .ToList(); +} diff --git a/src/SQLInsight.Rules/Servers/ServerRuleSettings.cs b/src/SQLInsight.Rules/Servers/ServerRuleSettings.cs new file mode 100644 index 0000000..020d6d8 --- /dev/null +++ b/src/SQLInsight.Rules/Servers/ServerRuleSettings.cs @@ -0,0 +1,106 @@ +using SQLInsight.Core.Collection; +using SQLInsight.Core.Servers; + +namespace SQLInsight.Rules.Servers; + +/// +/// Thresholds of the Windows server rules (SQLInsight:Servers in the agent's appsettings.json). Not sealed: the agent's +/// options add how and how often to collect. +/// +public class ServerRuleSettings +{ + public CpuThresholds Cpu { get; set; } = new(); + public MemoryThresholds Memory { get; set; } = new(); + public DiskThresholds Disks { get; set; } = new(); + public ServiceThresholds Services { get; set; } = new(); + public EventLogThresholds EventLog { get; set; } = new(); + public TaskThresholds Tasks { get; set; } = new(); + + /// A server not restarted for more than this many days is a (low) finding: it most likely lacks Windows updates. 0: off. + public int UptimeDays { get; set; } +} + +public sealed class CpuThresholds +{ + /// Average CPU at or above this (percent) … + public double HighPercent { get; set; } = 90; + + /// … for this many minutes is a finding (SQLI-WIN-001). + public int SustainedMinutes { get; set; } = 15; +} + +public sealed class MemoryThresholds +{ + /// Available memory below this share (percent) … + public double AvailablePercent { get; set; } = 5; + + /// … and below this many MB is a finding (SQLI-WIN-002). Both, so big SQL Server hosts with little left by design are not flagged. + public double AvailableMegabytes { get; set; } = 1024; + + /// Hard page faults per second above this, while memory is low, is paging (SQLI-WIN-003). + public double PagesPerSecond { get; set; } = 1000; + + /// Committed memory at or above this share of the commit limit (percent) is a finding (SQLI-WIN-004). + public double CommitPercent { get; set; } = 90; +} + +public sealed class DiskThresholds +{ + /// Free space below this (percent) is a finding (SQLI-WIN-005) … + public double FreePercent { get; set; } = 10; + + /// … High below this … + public double CriticalFreePercent { get; set; } = 5; + + /// … unless there is at least this much free (GB): a big volume with a small share free can be fine. + public double EnoughFreeGigabytes { get; set; } = 50; + + /// A volume forecast to be full within this many days is a finding (SQLI-WIN-006); within 7 days High, 2 days Critical. + public int ForecastDays { get; set; } = 30; + + /// How many days of SQLInsight's own samples the forecast is computed from. + public int ForecastHistoryDays { get; set; } = 14; +} + +public sealed class ServiceThresholds +{ + /// + /// Services that may be stopped although they start automatically (they stop by design), in addition to the + /// built-in list. Name, or a prefix ending in *. + /// + public IList Ignore { get; } = []; + + /// Right after a restart services are still starting: no finding for this many minutes. + public int StartupGraceMinutes { get; set; } = 15; +} + +public sealed class EventLogThresholds +{ + /// An ordinary error must repeat this many times in 24 hours before it is a finding; the important kinds count at once. + public int RepeatThreshold { get; set; } = 5; + + /// Events left out as noise, in addition to the built-in list: "Kilde:id", "Kilde" or "Kilde*". + public IList Ignore { get; } = []; + + /// At most this many event findings per server, most important first. + public int MaxFindings { get; set; } = 15; +} + +public sealed class TaskThresholds +{ + /// Also watch Microsoft's own tasks (\Microsoft\…). Off: there are hundreds, and their failures are rarely yours. + public bool IncludeMicrosoftTasks { get; set; } + + /// A task whose next run time is more than this many minutes in the past has not run as planned (SQLI-WIN-013). + public int LateMinutes { get; set; } = 60; + + /// Tasks never watched (path\name, or a prefix ending in *). + public IList Ignore { get; } = []; +} + +/// What the Windows server rules see: the server, the latest data per area, and the thresholds. +public sealed record ServerRuleContext(ServerIdentity Server, DateTimeOffset Now, CollectedData Data, ServerRuleSettings Settings) +{ + /// The interval performance samples are taken at, for "sustained" averages. + public TimeSpan PerformanceInterval { get; init; } = ServerArea.DefaultInterval(ServerArea.Performance); +} diff --git a/src/SQLInsight.Rules/Servers/SyntheticCheckRules.cs b/src/SQLInsight.Rules/Servers/SyntheticCheckRules.cs new file mode 100644 index 0000000..b8d7c7f --- /dev/null +++ b/src/SQLInsight.Rules/Servers/SyntheticCheckRules.cs @@ -0,0 +1,207 @@ +using System.Globalization; +using SQLInsight.Core.Findings; +using SQLInsight.Core.SyntheticChecks; +using SQLInsight.Rules.Engine; + +namespace SQLInsight.Rules.Servers; + +/// Thresholds of the synthetic check rules (SQLInsight:SyntheticChecks in the agent's appsettings.json). +public class SyntheticCheckRuleSettings +{ + /// A check must fail this many times in a row before it is a finding, so one lost packet is not an alarm. + public int FailuresBeforeFinding { get; set; } = 2; + + /// A slow answer (above the check's limit) this many times in a row is a finding. + public int SlowRunsBeforeFinding { get; set; } = 3; + + /// A TLS certificate expiring within this many days is a finding … + public int CertificateWarningDays { get; set; } = 21; + + /// … High within this many days. + public int CertificateCriticalDays { get; set; } = 7; +} + +/// A configured check as the rules see it. +public sealed record SyntheticCheckIdentity(int CheckId, string Name, string Kind, string Target) +{ + public string? ServerName { get; init; } + public string? ServiceName { get; init; } + public int? WarnMilliseconds { get; init; } + public bool Https { get; init; } +} + +/// The latest results, newest first. +public sealed record SyntheticCheckRuleContext(SyntheticCheckIdentity Check, DateTimeOffset Now, IReadOnlyList Recent, SyntheticCheckRuleSettings Settings); + +public static class SyntheticCheckRules +{ + public static IReadOnlyList> Create() => [new SyntheticCheckFailingRule(), new SyntheticCheckSlowRule(), new CertificateExpiryRule()]; + + internal static string Time(DateTimeOffset value) => value.UtcDateTime.ToString("yyyy-MM-dd HH:mm 'UTC'", CultureInfo.InvariantCulture); +} + +public sealed class SyntheticCheckFailingRule : ScopedRule +{ + public override string Id => "SQLI-SYN-001"; + public override string Title => "Et syntetisk tjek fejler"; + + public override RuleEvaluation Evaluate(SyntheticCheckRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + if (context.Recent.Count == 0) + { + return RuleEvaluation.NotMeasurable("Tjekket har ikke kørt endnu."); + } + + var needed = Math.Max(1, context.Settings.FailuresBeforeFinding); + var failures = context.Recent.TakeWhile(r => !r.Success).ToList(); + if (failures.Count < needed) + { + return RuleEvaluation.Passed(); + } + + var check = context.Check; + var latest = failures[0]; + var serviceRunning = string.Equals(latest.ServiceState, "Running", StringComparison.OrdinalIgnoreCase); + var serviceKnown = check.ServiceName is not null && latest.ServiceState is not null; + var title = serviceKnown && serviceRunning + ? $"{check.Name} svarer ikke, selv om tjenesten {check.ServiceName} kører" + : serviceKnown + ? $"{check.Name} svarer ikke: tjenesten {check.ServiceName} er {StateText(latest.ServiceState!)}" + : $"{check.Name} fejler: {latest.Message}"; + var explanation = What(check.Kind) + + (serviceKnown && serviceRunning + ? $" Tjenesten {check.ServiceName} på {check.ServerName} kører, så Windows ser alt som i orden, men den svarer ikke: den kan hænge, være overbelastet, " + + "vente på databasen eller være blokeret af en firewall. Det er netop den slags fejl, en tjenesteovervågning alene ikke ser." + : serviceKnown ? $" Tjenesten {check.ServiceName} på {check.ServerName} kører ikke." : string.Empty); + var evidence = new List + { + new("Fejl", latest.Message, Sensitive: true), + new("Mål", check.Target), + new("Fejlet i træk", failures.Count.ToString(CultureInfo.InvariantCulture)), + new("Fejler siden", SyntheticCheckRules.Time(failures[^1].At)), + new("Seneste forsøg", SyntheticCheckRules.Time(latest.At)), + }; + if (check.ServiceName is not null) + { + evidence.Add(new Evidence("Tjeneste", $"{check.ServiceName}: {(latest.ServiceState is { } state ? StateText(state) : "kunne ikke aflæses")}")); + } + + evidence.AddRange(latest.Details.Select(d => new Evidence(d.Name, d.Value, d.Sensitive))); + return RuleEvaluation.Found(CreateFinding( + Severity.High, + title, + explanation, + Recommendation(check.Kind, serviceKnown && serviceRunning), + evidence, + subject: "check")); + } + + private static string StateText(string state) => state switch + { + "Running" => "kører", + "Stopped" => "stoppet", + "Start Pending" => "ved at starte", + "Stop Pending" => "ved at stoppe", + "Paused" => "sat på pause", + _ => state, + }; + + private static string What(string kind) => kind switch + { + SyntheticCheckKind.Tcp => "Porten svarer ikke: de programmer og brugere, der forbinder til den, kan heller ikke.", + SyntheticCheckKind.Http => "Siden svarer ikke som forventet: brugere og integrationer, der bruger den, får fejl eller venter.", + SyntheticCheckKind.Sql => "Login eller forespørgsel mod SQL Server fejler: applikationer, der bruger samme vej ind, kan heller ikke arbejde.", + SyntheticCheckKind.Dns => "Navnet kan ikke slås op (eller peger forkert hen): alt, der forbinder via navnet, fejler eller rammer den forkerte server.", + SyntheticCheckKind.AxLogon => "Dynamics AX kan ikke nås: AOS-porten svarer ikke, eller et logon gennem Business Connector fejler. Brugere og batchjob kan ikke arbejde.", + _ => "Tjekket fejler.", + }; + + private static string Recommendation(string kind, bool serviceRunning) => + (serviceRunning ? "Se tjenestens egen log og Windows' hændelseslog på serveren; en tjeneste, der kører men ikke svarer, skal ofte genstartes af en driftsansvarlig. " : string.Empty) + + kind switch + { + SyntheticCheckKind.Tcp => "Tjek at serveren og tjenesten kører, og at firewallen tillader porten fra agentens server.", + SyntheticCheckKind.Http => "Åbn adressen i en browser fra agentens server, og se IIS-/webserverens log og applikationspuljen.", + SyntheticCheckKind.Sql => "Prøv at logge ind med samme login fra agentens server, og se SQL Server-fundene og fejlloggen (fx fejl 18456).", + SyntheticCheckKind.Dns => "Slå navnet op med nslookup fra agentens server, og tjek DNS-serverne og posterne.", + SyntheticCheckKind.AxLogon => "Tjek AOS-tjenesten og dens hændelseslog, at AX-databasen svarer, og at agentens konto er AX-bruger, hvis logon testes.", + _ => string.Empty, + } + " SQLInsight genstarter aldrig selv noget."; +} + +public sealed class SyntheticCheckSlowRule : ScopedRule +{ + public override string Id => "SQLI-SYN-002"; + public override string Title => "Et syntetisk tjek svarer langsomt"; + + public override RuleEvaluation Evaluate(SyntheticCheckRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + if (context.Check.WarnMilliseconds is not { } limit) + { + return RuleEvaluation.NotApplicable("Der er ikke sat en grænse for svartid på tjekket."); + } + + var needed = Math.Max(1, context.Settings.SlowRunsBeforeFinding); + var latest = context.Recent.Take(needed).ToList(); + if (latest.Count < needed || latest.Any(r => !r.Success || r.DurationMs <= limit)) + { + return context.Recent.Count == 0 ? RuleEvaluation.NotMeasurable("Tjekket har ikke kørt endnu.") : RuleEvaluation.Passed(); + } + + return RuleEvaluation.Found(CreateFinding( + Severity.Medium, + $"{context.Check.Name} svarer langsomt ({latest.Min(r => r.DurationMs).ToString("N0", CultureInfo.GetCultureInfo("da-DK"))}–{latest.Max(r => r.DurationMs).ToString("N0", CultureInfo.GetCultureInfo("da-DK"))} ms)", + "Tjekket lykkes, men svaret har de seneste gange taget længere end grænsen. Brugerne oplever det som en langsom applikation, længe før den holder op med at svare.", + "Se belastningen på serveren (CPU, hukommelse, disk) og på databasen bag i samme tidsrum, fx med Analyse af tidsrum.", + [ + new Evidence("Svartider", string.Join(", ", latest.Select(r => r.DurationMs.ToString(CultureInfo.InvariantCulture) + " ms"))), + new Evidence("Grænse", limit.ToString(CultureInfo.InvariantCulture) + " ms"), + new Evidence("Mål", context.Check.Target), + ], + subject: "slow")); + } +} + +public sealed class CertificateExpiryRule : ScopedRule +{ + public override string Id => "SQLI-SYN-003"; + public override string Title => "TLS-certifikatet udløber snart"; + + public override RuleEvaluation Evaluate(SyntheticCheckRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + if (context.Check.Kind != SyntheticCheckKind.Http || !context.Check.Https) + { + return RuleEvaluation.NotApplicable("Kun for HTTPS-tjek."); + } + + var latest = context.Recent.FirstOrDefault(r => r.CertificateExpiresAt is not null); + if (latest?.CertificateExpiresAt is not { } expires) + { + return RuleEvaluation.NotMeasurable("Certifikatet er ikke set endnu (tjekket har ikke fået forbindelse)."); + } + + var days = (expires - context.Now).TotalDays; + if (days > context.Settings.CertificateWarningDays) + { + return RuleEvaluation.Passed(); + } + + var severity = days <= 0 ? Severity.Critical : days <= context.Settings.CertificateCriticalDays ? Severity.High : Severity.Medium; + return RuleEvaluation.Found(CreateFinding( + severity, + days <= 0 + ? $"Certifikatet for {context.Check.Name} er udløbet" + : $"Certifikatet for {context.Check.Name} udløber om {Math.Floor(days).ToString(CultureInfo.InvariantCulture)} dage", + "Når certifikatet udløber, afviser browsere og integrationer forbindelsen (eller viser en sikkerhedsadvarsel), og siden er i praksis nede.", + "Forny certifikatet og bind det nye i webserveren (IIS: Bindinger på webstedet). Tjek også, om andre servere bruger samme certifikat.", + [ + new Evidence("Udløber", SyntheticCheckRules.Time(expires)), + new Evidence("Certifikat", latest.CertificateSubject ?? "–"), + new Evidence("Adresse", context.Check.Target), + ], + subject: "certificate")); + } +} diff --git a/src/SQLInsight.Rules/Servers/WindowsServerRules.cs b/src/SQLInsight.Rules/Servers/WindowsServerRules.cs new file mode 100644 index 0000000..7da4c81 --- /dev/null +++ b/src/SQLInsight.Rules/Servers/WindowsServerRules.cs @@ -0,0 +1,594 @@ +using System.Globalization; +using SQLInsight.Core.Findings; +using SQLInsight.Core.Servers; +using SQLInsight.Rules.Engine; + +namespace SQLInsight.Rules.Servers; + +/// The rules about Windows servers (SQL Server hosts, AOS and other application servers, file servers). +public static class WindowsServerRules +{ + public static IReadOnlyList> Create() => + [ + new ServerCpuRule(), + new ServerMemoryRule(), + new ServerPagingRule(), + new ServerCommitRule(), + new DiskFreeSpaceRule(), + new DiskForecastRule(), + new StoppedAutomaticServiceRule(), + new WatchedServiceRule(), + new WindowsEventLogRule(), + new ServerRestartRule(), + new ServerUptimeRule(), + new FailedScheduledTaskRule(), + new MissedScheduledTaskRule(), + ]; + + internal static string Number(double value, int decimals = 0) => value.ToString("N" + decimals.ToString(CultureInfo.InvariantCulture), CultureInfo.GetCultureInfo("da-DK")); + + internal static string Time(DateTimeOffset? value) => value is { } v ? v.UtcDateTime.ToString("yyyy-MM-dd HH:mm 'UTC'", CultureInfo.InvariantCulture) : "–"; + + internal static Reference Learn(string title, string path) => new(title, new Uri("https://learn.microsoft.com/" + path)); + + /// A name in a list where entries may end in * (prefix). + internal static bool Matches(string name, IEnumerable patterns) => patterns.Any(p => + p.EndsWith('*') ? name.StartsWith(p[..^1], StringComparison.OrdinalIgnoreCase) : name.Equals(p, StringComparison.OrdinalIgnoreCase)); +} + +public sealed class ServerCpuRule : ScopedRule +{ + public override string Id => "SQLI-WIN-001"; + public override string Title => "CPU-forbruget er højt i længere tid"; + + public override RuleEvaluation Evaluate(ServerRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var performance = context.Data.Get(); + var settings = context.Settings.Cpu; + var period = TimeSpan.FromMinutes(Math.Max(1, settings.SustainedMinutes)); + var average = ServerPerformanceMath.SustainedAverage(performance.Samples, s => s.CpuPercent, period, context.Now, context.PerformanceInterval); + if (average is null) + { + return !performance.Samples.Any(s => s.CpuPercent is not null) + ? RuleEvaluation.NotMeasurable("CPU måles som forskellen mellem to målinger; der er endnu ikke to.") + : RuleEvaluation.Passed(); + } + + if (average < settings.HighPercent) + { + return RuleEvaluation.Passed(); + } + + var max = performance.Samples.Where(s => s.At > context.Now - period).Max(s => s.CpuPercent ?? 0); + return RuleEvaluation.Found(CreateFinding( + average >= 98 ? Severity.Critical : Severity.High, + $"CPU har været {WindowsServerRules.Number(average.Value)} % i gennemsnit de seneste {settings.SustainedMinutes} minutter", + "Serverens processorer har været næsten fuldt belastet i længere tid. Alt på serveren bliver langsommere: forespørgsler, AOS-kald, " + + "fildeling og logon. Ved kortvarige toppe er det normalt; vedvarende høj CPU er det ikke.", + "Find processen med Jobliste/Ressourceovervågning eller Performance Monitor (Process(*)\\% Processor Time). På en SQL Server-host: se Top-SQL for " + + "instansen i samme tidsrum. På en AOS: se batchjob og brugere i AX. Tjek også, om antivirus scanner database- eller AX-filer.", + [ + new Evidence("Gennemsnit", WindowsServerRules.Number(average.Value, 1) + " %"), + new Evidence("Højeste minut", WindowsServerRules.Number(max, 1) + " %"), + new Evidence("Periode", $"{settings.SustainedMinutes} minutter"), + new Evidence("Grænse", WindowsServerRules.Number(settings.HighPercent) + " %"), + new Evidence("Logiske processorer", performance.Os.LogicalProcessors?.ToString(CultureInfo.InvariantCulture) ?? "–"), + ], + subject: "cpu")); + } +} + +public sealed class ServerMemoryRule : ScopedRule +{ + public override string Id => "SQLI-WIN-002"; + public override string Title => "Serveren har næsten ingen ledig hukommelse"; + + public override RuleEvaluation Evaluate(ServerRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var performance = context.Data.Get(); + var settings = context.Settings.Memory; + var period = TimeSpan.FromMinutes(5); + var availableMb = ServerPerformanceMath.SustainedAverage(performance.Samples, s => s.AvailableMb, period, context.Now, context.PerformanceInterval); + var availablePercent = ServerPerformanceMath.SustainedAverage(performance.Samples, s => s.AvailablePercent, period, context.Now, context.PerformanceInterval); + if (availableMb is null || availablePercent is null) + { + return performance.Samples.Any(s => s.AvailableMb is not null) ? RuleEvaluation.Passed() : RuleEvaluation.NotMeasurable("Ledig hukommelse kunne ikke aflæses."); + } + + if (availablePercent >= settings.AvailablePercent || availableMb >= settings.AvailableMegabytes) + { + return RuleEvaluation.Passed(); + } + + var severe = availablePercent < 2 && availableMb < 256; + return RuleEvaluation.Found(CreateFinding( + severe ? Severity.High : Severity.Medium, + $"Kun {WindowsServerRules.Number(availableMb.Value)} MB ledig hukommelse ({WindowsServerRules.Number(availablePercent.Value, 1)} %)", + "Når Windows løber tør for ledig hukommelse, flyttes data til sidefilen på disken, og alt bliver langsomt. Processer kan fejle med " + + "\"out of memory\", og SQL Server kan blive tvunget til at give hukommelse tilbage.", + "Find den proces, der bruger hukommelsen (Jobliste, kolonnen Hukommelse/Commit). På en SQL Server-host: sæt max server memory, så Windows har " + + "mindst 2–4 GB (mere på store servere). På en AOS eller terminalserver: se antal sessioner og brugere. Overvej mere RAM.", + [ + new Evidence("Ledig hukommelse (5 min.)", WindowsServerRules.Number(availableMb.Value) + " MB"), + new Evidence("Andel ledig", WindowsServerRules.Number(availablePercent.Value, 1) + " %"), + new Evidence("Hukommelse i alt", performance.Os.TotalMemoryMb is { } total ? WindowsServerRules.Number(total) + " MB" : "–"), + new Evidence("Grænse", $"under {WindowsServerRules.Number(settings.AvailablePercent)} % og {WindowsServerRules.Number(settings.AvailableMegabytes)} MB"), + ], + subject: "memory", + WindowsServerRules.Learn("Server memory configuration options", "sql/database-engine/configure-windows/server-memory-server-configuration-options"))); + } +} + +public sealed class ServerPagingRule : ScopedRule +{ + public override string Id => "SQLI-WIN-003"; + public override string Title => "Serveren pager hukommelse til disk"; + + public override RuleEvaluation Evaluate(ServerRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var performance = context.Data.Get(); + var settings = context.Settings.Memory; + var period = TimeSpan.FromMinutes(15); + var pages = ServerPerformanceMath.SustainedAverage(performance.Samples, s => s.PagesPerSecond, period, context.Now, context.PerformanceInterval); + var available = ServerPerformanceMath.SustainedAverage(performance.Samples, s => s.AvailablePercent, period, context.Now, context.PerformanceInterval); + if (pages is null) + { + return performance.Samples.Any(s => s.PagesPerSecond is not null) + ? RuleEvaluation.Passed() + : RuleEvaluation.NotMeasurable("Paging måles som forskellen mellem to målinger af Memory\\Pages/sec; der er endnu ikke to."); + } + + // Pages/sec alone is also file I/O through memory-mapped files; together with little free memory it is paging. + if (pages < settings.PagesPerSecond || available is null or >= 10) + { + return RuleEvaluation.Passed(); + } + + return RuleEvaluation.Found(CreateFinding( + Severity.Medium, + $"Serveren pager {WindowsServerRules.Number(pages.Value)} sider pr. sekund med lidt ledig hukommelse", + "Windows læser og skriver hukommelsessider til sidefilen, fordi der ikke er plads i RAM. Det er mange gange langsommere end hukommelse og " + + "belaster disken, som andre også bruger.", + "Se SQLI-WIN-002 for hvem der bruger hukommelsen. Reducér forbruget (max server memory, færre sessioner) eller tilføj RAM.", + [ + new Evidence("Pages/sec (15 min.)", WindowsServerRules.Number(pages.Value)), + new Evidence("Andel ledig hukommelse", WindowsServerRules.Number(available.Value, 1) + " %"), + new Evidence("Grænse", WindowsServerRules.Number(settings.PagesPerSecond) + " pages/sec"), + ], + subject: "paging")); + } +} + +public sealed class ServerCommitRule : ScopedRule +{ + public override string Id => "SQLI-WIN-004"; + public override string Title => "Den reserverede hukommelse (commit) er tæt på grænsen"; + + public override RuleEvaluation Evaluate(ServerRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var performance = context.Data.Get(); + var commit = ServerPerformanceMath.SustainedAverage(performance.Samples, s => s.CommitPercent, TimeSpan.FromMinutes(5), context.Now, context.PerformanceInterval); + if (commit is null) + { + return performance.Samples.Any(s => s.CommitPercent is not null) ? RuleEvaluation.Passed() : RuleEvaluation.NotMeasurable("Commit-grænsen kunne ikke aflæses."); + } + + if (commit < context.Settings.Memory.CommitPercent) + { + return RuleEvaluation.Passed(); + } + + return RuleEvaluation.Found(CreateFinding( + commit >= 97 ? Severity.High : Severity.Medium, + $"{WindowsServerRules.Number(commit.Value)} % af commit-grænsen er brugt", + "Commit charge er den hukommelse, programmerne har reserveret (RAM plus sidefil). Når grænsen nås, kan Windows ikke give mere hukommelse ud: " + + "programmer og tjenester fejler eller går ned, selv om der måske er ledig RAM.", + "Find processen med størst commit (Jobliste, kolonnen Commit-størrelse). Et program, der vokser støt, lækker hukommelse. Øg sidefilen eller RAM, " + + "hvis forbruget er forventet.", + [new Evidence("Commit (5 min.)", WindowsServerRules.Number(commit.Value, 1) + " %"), new Evidence("Grænse", WindowsServerRules.Number(context.Settings.Memory.CommitPercent) + " %")], + subject: "commit")); + } +} + +public sealed class DiskFreeSpaceRule : ScopedRule +{ + public override string Id => "SQLI-WIN-005"; + public override string Title => "Lidt ledig plads på et drev"; + + public override RuleEvaluation Evaluate(ServerRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var disks = context.Data.Get(); + var settings = context.Settings.Disks; + return Result(disks.Disks + .Where(d => d.SizeGb > 0 && d.FreePercent < settings.FreePercent && d.FreeGb < settings.EnoughFreeGigabytes) + .Select(d => + { + var severity = d.FreeGb < 1 || d.FreePercent < 1 ? Severity.Critical : d.FreePercent < settings.CriticalFreePercent ? Severity.High : Severity.Medium; + var forecast = disks.Forecasts.FirstOrDefault(f => string.Equals(f.Volume, d.Volume, StringComparison.OrdinalIgnoreCase)); + return CreateFinding( + severity, + $"Drev {d.Volume} har kun {WindowsServerRules.Number(d.FreeGb, 1)} GB ledig ({WindowsServerRules.Number(d.FreePercent, 1)} %)", + "Når et drev løber fuldt, stopper alt, der skriver til det: SQL Server kan ikke vokse data- og logfiler (databasen bliver skrivebeskyttet " + + "i praksis), backup fejler, AOS og Windows kan ikke skrive logfiler og midlertidige filer, og brugere kan ikke gemme.", + "Find hvad der fylder (fx med WinDirStat eller TreeSize), ryd op i gamle backupfiler, logfiler og midlertidige filer, eller udvid drevet. " + + "Flyt ikke database- eller logfiler uden at planlægge det.", + [ + new Evidence("Ledig", WindowsServerRules.Number(d.FreeGb, 1) + " GB"), + new Evidence("Størrelse", WindowsServerRules.Number(d.SizeGb, 1) + " GB"), + new Evidence("Andel ledig", WindowsServerRules.Number(d.FreePercent, 1) + " %"), + new Evidence("Prognose", forecast?.Describe() ?? "–"), + new Evidence("Navn", d.Label ?? "–"), + ], + subject: "disk: " + d.Volume); + })); + } +} + +public sealed class DiskForecastRule : ScopedRule +{ + public override string Id => "SQLI-WIN-006"; + public override string Title => "Et drev bliver fuldt om få dage"; + + public override RuleEvaluation Evaluate(ServerRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var disks = context.Data.Get(); + var settings = context.Settings.Disks; + if (disks.Disks.Count > 0 && disks.Forecasts.Count == 0) + { + return RuleEvaluation.NotMeasurable("Prognosen beregnes ud fra SQLInsights egne målinger af ledig plads; der er endnu ingen historik."); + } + + return Result(disks.Forecasts + .Where(f => f.DaysUntilFull is { } days && days <= settings.ForecastDays) + .Select(f => + { + var days = f.DaysUntilFull!.Value; + var disk = disks.Disks.FirstOrDefault(d => string.Equals(d.Volume, f.Volume, StringComparison.OrdinalIgnoreCase)); + return CreateFinding( + days <= 2 ? Severity.Critical : days <= 7 ? Severity.High : Severity.Medium, + $"Disken {f.Volume} er {f.Describe()}", + $"Pladsen på drevet er de seneste {WindowsServerRules.Number(f.HistoryDays)} dage brugt med ca. {WindowsServerRules.Number(f.GrowthGbPerDay, 1)} GB " + + "om dagen. Fortsætter det, er drevet fuldt, og så stopper alt, der skriver til det (databaser, backup, logfiler). Prognosen er en ret linje " + + "gennem SQLInsights egne målinger og tager ikke højde for oprydning, der ikke er sket endnu.", + "Find hvad der vokser (databasefiler, backup, logfiler, brugerdata), og ryd op, flyt eller udvid drevet i god tid. Tjek at oprydningsjob og " + + "backup-rotation faktisk kører.", + [ + new Evidence("Fuldt om ca.", WindowsServerRules.Number(days, days < 10 ? 1 : 0) + " dage"), + new Evidence("Forventet fuld", f.FullAt(context.Now) is { } at ? at.UtcDateTime.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture) : "–"), + new Evidence("Vækst pr. dag", WindowsServerRules.Number(f.GrowthGbPerDay, 2) + " GB"), + new Evidence("Ledig nu", disk is null ? "–" : WindowsServerRules.Number(disk.FreeGb, 1) + " GB af " + WindowsServerRules.Number(disk.SizeGb, 1) + " GB"), + new Evidence("Historik", $"{WindowsServerRules.Number(f.HistoryDays, 1)} dage, {f.Samples} målinger, R² {WindowsServerRules.Number(f.Fit, 2)}"), + ], + subject: "forecast: " + f.Volume); + })); + } +} + +/// Services set to start automatically that are not running, except those that stop by design. +public sealed class StoppedAutomaticServiceRule : ScopedRule +{ + /// Automatic services that stop by themselves when they have nothing to do. + public static readonly IReadOnlyList StopByDesign = + [ + "sppsvc", "gupdate", "gupdatem", "edgeupdate", "edgeupdatem", "MapsBroker", "RemoteRegistry", "tiledatamodelsvc", "WbioSrvc", "DoSvc", "BITS", + "wuauserv", "TrustedInstaller", "ShellHWDetection", "clr_optimization_*", "OneSyncSvc_*", "CDPUserSvc_*", "GoogleUpdater*", "MozillaMaintenance", + "WaaSMedicSvc", "UsoSvc", "sedsvc", "AGMService", "AdobeARMservice", "Intel(R) TPM Provisioning Service", "SysmonLog", "TabletInputService", + ]; + + public override string Id => "SQLI-WIN-007"; + public override string Title => "En tjeneste, der skal starte automatisk, kører ikke"; + + public override RuleEvaluation Evaluate(ServerRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var services = context.Data.Get(); + if (InGracePeriod(context)) + { + return RuleEvaluation.Passed(); + } + + var ignore = StopByDesign.Concat(context.Settings.Services.Ignore).Concat(context.Server.IgnoredServices).ToList(); + return Result(services.Services + .Where(s => s.IsAutomatic && !s.IsRunning && !WindowsServerRules.Matches(s.Name, ignore) && !WindowsServerRules.Matches(s.Name, context.Server.WatchedServices)) + .Select(s => CreateFinding( + Severity.Medium, + $"Tjenesten {s.DisplayName} ({s.Name}) kører ikke", + $"Tjenesten er sat til at starte automatisk, men står som \"{s.State}\"" + (s.ExitCode is { } code and not 0 and not 1077 ? $" (afslutningskode {code})" : string.Empty) + + ". Det, tjenesten gør, sker ikke, og en automatisk tjeneste, der er stoppet, er oftest gået ned.", + "Se Windows' hændelseslog (System, Service Control Manager) for hvorfor tjenesten stoppede, og start den. SQLInsight starter aldrig selv tjenester. " + + "Skal tjenesten ikke køre, så sæt den til Manuel eller Deaktiveret, eller tilføj den til listen over tjenester, der ignoreres for serveren.", + [ + new Evidence("Tjeneste", s.Name), + new Evidence("Visningsnavn", s.DisplayName), + new Evidence("Status", s.State), + new Evidence("Start", s.StartMode + (s.DelayedAutoStart ? " (forsinket)" : string.Empty)), + new Evidence("Afslutningskode", s.ExitCode?.ToString(CultureInfo.InvariantCulture) ?? "–"), + ], + subject: "service: " + s.Name))); + } + + internal static bool InGracePeriod(ServerRuleContext context) => + context.Data.TryGet(out var performance) && performance.Os.LastBootUpTime is { } boot + && context.Now - boot < TimeSpan.FromMinutes(context.Settings.Services.StartupGraceMinutes); +} + +/// Services chosen for the server (the AOS, IIS, SQL Server …) that must run, whatever their start mode. +public sealed class WatchedServiceRule : ScopedRule +{ + public override string Id => "SQLI-WIN-008"; + public override string Title => "En overvåget tjeneste kører ikke"; + + public override RuleEvaluation Evaluate(ServerRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + if (context.Server.WatchedServices.Count == 0) + { + return RuleEvaluation.NotApplicable("Der er ikke valgt tjenester at overvåge på serveren."); + } + + var services = context.Data.Get(); + if (StoppedAutomaticServiceRule.InGracePeriod(context)) + { + return RuleEvaluation.Passed(); + } + + var findings = new List(); + foreach (var name in context.Server.WatchedServices) + { + var service = services.Services.FirstOrDefault(s => s.Name.Equals(name, StringComparison.OrdinalIgnoreCase) || s.DisplayName.Equals(name, StringComparison.OrdinalIgnoreCase)); + if (service is null) + { + findings.Add(CreateFinding( + Severity.Medium, + $"Den overvågede tjeneste {name} findes ikke eller kan ikke ses", + "Tjenesten står på listen over tjenester, der skal køre på serveren, men Windows viser den ikke. Den kan være afinstalleret eller omdøbt, " + + "eller agenten må ikke se den (tjenestens egne sikkerhedsindstillinger).", + "Tjek tjenestens navn (ikke visningsnavnet) i services.msc, og ret listen. Kør tildelingsscriptet igen, hvis agenten mangler rettigheder.", + [new Evidence("Tjeneste", name), new Evidence("Tjenester agenten kan se", services.Services.Count.ToString(CultureInfo.InvariantCulture))], + subject: "watched: " + name)); + continue; + } + + if (!service.IsRunning) + { + findings.Add(CreateFinding( + Severity.High, + $"Tjenesten {service.DisplayName} ({service.Name}) kører ikke", + $"Tjenesten er valgt som en, der skal køre på serveren, men står som \"{service.State}\". Det den leverer (fx AOS'en for AX-brugerne, IIS, " + + "SQL Server) er ikke tilgængeligt.", + "Se Windows' hændelseslog (System og Application) for hvorfor den stoppede, og start den igen. SQLInsight starter aldrig selv tjenester.", + [ + new Evidence("Tjeneste", service.Name), + new Evidence("Status", service.State), + new Evidence("Start", service.StartMode), + new Evidence("Afslutningskode", service.ExitCode?.ToString(CultureInfo.InvariantCulture) ?? "–"), + ], + subject: "watched: " + service.Name)); + } + } + + return Result(findings); + } +} + +public sealed class WindowsEventLogRule : ScopedRule +{ + public override string Id => "SQLI-WIN-009"; + public override string Title => "Fejl i Windows' hændelseslog"; + + public override RuleEvaluation Evaluate(ServerRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var events = context.Data.Get(); + var settings = context.Settings.EventLog; + return Result(events.Recent + .Where(g => g.Category != ServerEventCategory.UnexpectedShutdown) + .Where(g => g.Category != ServerEventCategory.Error || g.Count >= Math.Max(1, settings.RepeatThreshold)) + .OrderByDescending(g => g.Category).ThenByDescending(g => g.Count) + .Take(Math.Max(1, settings.MaxFindings)) + .Select(g => CreateFinding( + SeverityOf(g), + $"{ServerEventAnalysis.Display(g.Category)}: {g.Source} {g.EventId} ({g.Count} {(g.Count == 1 ? "gang" : "gange")} i {g.Log}-loggen)", + Explanation(g.Category), + Recommendation(g.Category), + [ + new Evidence("Log", g.Log), + new Evidence("Kilde", g.Source), + new Evidence("Hændelses-id", g.EventId.ToString(CultureInfo.InvariantCulture)), + new Evidence("Niveau", g.Level == 1 ? "Kritisk" : "Fejl"), + new Evidence("Antal (24 timer)", g.Count.ToString(CultureInfo.InvariantCulture)), + new Evidence("Første", WindowsServerRules.Time(g.FirstAt)), + new Evidence("Seneste", WindowsServerRules.Time(g.LastAt)), + new Evidence("Eksempel", g.Sample ?? "(ingen tekst)", Sensitive: true), + ], + subject: "event: " + g.Key))); + } + + private static Severity SeverityOf(ServerEventGroup group) => group.Category switch + { + ServerEventCategory.Disk or ServerEventCategory.Resources or ServerEventCategory.Critical => Severity.High, + ServerEventCategory.ServiceFailure => group.Count >= 3 ? Severity.High : Severity.Medium, + ServerEventCategory.DynamicsAx or ServerEventCategory.ApplicationCrash => Severity.Medium, + _ => Severity.Low, + }; + + private static string Explanation(ServerEventCategory category) => category switch + { + ServerEventCategory.Disk => + "Windows har haft fejl mod en disk, controller eller filsystemet: timeouts, nulstillinger eller sider, der ikke kunne læses eller skrives. " + + "Det går forud for datatab og meget langsom I/O, også i SQL Server (fejl 823/824/833).", + ServerEventCategory.Resources => + "Windows er løbet tør for en ressource (hukommelse, nonpaged pool, virtuel hukommelse). Programmer og tjenester fejler, når de ikke kan få mere.", + ServerEventCategory.Critical => "Windows har logget en kritisk hændelse: den højeste alvorsgrad, som bruges, når noget vigtigt ikke virker.", + ServerEventCategory.ServiceFailure => + "En tjeneste stoppede uventet eller kunne ikke starte. Det, tjenesten leverer, er væk, indtil den startes igen (eller Windows genstarter den).", + ServerEventCategory.DynamicsAx => + "Dynamics AX' Application Object Server (AOS) har logget fejl. De kan betyde, at brugere eller batchjob får fejl, eller at AOS'en har problemer " + + "med databasen.", + ServerEventCategory.ApplicationCrash => "Et program eller en tjeneste er gået ned eller holdt op med at svare. Det, programmet lavede, blev afbrudt.", + _ => "Den samme fejl er logget gentagne gange det seneste døgn. Gentagne fejl er sjældent tilfældige.", + }; + + private static string Recommendation(ServerEventCategory category) => category switch + { + ServerEventCategory.Disk => + "Se hændelsen i Logbog (Event Viewer) på serveren, og tjek disk, controller, drivere og SAN/hypervisor. Kør CHECKDB på databaser på drevet, " + + "og sørg for gyldige backups.", + ServerEventCategory.Resources => "Find processen, der bruger ressourcen (Jobliste, poolmon for nonpaged pool), og se SQLI-WIN-002 og -004.", + ServerEventCategory.ServiceFailure => "Se den næste hændelse fra tjenesten selv for årsagen, og tjek tjenestekontoens adgangskode og afhængigheder.", + ServerEventCategory.DynamicsAx => "Se hele hændelsen i Logbog på AOS-serveren og AOS'ens egen log. Gentagne databasefejl: se også SQL Server-fundene for AX-databasen.", + ServerEventCategory.ApplicationCrash => "Hændelsen nævner programmet og modulet, der fejlede. Søg på dem, installér opdateringer, og se om nedbruddet gentager sig.", + _ => "Se hændelsen i Logbog (Event Viewer) og slå kilden og hændelses-id'et op. Er den harmløs, kan den ignoreres under SQLInsight:Servers:EventLog:Ignore.", + }; +} + +/// A restart in the last day: unexpected (crash, power, blue screen) or planned. +public sealed class ServerRestartRule : ScopedRule +{ + public override string Id => "SQLI-WIN-010"; + public override string Title => "Serveren er genstartet"; + + public override RuleEvaluation Evaluate(ServerRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var performance = context.Data.Get(); + if (performance.Os.LastBootUpTime is not { } boot) + { + return RuleEvaluation.NotMeasurable("Starttidspunktet kunne ikke aflæses (Win32_OperatingSystem.LastBootUpTime)."); + } + + if (context.Now - boot >= ServerEventAnalysis.RecentWindow) + { + return RuleEvaluation.Passed(); + } + + context.Data.TryGet(out var events); + var crash = events?.Recent.FirstOrDefault(g => g.Category == ServerEventCategory.UnexpectedShutdown && g.LastAt >= boot.AddHours(-1)); + var evidence = new List + { + new("Startet", WindowsServerRules.Time(boot)), + new("Oppe i", Engine.Format.Age(context.Now - boot)), + }; + if (crash is not null) + { + evidence.Add(new Evidence("Hændelse", $"{crash.Source} {crash.EventId}")); + evidence.Add(new Evidence("Tekst", crash.Sample ?? "–", Sensitive: true)); + return RuleEvaluation.Found(CreateFinding( + Severity.High, + $"Serveren genstartede uventet {WindowsServerRules.Time(boot)}", + "Windows blev ikke lukket ned ordentligt: strømmen forsvandt, serveren gik ned (blå skærm) eller blev nulstillet hårdt, fx af hypervisoren. " + + "Alt, der kørte, blev afbrudt midt i det, og databaser og AX skal gendanne sig selv ved start.", + "Se Logbog (System) omkring tidspunktet: Kernel-Power 41, EventLog 6008 og BugCheck 1001 fortæller hvorfor. Ved blå skærm: se dumpfilen i " + + "C:\\Windows\\Minidump. Tjek hypervisor- og strømlogs, og at SQL Server-databaser og AOS kom korrekt op.", + evidence, + subject: "restart")); + } + + return RuleEvaluation.Found(CreateFinding( + Severity.Info, + $"Serveren er genstartet {WindowsServerRules.Time(boot)}", + "Serveren er genstartet det seneste døgn, uden tegn på at det skete uventet (fx planlagte Windows-opdateringer). Fundet er til orientering og " + + "forsvinder af sig selv efter et døgn.", + "Ingen handling, hvis genstarten var planlagt. Var den ikke, så se Logbog (System, User32 1074) for hvem eller hvad der genstartede serveren.", + evidence, + subject: "restart")); + } +} + +public sealed class ServerUptimeRule : ScopedRule +{ + public override string Id => "SQLI-WIN-011"; + public override string Title => "Serveren er ikke genstartet i lang tid"; + + public override RuleEvaluation Evaluate(ServerRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + if (context.Settings.UptimeDays <= 0) + { + return RuleEvaluation.NotApplicable("Slået fra (SQLInsight:Servers:UptimeDays er 0)."); + } + + var performance = context.Data.Get(); + if (performance.Os.LastBootUpTime is not { } boot) + { + return RuleEvaluation.NotMeasurable("Starttidspunktet kunne ikke aflæses."); + } + + var uptime = context.Now - boot; + return uptime.TotalDays < context.Settings.UptimeDays + ? RuleEvaluation.Passed() + : RuleEvaluation.Found(CreateFinding( + Severity.Low, + $"Serveren har kørt i {WindowsServerRules.Number(uptime.TotalDays)} dage uden genstart", + "De fleste Windows-opdateringer kræver en genstart for at virke. En server, der ikke er genstartet i flere måneder, mangler derfor næsten " + + "sikkert sikkerhedsopdateringer.", + "Planlæg et servicevindue og installér Windows-opdateringer. Tjek at opdateringsværktøjet (WSUS, Intune, …) faktisk når serveren.", + [new Evidence("Startet", WindowsServerRules.Time(boot)), new Evidence("Grænse", $"{context.Settings.UptimeDays} dage")], + subject: "uptime")); + } +} + +public sealed class FailedScheduledTaskRule : ScopedRule +{ + public override string Id => "SQLI-WIN-012"; + public override string Title => "En planlagt opgave fejlede sidste gang, den kørte"; + + public override RuleEvaluation Evaluate(ServerRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var tasks = context.Data.Get(); + return Result(tasks.Tasks + .Where(t => t.Enabled && TaskResults.IsFailure(t.LastTaskResult) && !WindowsServerRules.Matches(t.FullName, context.Settings.Tasks.Ignore)) + .Select(t => CreateFinding( + Severity.Medium, + $"Opgaven {t.FullName} fejlede: {TaskResults.Describe(t.LastTaskResult!.Value)}", + "Opgaven i Opgavestyring (Task Scheduler) sluttede med en fejlkode sidste gang, den kørte. Planlagte opgaver er ofte oprydning, eksport, " + + "integrationer eller backup af filer: når de fejler, opdager ingen det, før noget mangler.", + "Åbn opgaven i Opgavestyring, se fanen Historik og opgavens egen log. Tjek kontoen, opgaven kører som (adgangskode, rettigheder), og at " + + "programmet eller scriptet findes.", + [ + new Evidence("Opgave", t.FullName), + new Evidence("Resultat", TaskResults.Describe(t.LastTaskResult.Value)), + new Evidence("Seneste kørsel", WindowsServerRules.Time(t.LastRunTime)), + new Evidence("Næste kørsel", WindowsServerRules.Time(t.NextRunTime)), + ], + subject: "task: " + t.FullName))); + } +} + +public sealed class MissedScheduledTaskRule : ScopedRule +{ + public override string Id => "SQLI-WIN-013"; + public override string Title => "En planlagt opgave kørte ikke, da den skulle"; + + public override RuleEvaluation Evaluate(ServerRuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var tasks = context.Data.Get(); + var late = TimeSpan.FromMinutes(Math.Max(1, context.Settings.Tasks.LateMinutes)); + return Result(tasks.Tasks + .Where(t => t.Enabled && !string.Equals(t.State, "Running", StringComparison.OrdinalIgnoreCase) + && !WindowsServerRules.Matches(t.FullName, context.Settings.Tasks.Ignore) + && (t.MissedRuns > 0 || (t.NextRunTime is { } next && next < context.Now - late && (t.LastRunTime is null || t.LastRunTime < next)))) + .Select(t => CreateFinding( + Severity.Medium, + t.MissedRuns > 0 ? $"Opgaven {t.FullName} har sprunget {t.MissedRuns} kørsler over" : $"Opgaven {t.FullName} kørte ikke {WindowsServerRules.Time(t.NextRunTime)}", + "Opgaven skulle have kørt, men gjorde det ikke. Det sker, når Opgavestyring ikke kører, når opgavens betingelser ikke er opfyldt (fx \"kun når " + + "brugeren er logget på\"), eller når serveren var slukket. Det, opgaven gør, er ikke sket.", + "Se opgavens Historik i Opgavestyring og dens betingelser og indstillinger (\"Kør opgaven hurtigst muligt efter en mistet start\").", + [ + new Evidence("Opgave", t.FullName), + new Evidence("Planlagt", WindowsServerRules.Time(t.NextRunTime)), + new Evidence("Seneste kørsel", WindowsServerRules.Time(t.LastRunTime)), + new Evidence("Mistede kørsler", t.MissedRuns.ToString(CultureInfo.InvariantCulture)), + new Evidence("Status", t.State), + ], + subject: "task-missed: " + t.FullName))); + } +} diff --git a/tests/SQLInsight.Rules.Tests/WindowsServerRuleTests.cs b/tests/SQLInsight.Rules.Tests/WindowsServerRuleTests.cs new file mode 100644 index 0000000..ce3605b --- /dev/null +++ b/tests/SQLInsight.Rules.Tests/WindowsServerRuleTests.cs @@ -0,0 +1,334 @@ +using SQLInsight.Core.Collection; +using SQLInsight.Core.Findings; +using SQLInsight.Core.Servers; +using SQLInsight.Rules.Engine; +using SQLInsight.Rules.Servers; + +namespace SQLInsight.Rules.Tests; + +public class WindowsServerRuleTests +{ + private static readonly DateTimeOffset Now = new(2026, 10, 2, 12, 0, 0, TimeSpan.Zero); + + private static ServerIdentity Server(IReadOnlyList? watched = null, IReadOnlyList? ignored = null) => + new(1, "AOS01", "aos01.firma.local", ServerRoles.Application) { WatchedServices = watched ?? [], IgnoredServices = ignored ?? [] }; + + private static ServerRuleContext Context(CollectedData data, ServerIdentity? server = null, ServerRuleSettings? settings = null) => + new(server ?? Server(), Now, data, settings ?? new ServerRuleSettings()); + + private static ServerPerformanceSnapshot Performance(Func sample, int minutes = 60, DateTimeOffset? boot = null) => new() + { + CapturedAt = Now, + Os = new ServerOsInfo { Caption = "Windows Server 2022", TotalMemoryMb = 16384, LastBootUpTime = boot ?? Now.AddDays(-30), LogicalProcessors = 8 }, + Samples = Enumerable.Range(0, minutes).Select(m => sample(m) with { At = Now.AddMinutes(-m) }).Reverse().ToList(), + }; + + private static ServerPerformanceSample Sample(double? cpu = 20, double? availableMb = 8000, double? pages = 10, double? commit = 40) => + new(Now, cpu, availableMb, 16384, pages, commit); + + private static CollectedData With(T value) + where T : class + { + var data = new CollectedData(); + data.Set(value); + return data; + } + + private static RuleResult Run(IScopedRule rule, ServerRuleContext context) => Assert.Single(ScopedRuleEngine.Evaluate([rule], context)); + + [Fact] + public void Every_rule_has_a_unique_well_formed_id_and_a_title() + { + var all = ServerRuleCatalog.All; + + Assert.Equal(all.Count, all.Select(r => r.Id).Distinct().Count()); + Assert.All(all, r => Assert.Matches("^SQLI-(WIN|SYN)-[0-9]{3}$", r.Id)); + Assert.All(all, r => Assert.False(string.IsNullOrWhiteSpace(r.Title))); + Assert.DoesNotContain(all, r => BuiltInRules.Create().Any(b => b.Id == r.Id)); + } + + [Fact] + public void Missing_data_is_not_measurable_with_the_reason() + { + var data = new CollectedData(); + data.MarkUnavailable("Adgang nægtet til root\\cimv2."); + + var result = Run(new DiskFreeSpaceRule(), Context(data)); + + Assert.Equal(RuleOutcome.NotMeasurable, result.Outcome); + Assert.Equal("Adgang nægtet til root\\cimv2.", result.Reason); + } + + [Fact] + public void Sustained_high_cpu_is_a_finding_a_burst_is_not() + { + var sustained = Run(new ServerCpuRule(), Context(With(Performance(m => Sample(cpu: 95))))); + var burst = Run(new ServerCpuRule(), Context(With(Performance(m => Sample(cpu: m < 3 ? 100 : 30))))); + var first = Run(new ServerCpuRule(), Context(With(Performance(_ => Sample(cpu: null))))); + + Assert.Equal(Severity.High, Assert.Single(sustained.Findings).Severity); + Assert.Contains("95 %", sustained.Findings[0].Title, StringComparison.Ordinal); + Assert.Equal(RuleOutcome.Passed, burst.Outcome); + Assert.Equal(RuleOutcome.NotMeasurable, first.Outcome); + } + + [Fact] + public void Low_memory_needs_both_little_share_and_few_megabytes() + { + var low = Run(new ServerMemoryRule(), Context(With(Performance(_ => Sample(availableMb: 300))))); + var bigServer = Run(new ServerMemoryRule(), Context(With(Performance(_ => new ServerPerformanceSample(Now, 10, 4000, 262144, 0, 40))))); + + Assert.Equal(Severity.Medium, Assert.Single(low.Findings).Severity); + Assert.Equal(RuleOutcome.Passed, bigServer.Outcome); // 1,5 % of 256 GB, but 4 GB free + } + + [Fact] + public void Paging_only_counts_with_little_free_memory() + { + Assert.Single(Run(new ServerPagingRule(), Context(With(Performance(_ => Sample(availableMb: 900, pages: 5000))))).Findings); + Assert.Equal(RuleOutcome.Passed, Run(new ServerPagingRule(), Context(With(Performance(_ => Sample(availableMb: 8000, pages: 5000))))).Outcome); + Assert.Equal(Severity.High, Assert.Single(Run(new ServerCommitRule(), Context(With(Performance(_ => Sample(commit: 98))))).Findings).Severity); + } + + [Fact] + public void Disk_free_space_by_share_unless_there_is_plenty() + { + var disks = new ServerDiskSnapshot + { + CapturedAt = Now, + Disks = + [ + new ServerDisk(@"C:\", "System", "NTFS", 100, 30), + new ServerDisk(@"D:\", "Data", "NTFS", 500, 30), // 6 %: Medium + new ServerDisk(@"E:\", "Log", "NTFS", 100, 3), // 3 %: High + new ServerDisk(@"F:\", "Backup", "NTFS", 4000, 200), // 5 % but 200 GB: fine + new ServerDisk(@"G:\", "Temp", "NTFS", 50, 0.5), // under 1 GB: Critical + ], + }; + + var findings = Run(new DiskFreeSpaceRule(), Context(With(disks))).Findings.ToDictionary(f => f.Subject!, f => f.Severity); + + Assert.Equal(new Dictionary { [@"disk: D:\"] = Severity.Medium, [@"disk: E:\"] = Severity.High, [@"disk: G:\"] = Severity.Critical }, findings); + } + + [Fact] + public void Forecast_findings_by_days_until_full() + { + var disks = new ServerDiskSnapshot + { + CapturedAt = Now, + Disks = [new ServerDisk(@"D:\", null, "NTFS", 500, 100), new ServerDisk(@"E:\", null, "NTFS", 500, 100)], + Forecasts = + [ + new DiskForecast(@"D:\", 12, 8, 300, 14, 0.95), + new DiskForecast(@"E:\", 1.5, 60, 300, 14, 0.95), + new DiskForecast(@"F:\", null, 0, 300, 14, 0.1, "ingen tydelig tendens"), + new DiskForecast(@"G:\", 200, 0.5, 300, 14, 0.9), + ], + }; + + var findings = Run(new DiskForecastRule(), Context(With(disks))).Findings; + + Assert.Equal(2, findings.Count); + Assert.Contains(findings, f => f.Title == @"Disken D:\ er fuld om ca. 12 dage" && f.Severity == Severity.Medium); + Assert.Contains(findings, f => f.Subject == @"forecast: E:\" && f.Severity == Severity.Critical); + Assert.Equal(RuleOutcome.NotMeasurable, Run(new DiskForecastRule(), Context(With(disks with { Forecasts = [] }))).Outcome); + } + + private static ServerServiceSnapshot Services(params ServerService[] services) => new() { CapturedAt = Now, Services = services }; + + [Fact] + public void Stopped_automatic_services_except_the_ones_that_stop_by_design() + { + var data = With(Services( + new ServerService("AOS60$01", "Microsoft Dynamics AX Object Server 6.0$01", "Stopped", "Auto", ExitCode: 1067), + new ServerService("Spooler", "Print Spooler", "Running", "Auto"), + new ServerService("sppsvc", "Software Protection", "Stopped", "Auto", DelayedAutoStart: true), + new ServerService("clr_optimization_v4.0.30319_64", ".NET Optimization", "Stopped", "Auto"), + new ServerService("MyBatch", "Batch", "Stopped", "Auto"), + new ServerService("Manual1", "Manual", "Stopped", "Manual"))); + data.Set(Performance(_ => Sample())); + + var result = Run(new StoppedAutomaticServiceRule(), Context(data, Server(ignored: ["MyBatch"]))); + + var finding = Assert.Single(result.Findings); + Assert.Equal("service: AOS60$01", finding.Subject); + Assert.Contains("1067", finding.Explanation, StringComparison.Ordinal); + } + + [Fact] + public void Right_after_a_restart_stopped_services_are_still_starting() + { + var data = With(Services(new ServerService("AOS60$01", "AOS", "Stopped", "Auto"))); + data.Set(Performance(_ => Sample(), boot: Now.AddMinutes(-5))); + + Assert.Equal(RuleOutcome.Passed, Run(new StoppedAutomaticServiceRule(), Context(data)).Outcome); + } + + [Fact] + public void Watched_services_must_run_and_exist() + { + var data = With(Services(new ServerService("AOS60$01", "AOS", "Stopped", "Manual"), new ServerService("W3SVC", "IIS", "Running", "Auto"))); + + var result = Run(new WatchedServiceRule(), Context(data, Server(watched: ["AOS60$01", "W3SVC", "ReportServer"]))); + var none = Run(new WatchedServiceRule(), Context(data)); + var automatic = Run(new StoppedAutomaticServiceRule(), Context(data, Server(watched: ["AOS60$01"]))); + + Assert.Equal(2, result.Findings.Count); + Assert.Contains(result.Findings, f => f.Subject == "watched: AOS60$01" && f.Severity == Severity.High); + Assert.Contains(result.Findings, f => f.Subject == "watched: ReportServer" && f.Severity == Severity.Medium); + Assert.Equal(RuleOutcome.NotApplicable, none.Outcome); + Assert.Equal(RuleOutcome.Passed, automatic.Outcome); // not reported twice + } + + private static ServerEventGroup Group(string source, long id, ServerEventCategory category, int count, int level = 2) => + new("System", source, id, level, category, count, Now.AddHours(-2), Now.AddMinutes(-5), "besked med \\\\fil01\\hemmeligt"); + + [Fact] + public void Events_important_kinds_at_once_ordinary_errors_when_they_repeat() + { + var events = new ServerEventSnapshot + { + CapturedAt = Now, + ReadUntil = Now, + Recent = + [ + Group("disk", 153, ServerEventCategory.Disk, 1), + Group("Service Control Manager", 7031, ServerEventCategory.ServiceFailure, 4), + Group("Some-App", 9, ServerEventCategory.Error, 2), + Group("Other-App", 3, ServerEventCategory.Error, 12), + Group("EventLog", 6008, ServerEventCategory.UnexpectedShutdown, 1), + ], + }; + + var findings = Run(new WindowsEventLogRule(), Context(With(events))).Findings; + + Assert.Equal(["event: System/disk/153", "event: System/Service Control Manager/7031", "event: System/Other-App/3"], findings.Select(f => f.Subject).ToList()); + Assert.Equal([Severity.High, Severity.High, Severity.Low], findings.Select(f => f.Severity).ToList()); + Assert.True(findings[0].Evidence.Single(e => e.Name == "Eksempel").Sensitive); + } + + [Fact] + public void Event_findings_are_capped() + { + var events = new ServerEventSnapshot + { + CapturedAt = Now, + ReadUntil = Now, + Recent = Enumerable.Range(1, 40).Select(i => Group("App" + i, i, ServerEventCategory.ApplicationCrash, 1)).ToList(), + }; + + Assert.Equal(15, Run(new WindowsEventLogRule(), Context(With(events))).Findings.Count); + } + + [Fact] + public void Restart_unexpected_with_a_crash_event_otherwise_info() + { + var data = With(Performance(_ => Sample(), boot: Now.AddHours(-3))); + var planned = Run(new ServerRestartRule(), Context(data)); + data.Set(new ServerEventSnapshot + { + CapturedAt = Now, ReadUntil = Now, + Recent = [Group("Microsoft-Windows-Kernel-Power", 41, ServerEventCategory.UnexpectedShutdown, 1, level: 1) with { LastAt = Now.AddHours(-3) }], + }); + var crash = Run(new ServerRestartRule(), Context(data)); + var old = Run(new ServerRestartRule(), Context(With(Performance(_ => Sample(), boot: Now.AddDays(-3))))); + + Assert.Equal(Severity.Info, Assert.Single(planned.Findings).Severity); + Assert.Equal(Severity.High, Assert.Single(crash.Findings).Severity); + Assert.Contains("uventet", crash.Findings[0].Title, StringComparison.Ordinal); + Assert.Equal(RuleOutcome.Passed, old.Outcome); + } + + [Fact] + public void Uptime_rule_is_off_by_default() + { + var data = With(Performance(_ => Sample(), boot: Now.AddDays(-200))); + + Assert.Equal(RuleOutcome.NotApplicable, Run(new ServerUptimeRule(), Context(data)).Outcome); + Assert.Equal(Severity.Low, Assert.Single(Run(new ServerUptimeRule(), Context(data, settings: new ServerRuleSettings { UptimeDays = 120 })).Findings).Severity); + } + + [Fact] + public void Scheduled_tasks_failed_and_missed() + { + var tasks = new ServerTaskSnapshot + { + CapturedAt = Now, + Visible = 120, + Tasks = + [ + new ServerTask { Path = @"\", Name = "AX export", State = "Ready", LastTaskResult = 1, LastRunTime = Now.AddHours(-1), NextRunTime = Now.AddHours(23) }, + new ServerTask { Path = @"\", Name = "OK", State = "Ready", LastTaskResult = 0, LastRunTime = Now.AddHours(-1), NextRunTime = Now.AddHours(1) }, + new ServerTask { Path = @"\Firma\", Name = "Late", State = "Ready", LastTaskResult = 0, LastRunTime = Now.AddDays(-2), NextRunTime = Now.AddHours(-5) }, + new ServerTask { Path = @"\", Name = "Disabled", State = "Disabled", LastTaskResult = 1, NextRunTime = Now.AddHours(-5) }, + new ServerTask { Path = @"\", Name = "Missed", State = "Ready", LastTaskResult = 0, MissedRuns = 2 }, + new ServerTask { Path = @"\", Name = "Running", State = "Running", LastTaskResult = TaskResults.Running, NextRunTime = Now.AddHours(-5) }, + ], + }; + var data = With(tasks); + + var failed = Run(new FailedScheduledTaskRule(), Context(data)); + var missed = Run(new MissedScheduledTaskRule(), Context(data)); + + Assert.Equal(@"task: \AX export", Assert.Single(failed.Findings).Subject); + Assert.Equal([@"task-missed: \Firma\Late", @"task-missed: \Missed"], missed.Findings.Select(f => f.Subject).ToList()); + } +} + +public class SyntheticCheckRuleTests +{ + private static readonly DateTimeOffset Now = new(2026, 10, 2, 12, 0, 0, TimeSpan.Zero); + + private static SyntheticCheckIdentity Check(string kind = Core.SyntheticChecks.SyntheticCheckKind.Tcp, string? service = null, int? warn = null, bool https = false) => + new(7, "AOS01 port", kind, "aos01:2712") { ServerName = "AOS01", ServiceName = service, WarnMilliseconds = warn, Https = https }; + + private static Core.SyntheticChecks.SyntheticCheckResult Result(bool success, int minutesAgo, int ms = 20, string? service = null, DateTimeOffset? certificate = null) => + new() { At = Now.AddMinutes(-minutesAgo), Success = success, DurationMs = ms, Message = success ? "ok" : "Forbindelsen blev afvist.", ServiceState = service, CertificateExpiresAt = certificate }; + + private static RuleResult Run(IScopedRule rule, SyntheticCheckIdentity check, params Core.SyntheticChecks.SyntheticCheckResult[] newestFirst) => + Assert.Single(ScopedRuleEngine.Evaluate([rule], new SyntheticCheckRuleContext(check, Now, newestFirst, new SyntheticCheckRuleSettings()))); + + [Fact] + public void One_failure_is_not_an_alarm_two_in_a_row_are() + { + Assert.Equal(RuleOutcome.Passed, Run(new SyntheticCheckFailingRule(), Check(), Result(false, 0), Result(true, 5)).Outcome); + var failing = Run(new SyntheticCheckFailingRule(), Check(), Result(false, 0), Result(false, 5), Result(true, 10)); + Assert.Equal(Severity.High, Assert.Single(failing.Findings).Severity); + Assert.Equal("AOS01 port fejler: Forbindelsen blev afvist.", failing.Findings[0].Title); + Assert.Equal(RuleOutcome.NotMeasurable, Run(new SyntheticCheckFailingRule(), Check()).Outcome); + } + + [Fact] + public void Service_running_but_not_answering_is_said_plainly() + { + var running = Run(new SyntheticCheckFailingRule(), Check(service: "AOS60$01"), Result(false, 0, service: "Running"), Result(false, 5, service: "Running")); + var stopped = Run(new SyntheticCheckFailingRule(), Check(service: "AOS60$01"), Result(false, 0, service: "Stopped"), Result(false, 5, service: "Stopped")); + + Assert.Equal("AOS01 port svarer ikke, selv om tjenesten AOS60$01 kører", running.Findings[0].Title); + Assert.Contains("en tjenesteovervågning alene ikke ser", running.Findings[0].Explanation, StringComparison.Ordinal); + Assert.Equal("AOS01 port svarer ikke: tjenesten AOS60$01 er stoppet", stopped.Findings[0].Title); + } + + [Fact] + public void Slow_answers_three_times_in_a_row() + { + Assert.Equal(RuleOutcome.NotApplicable, Run(new SyntheticCheckSlowRule(), Check(), Result(true, 0, 5000)).Outcome); + var slow = Run(new SyntheticCheckSlowRule(), Check(warn: 1000), Result(true, 0, 1500), Result(true, 5, 2500), Result(true, 10, 1200)); + Assert.Equal(Severity.Medium, Assert.Single(slow.Findings).Severity); + Assert.Equal(RuleOutcome.Passed, Run(new SyntheticCheckSlowRule(), Check(warn: 1000), Result(true, 0, 1500), Result(true, 5, 500), Result(true, 10, 1200)).Outcome); + } + + [Fact] + public void Certificate_expiry() + { + var check = Check(Core.SyntheticChecks.SyntheticCheckKind.Http, https: true); + + Assert.Equal(RuleOutcome.Passed, Run(new CertificateExpiryRule(), check, Result(true, 0, certificate: Now.AddDays(60))).Outcome); + Assert.Equal(Severity.Medium, Run(new CertificateExpiryRule(), check, Result(true, 0, certificate: Now.AddDays(20))).Findings[0].Severity); + Assert.Equal(Severity.High, Run(new CertificateExpiryRule(), check, Result(true, 0, certificate: Now.AddDays(3))).Findings[0].Severity); + Assert.Equal(Severity.Critical, Run(new CertificateExpiryRule(), check, Result(false, 0, certificate: Now.AddDays(-1))).Findings[0].Severity); + Assert.Equal(RuleOutcome.NotApplicable, Run(new CertificateExpiryRule(), Check(), Result(true, 0)).Outcome); + Assert.Equal(RuleOutcome.NotMeasurable, Run(new CertificateExpiryRule(), check, Result(false, 0)).Outcome); + } +}