Availability groups: cluster node or witness down (D)

The availability group collector also reads sys.dm_hadr_cluster_members
(empty or unavailable without a WSFC), and SQLI-AG-003 (High) reports a
node or witness that is not up, with how many members are still up.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

README.md

@@ -4,7 +4,7 @@
 
 SQLInsight Pro er et overvågningsværktøj til Microsoft SQL Server, skrevet i C# på .NET. Det kører hos kunden og forlader ikke netværket, medmindre kunden selv vælger det. Værktøjet nøjes ikke med at vise tal. Det forklarer, hvad der er galt, hvorfor det er et problem, og hvordan det løses. Rådene har samme niveau som dem, man får fra en erfaren DBA.
 
-> **Status:** Fase 1–4 er implementeret: agent, 81 indbyggede regler plus community-regler, notifikationer, lokalt repository og web-UI, licenser, signerede opdateringer, SQLInsight Hub med portal, erfaring på tværs af kunder, PDF-rapporter og integration med Prometheus, Grafana og SCOM, plus MSI-installer, community-websted og Discourse-forum. Se [installation](docs/installation.md), [opdateringer](docs/opdatering.md), [integration](docs/integration.md), [rapporter](docs/rapporter.md), [egne regler](docs/regler.md), [databehandling](docs/databehandling.md), [community](docs/community.md), [Roadmap](#roadmap) og [CONTRIBUTING.md](CONTRIBUTING.md).
+> **Status:** Fase 1–4 er implementeret: agent, 82 indbyggede regler plus community-regler, notifikationer, lokalt repository og web-UI, licenser, signerede opdateringer, SQLInsight Hub med portal, erfaring på tværs af kunder, PDF-rapporter og integration med Prometheus, Grafana og SCOM, plus MSI-installer, community-websted og Discourse-forum. Se [installation](docs/installation.md), [opdateringer](docs/opdatering.md), [integration](docs/integration.md), [rapporter](docs/rapporter.md), [egne regler](docs/regler.md), [databehandling](docs/databehandling.md), [community](docs/community.md), [Roadmap](#roadmap) og [CONTRIBUTING.md](CONTRIBUTING.md).
 
 ---
 

docs/konfiguration.md

@@ -44,3 +44,7 @@ Uden den er reglerne "ikke målbar" med denne vejledning. På andre instanser en
 |---|---|---|
 | SQLI-REPL-001 | Høj | En distributionsagent fejlede eller prøver igen |
 | SQLI-REPL-002 | Middel | Ændringer er over 5 minutter om at nå abonnenten |
+
+## Availability groups og clusteret
+
+Ud over replikaernes og databasernes tilstand (SQLI-AG-001/002) læser agenten Windows-clusterets noder og vidne (`sys.dm_hadr_cluster_members`). **SQLI-AG-003** (Høj) melder, når en node eller vidnet ikke er oppe: clusteret har så færre stemmer, og mistes en til, går quorum og alle availability groups offline. Uden et Windows-cluster (CLUSTER_TYPE NONE eller EXTERNAL) er listen tom.

src/SQLInsight.Collectors/Instance/AvailabilityGroupCollector.cs

@@ -24,6 +24,11 @@ public sealed class AvailabilityGroupCollector : Collector<AvailabilityGroupSnap
         LEFT JOIN sys.dm_hadr_availability_replica_states ars ON ars.replica_id = drs.replica_id
         """;
 
+    // Without a Windows Server Failover Cluster (CLUSTER_TYPE = NONE/EXTERNAL) the view is empty or unavailable.
+    internal const string ClusterMembersSql = """
+        SELECT member_name, member_type_desc, member_state, member_state_desc, number_of_quorum_votes FROM sys.dm_hadr_cluster_members
+        """;
+
     public override string Name => "availability-groups";
     public override TimeSpan DefaultInterval => TimeSpan.FromMinutes(1);
     public override IReadOnlyCollection<Capability> RequiredCapabilities => [Capability.HadrEnabled];
@@ -61,6 +66,17 @@ public sealed class AvailabilityGroupCollector : Collector<AvailabilityGroupSnap
             LastCommitTime = r.NullableDateTime(13),
         }, ct).ConfigureAwait(false);
 
-        return new AvailabilityGroupSnapshot { CapturedAt = context.Now, Replicas = replicas, Databases = databases };
+        IReadOnlyList<ClusterMember> members;
+        try
+        {
+            members = await Db.QueryAsync(context.Connection, ClusterMembersSql, r => new ClusterMember(
+                r.String(0), r.NullableString(1) ?? "?", r.NullableInt(2) ?? 0, r.NullableString(3) ?? "?", r.NullableInt(4)), ct).ConfigureAwait(false);
+        }
+        catch (System.Data.Common.DbException)
+        {
+            members = [];
+        }
+
+        return new AvailabilityGroupSnapshot { CapturedAt = context.Now, Replicas = replicas, Databases = databases, ClusterMembers = members };
     }
 }

src/SQLInsight.Core/Data/HighAvailability.cs

@@ -8,6 +8,9 @@ public sealed record AvailabilityGroupSnapshot
     public required DateTimeOffset CapturedAt { get; init; }
     public required IReadOnlyList<ReplicaState> Replicas { get; init; }
     public required IReadOnlyList<DatabaseReplicaState> Databases { get; init; }
+
+    /// <summary>The Windows cluster's nodes and witness (sys.dm_hadr_cluster_members); empty without a WSFC.</summary>
+    public IReadOnlyList<ClusterMember> ClusterMembers { get; init; } = [];
 }
 
 public sealed record ReplicaState
@@ -83,3 +86,10 @@ public sealed record AgentJob
 
     public bool LastRunFailed => LastRunStatus == 0;
 }
+
+/// <param name="Type">CLUSTER_NODE, DISK_WITNESS, FILE_SHARE_WITNESS, CLOUD_WITNESS.</param>
+/// <param name="State">0 offline, 1 up.</param>
+public sealed record ClusterMember(string Name, string Type, int State, string StateText, int? QuorumVotes)
+{
+    public bool IsUp => State == 1;
+}

src/SQLInsight.Rules/Availability/AvailabilityRules.cs

@@ -175,3 +175,35 @@ public sealed class AgentStoppedRule : Rule
             references: Docs.Learn("SQL Server Agent", "ssms/agent/sql-server-agent")));
     }
 }
+
+/// <summary>
+/// A node or the witness of the Windows cluster under the availability groups is down: the cluster has fewer votes,
+/// and losing one more can take quorum, and with it every availability group, offline.
+/// </summary>
+public sealed class ClusterMemberDownRule : Rule
+{
+    public override string Id => "SQLI-AG-003";
+    public override string Title => "En node eller et vidne i clusteret er nede";
+    public override IReadOnlyCollection<Capability> RequiredCapabilities => [Capability.HadrEnabled];
+
+    public override RuleEvaluation Evaluate(RuleContext context)
+    {
+        ArgumentNullException.ThrowIfNull(context);
+        var snapshot = context.Data.Get<AvailabilityGroupSnapshot>();
+        return Result(snapshot.ClusterMembers.Where(m => !m.IsUp).Select(m => CreateFinding(
+            Severity.High,
+            $"{(m.Type == "CLUSTER_NODE" ? "Clusternoden" : "Vidnet")} {m.Name} er {m.StateText.ToLowerInvariant()}",
+            "Windows-clusteret under availability groups har mistet en node eller sit vidne. Clusteret kører videre, så længe der er quorum, "
+            + "men der er færre stemmer tilbage: mistes en til, går clusteret og dermed alle availability groups offline.",
+            "Få noden eller vidnet op igen (Failover Cluster Manager, netværk, fildelingen eller Azure-vidnet), og tjek clusterets hændelseslog. "
+            + "Overvej dynamisk quorum og vidnets placering, så en enkelt fejl ikke truer quorum.",
+            [
+                new Evidence("Medlem", m.Name),
+                new Evidence("Type", m.Type),
+                new Evidence("Tilstand", m.StateText),
+                new Evidence("Stemmer", m.QuorumVotes?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "–"),
+                new Evidence("Medlemmer oppe", $"{snapshot.ClusterMembers.Count(x => x.IsUp)} af {snapshot.ClusterMembers.Count}"),
+            ],
+            subject: $"cluster member: {m.Name}")));
+    }
+}

src/SQLInsight.Rules/BuiltInRules.cs

@@ -72,6 +72,7 @@ public static class BuiltInRules
             new StoppedServiceRule(),
             new ReplicationAgentFailedRule(),
             new ReplicationLatencyRule(),
+            new ClusterMemberDownRule(),
             new MemoryGrantsPendingRule(),
             new AnomalyRule(),
             new PlanRegressionRule(),

tests/SQLInsight.Rules.Tests/AvailabilityRuleTests.cs

@@ -19,6 +19,27 @@ public class AvailabilityRuleTests
         return context;
     }
 
+    [Fact]
+    public void A_cluster_node_or_witness_that_is_not_up_is_high()
+    {
+        var snapshot = new AvailabilityGroupSnapshot
+        {
+            CapturedAt = Now, Replicas = [], Databases = [],
+            ClusterMembers =
+            [
+                new ClusterMember("SQL01", "CLUSTER_NODE", 1, "UP", 1),
+                new ClusterMember("SQL02", "CLUSTER_NODE", 0, "DOWN", 1),
+                new ClusterMember("Cloud Witness", "CLOUD_WITNESS", 1, "UP", 1),
+            ],
+        };
+
+        var finding = Assert.Single(Run(new ClusterMemberDownRule(), Hadr(snapshot)).Findings);
+
+        Assert.Equal(Severity.High, finding.Severity);
+        Assert.Equal("cluster member: SQL02", finding.Subject);
+        Assert.Equal("2 af 3", finding.Evidence.Single(e => e.Name == "Medlemmer oppe").Value);
+    }
+
     private static DatabaseReplicaState Db(string replica, bool primary, DateTime? commit = null, long redoKb = 0, long redoRate = 1000, bool suspended = false, string health = "HEALTHY") => new()
     {
         Group = "ag1", Database = "Sales", Replica = replica, IsPrimaryReplica = primary, LastCommitTime = commit,

tests/SQLInsight.Rules.Tests/RuleCatalogTests.cs

@@ -18,6 +18,6 @@ public class RuleCatalogTests
     [Fact]
     public void Rule_count()
     {
-        Assert.Equal(81, BuiltInRules.Create().Count);
+        Assert.Equal(82, BuiltInRules.Create().Count);
     }
 }