Fase 2: Always On availability groups and SQL Agent jobs

- Availability group collector (replica and database replica states,
  queues, commit times) and rules: SQLI-AG-001 unhealthy/disconnected
  replica or suspended data movement (scripts HADR RESUME), SQLI-AG-002
  secondary behind (estimated data loss and redo time)
- SQL Agent collector using only granted msdb tables (no EXECUTE on msdb
  functions): SQLI-JOB-001 failed job with the failing step's message
  (backup/CHECKDB jobs are High), SQLI-JOB-002 SQL Agent not running
- Integration tests create a real availability group (CLUSTER_TYPE NONE)
  and a failing job in the test container (HADR and Agent enabled)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01E3EExwndy8Z22EK2Qpt6BK

src/SQLInsight.Collectors/BuiltInCollectors.cs

@@ -24,5 +24,7 @@ public static class BuiltInCollectors
         new QueryStoreCollector(),
         new PlanAnalysisCollector(),
         new IndexCollector(),
+        new AvailabilityGroupCollector(),
+        new AgentJobCollector(),
     ];
 }

src/SQLInsight.Collectors/Instance/AgentJobCollector.cs

@@ -0,0 +1,57 @@
+using System.Globalization;
+using SQLInsight.Core.Data;
+using SQLInsight.Core.Instances;
+
+namespace SQLInsight.Collectors.Instance;
+
+/// <summary>
+/// SQL Agent jobs from msdb. Uses only the tables granted by the least-privilege script (no msdb functions, which
+/// would need EXECUTE); run_date/run_time integers are converted here.
+/// </summary>
+public sealed class AgentJobCollector : Collector<AgentJobSnapshot>
+{
+    internal const string RunningSql = "SELECT COUNT(*) FROM sys.dm_exec_sessions WHERE program_name LIKE N'SQLAgent - %'";
+
+    internal const string JobsSql = """
+        DECLARE @since int = CONVERT(int, CONVERT(char(8), DATEADD(day, -1, GETDATE()), 112));
+        SELECT j.name, j.enabled, last_run.run_status, last_run.run_date, last_run.run_time,
+               failed_step.step_name, LEFT(failed_step.message, 2000), failures.n
+        FROM msdb.dbo.sysjobs j
+        OUTER APPLY (SELECT TOP (1) h.run_status, h.run_date, h.run_time FROM msdb.dbo.sysjobhistory h
+                     WHERE h.job_id = j.job_id AND h.step_id = 0 ORDER BY h.instance_id DESC) last_run
+        OUTER APPLY (SELECT TOP (1) h.step_name, h.message FROM msdb.dbo.sysjobhistory h
+                     WHERE h.job_id = j.job_id AND h.step_id > 0 AND h.run_status = 0 ORDER BY h.instance_id DESC) failed_step
+        OUTER APPLY (SELECT COUNT(*) AS n FROM msdb.dbo.sysjobhistory h
+                     WHERE h.job_id = j.job_id AND h.step_id = 0 AND h.run_status = 0 AND h.run_date >= @since) failures
+        """;
+
+    public override string Name => "agent-jobs";
+    public override TimeSpan DefaultInterval => TimeSpan.FromMinutes(5);
+    public override IReadOnlyCollection<Capability> RequiredCapabilities => [Capability.DynamicManagementViews];
+
+    protected override async Task<AgentJobSnapshot> CollectAsync(CollectorContext context, AgentJobSnapshot? previous)
+    {
+        var ct = context.CancellationToken;
+        var sessions = await Db.QueryAsync(context.Connection, RunningSql, r => r.Int(0), ct).ConfigureAwait(false);
+        var jobs = await Db.QueryAsync(context.Connection, JobsSql, r => new AgentJob
+        {
+            Name = r.String(0),
+            IsEnabled = r.Bool(1),
+            LastRunStatus = r.NullableInt(2),
+            LastRunAt = r.IsNull(3) ? null : ToDateTime(r.Int(3), r.Int(4)),
+            LastFailedStepName = r.NullableString(5),
+            LastFailedStepMessage = r.NullableString(6),
+            FailuresLastDay = r.NullableInt(7) ?? 0,
+        }, ct).ConfigureAwait(false);
+
+        return new AgentJobSnapshot { CapturedAt = context.Now, IsAgentRunning = sessions[0] > 0, Jobs = jobs };
+    }
+
+    /// <summary>msdb stores 2026-09-29 13:05:07 as run_date 20260929 and run_time 130507 (server local time).</summary>
+    internal static DateTime? ToDateTime(int runDate, int runTime) =>
+        DateTime.TryParseExact(
+            runDate.ToString("00000000", CultureInfo.InvariantCulture) + runTime.ToString("000000", CultureInfo.InvariantCulture),
+            "yyyyMMddHHmmss", CultureInfo.InvariantCulture, DateTimeStyles.None, out var value)
+            ? value
+            : null;
+}

src/SQLInsight.Collectors/Instance/AvailabilityGroupCollector.cs

@@ -0,0 +1,66 @@
+using SQLInsight.Core.Data;
+using SQLInsight.Core.Instances;
+
+namespace SQLInsight.Collectors.Instance;
+
+public sealed class AvailabilityGroupCollector : Collector<AvailabilityGroupSnapshot>
+{
+    internal const string ReplicasSql = """
+        SELECT ag.name, ar.replica_server_name, ars.is_local, ars.role_desc, ars.connected_state_desc, ars.synchronization_health_desc,
+               ar.availability_mode_desc, ar.failover_mode_desc
+        FROM sys.availability_groups ag
+        JOIN sys.availability_replicas ar ON ar.group_id = ag.group_id
+        LEFT JOIN sys.dm_hadr_availability_replica_states ars ON ars.replica_id = ar.replica_id
+        """;
+
+    internal const string DatabasesSql = """
+        SELECT ag.name, DB_NAME(drs.database_id), ar.replica_server_name, drs.is_local,
+               CASE WHEN ars.role = 1 THEN 1 ELSE 0 END,
+               drs.synchronization_state_desc, drs.synchronization_health_desc, drs.is_suspended, drs.suspend_reason_desc,
+               drs.log_send_queue_size, drs.log_send_rate, drs.redo_queue_size, drs.redo_rate, drs.last_commit_time
+        FROM sys.dm_hadr_database_replica_states drs
+        JOIN sys.availability_replicas ar ON ar.replica_id = drs.replica_id
+        JOIN sys.availability_groups ag ON ag.group_id = drs.group_id
+        LEFT JOIN sys.dm_hadr_availability_replica_states ars ON ars.replica_id = drs.replica_id
+        """;
+
+    public override string Name => "availability-groups";
+    public override TimeSpan DefaultInterval => TimeSpan.FromMinutes(1);
+    public override IReadOnlyCollection<Capability> RequiredCapabilities => [Capability.HadrEnabled];
+
+    protected override async Task<AvailabilityGroupSnapshot> CollectAsync(CollectorContext context, AvailabilityGroupSnapshot? previous)
+    {
+        var ct = context.CancellationToken;
+        var replicas = await Db.QueryAsync(context.Connection, ReplicasSql, r => new ReplicaState
+        {
+            Group = r.String(0),
+            Replica = r.String(1),
+            IsLocal = r.Bool(2),
+            Role = r.NullableString(3),
+            ConnectedState = r.NullableString(4),
+            SynchronizationHealth = r.NullableString(5),
+            AvailabilityMode = r.NullableString(6),
+            FailoverMode = r.NullableString(7),
+        }, ct).ConfigureAwait(false);
+
+        var databases = await Db.QueryAsync(context.Connection, DatabasesSql, r => new DatabaseReplicaState
+        {
+            Group = r.String(0),
+            Database = r.NullableString(1) ?? "?",
+            Replica = r.String(2),
+            IsLocal = r.Bool(3),
+            IsPrimaryReplica = r.Bool(4),
+            SynchronizationState = r.NullableString(5),
+            SynchronizationHealth = r.NullableString(6),
+            IsSuspended = r.Bool(7),
+            SuspendReason = r.NullableString(8),
+            LogSendQueueKb = r.NullableLong(9),
+            LogSendRateKbPerSecond = r.NullableLong(10),
+            RedoQueueKb = r.NullableLong(11),
+            RedoRateKbPerSecond = r.NullableLong(12),
+            LastCommitTime = r.NullableDateTime(13),
+        }, ct).ConfigureAwait(false);
+
+        return new AvailabilityGroupSnapshot { CapturedAt = context.Now, Replicas = replicas, Databases = databases };
+    }
+}

src/SQLInsight.Core/Data/HighAvailability.cs

@@ -0,0 +1,85 @@
+namespace SQLInsight.Core.Data;
+
+/// <summary>
+/// Always On availability group state as seen from this instance. A primary sees all replicas; a secondary only itself.
+/// </summary>
+public sealed record AvailabilityGroupSnapshot
+{
+    public required DateTimeOffset CapturedAt { get; init; }
+    public required IReadOnlyList<ReplicaState> Replicas { get; init; }
+    public required IReadOnlyList<DatabaseReplicaState> Databases { get; init; }
+}
+
+public sealed record ReplicaState
+{
+    public required string Group { get; init; }
+    public required string Replica { get; init; }
+    public bool IsLocal { get; init; }
+
+    /// <summary>PRIMARY, SECONDARY or RESOLVING; null when this instance cannot see the replica's state.</summary>
+    public string? Role { get; init; }
+
+    public string? ConnectedState { get; init; }
+
+    /// <summary>HEALTHY, PARTIALLY_HEALTHY or NOT_HEALTHY.</summary>
+    public string? SynchronizationHealth { get; init; }
+
+    public string? AvailabilityMode { get; init; }
+    public string? FailoverMode { get; init; }
+}
+
+public sealed record DatabaseReplicaState
+{
+    public required string Group { get; init; }
+    public required string Database { get; init; }
+    public required string Replica { get; init; }
+    public bool IsLocal { get; init; }
+    public bool IsPrimaryReplica { get; init; }
+    public string? SynchronizationState { get; init; }
+    public string? SynchronizationHealth { get; init; }
+    public bool IsSuspended { get; init; }
+    public string? SuspendReason { get; init; }
+
+    /// <summary>Log not yet sent to this secondary, in KB.</summary>
+    public long? LogSendQueueKb { get; init; }
+
+    public long? LogSendRateKbPerSecond { get; init; }
+
+    /// <summary>Log received but not yet redone on this secondary, in KB.</summary>
+    public long? RedoQueueKb { get; init; }
+
+    public long? RedoRateKbPerSecond { get; init; }
+    public DateTime? LastCommitTime { get; init; }
+}
+
+/// <summary>
+/// SQL Server Agent: whether it runs, and the latest outcome of each job.
+/// </summary>
+public sealed record AgentJobSnapshot
+{
+    public required DateTimeOffset CapturedAt { get; init; }
+
+    /// <summary>False when no SQL Agent session exists (the service is stopped).</summary>
+    public required bool IsAgentRunning { get; init; }
+
+    public required IReadOnlyList<AgentJob> Jobs { get; init; }
+}
+
+public sealed record AgentJob
+{
+    public required string Name { get; init; }
+    public bool IsEnabled { get; init; }
+
+    /// <summary>0 failed, 1 succeeded, 2 retry, 3 canceled, 4 in progress; null if never run.</summary>
+    public int? LastRunStatus { get; init; }
+
+    public DateTime? LastRunAt { get; init; }
+
+    /// <summary>The message of the most recent failed step, which says why (the job-level message does not).</summary>
+    public string? LastFailedStepMessage { get; init; }
+
+    public string? LastFailedStepName { get; init; }
+    public int FailuresLastDay { get; init; }
+
+    public bool LastRunFailed => LastRunStatus == 0;
+}

src/SQLInsight.Rules/Availability/AvailabilityRules.cs

@@ -0,0 +1,177 @@
+using SQLInsight.Core.Data;
+using SQLInsight.Core.Findings;
+using SQLInsight.Core.Instances;
+using SQLInsight.Rules.Engine;
+
+namespace SQLInsight.Rules.Availability;
+
+/// <summary>
+/// Availability replicas or databases that are not healthy: disconnected, not synchronizing or suspended.
+/// </summary>
+public sealed class AvailabilityGroupHealthRule : Rule
+{
+    public override string Id => "SQLI-AG-001";
+    public override string Title => "Always On availability group er ikke sund";
+    public override IReadOnlyCollection<Capability> RequiredCapabilities => [Capability.HadrEnabled];
+
+    private static readonly Reference Docs = Engine.Docs.Learn("Monitor availability groups", "database-engine/availability-groups/windows/monitoring-of-availability-groups-sql-server");
+
+    public override RuleEvaluation Evaluate(RuleContext context)
+    {
+        ArgumentNullException.ThrowIfNull(context);
+        var snapshot = context.Data.Get<AvailabilityGroupSnapshot>();
+        var findings = new List<Finding>();
+
+        foreach (var r in snapshot.Replicas.Where(r => r.ConnectedState == "DISCONNECTED" || r.SynchronizationHealth is "NOT_HEALTHY" or "PARTIALLY_HEALTHY"))
+        {
+            findings.Add(CreateFinding(
+                r.ConnectedState == "DISCONNECTED" || r.SynchronizationHealth == "NOT_HEALTHY" ? Severity.High : Severity.Medium,
+                $"{r.Group}: replika {r.Replica} er {(r.ConnectedState == "DISCONNECTED" ? "afbrudt" : r.SynchronizationHealth)}",
+                "Replikaen synkroniserer ikke normalt. Så længe det står på, er en failover til den enten umulig eller vil miste data, "
+                + "og på den primære kan transaktionsloggen ikke genbruges, fordi loggen skal gemmes til replikaen.",
+                "Tjek netværk og endpoint mellem replikaerne, SQL Server error log og AlwaysOn_health-sessionen på begge noder. "
+                + "Tjek også, om databaserne er suspenderet (se de enkelte databasefund).",
+                [new Evidence("Rolle", r.Role ?? "ukendt"), new Evidence("Forbindelse", r.ConnectedState ?? "ukendt"),
+                 new Evidence("Sundhed", r.SynchronizationHealth ?? "ukendt"), new Evidence("Tilstand", $"{r.AvailabilityMode}, {r.FailoverMode}")],
+                subject: $"replica: {r.Group}/{r.Replica}",
+                references: Docs));
+        }
+
+        foreach (var d in snapshot.Databases.Where(d => d.IsSuspended || d.SynchronizationHealth is "NOT_HEALTHY" or "PARTIALLY_HEALTHY"))
+        {
+            findings.Add(CreateFinding(
+                d.IsSuspended || d.SynchronizationHealth == "NOT_HEALTHY" ? Severity.High : Severity.Medium,
+                d.IsSuspended
+                    ? $"{d.Group}: data movement for {d.Database} på {d.Replica} er suspenderet ({d.SuspendReason})"
+                    : $"{d.Group}: {d.Database} på {d.Replica} er {d.SynchronizationHealth}",
+                "Databasen synkroniserer ikke til replikaen. Den primære skal gemme al log, til synkroniseringen genoptages, så logfilen vokser, "
+                + "og en failover vil miste de ændringer, der ikke er nået frem.",
+                d.IsSuspended
+                    ? "Find årsagen (suspend_reason og error log), ret den, og genoptag: ALTER DATABASE … SET HADR RESUME. "
+                      + "Er den suspenderet længe, kan det være hurtigere at gen-seede databasen."
+                    : "Tjek synkroniseringstilstand, netværk og redo på den sekundære. Se AlwaysOn_health-sessionen.",
+                [new Evidence("Synkronisering", d.SynchronizationState ?? "ukendt"), new Evidence("Sundhed", d.SynchronizationHealth ?? "ukendt"),
+                 new Evidence("Log send queue", d.LogSendQueueKb is { } q ? $"{Format.Number(q / 1024.0)} MB" : "–")],
+                subject: $"ag-database: {d.Group}/{d.Replica}/{d.Database}",
+                fixScript: d.IsSuspended ? $"ALTER DATABASE [{d.Database.Replace("]", "]]", StringComparison.Ordinal)}] SET HADR RESUME;" : null,
+                references: Docs));
+        }
+
+        return Result(findings);
+    }
+}
+
+/// <summary>
+/// Secondaries that are far behind: potential data loss (send queue / commit lag) or long failover time (redo queue).
+/// </summary>
+public sealed class AvailabilityGroupLagRule : Rule
+{
+    public const double MaxDataLossSeconds = 60;
+    public const double MaxRecoverySeconds = 600;
+
+    public override string Id => "SQLI-AG-002";
+    public override string Title => "Sekundær replika er langt bagud";
+    public override IReadOnlyCollection<Capability> RequiredCapabilities => [Capability.HadrEnabled];
+
+    public override RuleEvaluation Evaluate(RuleContext context)
+    {
+        ArgumentNullException.ThrowIfNull(context);
+        var snapshot = context.Data.Get<AvailabilityGroupSnapshot>();
+        var findings = new List<Finding>();
+        foreach (var secondary in snapshot.Databases.Where(d => !d.IsPrimaryReplica && !d.IsSuspended))
+        {
+            var primary = snapshot.Databases.FirstOrDefault(p => p.IsPrimaryReplica && p.Group == secondary.Group && p.Database == secondary.Database);
+            double? dataLoss = primary?.LastCommitTime is { } p && secondary.LastCommitTime is { } s ? Math.Max(0, (p - s).TotalSeconds) : null;
+            double? recovery = secondary.RedoQueueKb is { } redo && secondary.RedoRateKbPerSecond is > 0 ? (double)redo / secondary.RedoRateKbPerSecond.Value : null;
+
+            if ((dataLoss ?? 0) < MaxDataLossSeconds && (recovery ?? 0) < MaxRecoverySeconds)
+            {
+                continue;
+            }
+
+            findings.Add(CreateFinding(
+                Severity.Medium,
+                $"{secondary.Group}: {secondary.Database} på {secondary.Replica} er bagud"
+                    + (dataLoss >= MaxDataLossSeconds ? $" (op til {Format.Number(dataLoss!.Value)} s datatab)" : string.Empty)
+                    + (recovery >= MaxRecoverySeconds ? $" (redo ca. {Format.Number(recovery!.Value / 60)} min)" : string.Empty),
+                "En stor send queue betyder, at ændringer endnu ikke er nået til replikaen; ved en failover nu mistes de. En stor redo queue "
+                + "betyder, at replikaen har loggen, men mangler at anvende den; en failover tager så tilsvarende længere tid.",
+                "Tjek netværksbåndbredde og latens mellem replikaerne, og I/O på den sekundære (redo er ofte begrænset af skrivelatens eller "
+                + "læsetrafik på en readable secondary). Store indexvedligehold og bulk loads giver midlertidige køer.",
+                [
+                    new Evidence("Log send queue", secondary.LogSendQueueKb is { } q ? $"{Format.Number(q / 1024.0)} MB" : "–"),
+                    new Evidence("Redo queue", secondary.RedoQueueKb is { } r ? $"{Format.Number(r / 1024.0)} MB" : "–"),
+                    new Evidence("Redo rate", secondary.RedoRateKbPerSecond is { } rr ? $"{Format.Number(rr / 1024.0, 1)} MB/s" : "–"),
+                ],
+                subject: $"ag-database: {secondary.Group}/{secondary.Replica}/{secondary.Database}",
+                references: Docs.Learn("Monitor performance for availability groups", "database-engine/availability-groups/windows/monitor-performance-for-always-on-availability-groups")));
+        }
+
+        return Result(findings);
+    }
+}
+
+/// <summary>
+/// SQL Agent jobs whose most recent run failed.
+/// </summary>
+public sealed class FailedJobRule : Rule
+{
+    private static readonly string[] Critical = ["backup", "checkdb", "integrity", "dbcc"];
+
+    public override string Id => "SQLI-JOB-001";
+    public override string Title => "SQL Agent-job fejler";
+    public override IReadOnlyCollection<Capability> RequiredCapabilities => [Capability.DynamicManagementViews];
+
+    public override RuleEvaluation Evaluate(RuleContext context)
+    {
+        ArgumentNullException.ThrowIfNull(context);
+        return Result(context.Data.Get<AgentJobSnapshot>().Jobs.Where(j => j.IsEnabled && j.LastRunFailed).Select(j =>
+        {
+            var critical = Critical.Any(c => j.Name.Contains(c, StringComparison.OrdinalIgnoreCase));
+            return CreateFinding(
+                critical ? Severity.High : Severity.Medium,
+                $"Jobbet '{j.Name}' fejlede senest {Format.Date(j.LastRunAt)}",
+                critical
+                    ? "Jobbet ser ud til at stå for backup eller integritetskontrol. Så længe det fejler, bliver der ikke taget backup eller kørt CHECKDB."
+                    : "Seneste kørsel af jobbet fejlede. Det, jobbet skulle gøre, er ikke sket.",
+                "Læs fejlbeskeden fra det fejlende trin (nedenfor eller i jobhistorikken), ret årsagen, og kør jobbet igen. "
+                + "Sørg for, at fejlende jobs sender en notifikation til en operator.",
+                [
+                    new Evidence("Fejlende trin", j.LastFailedStepName ?? "–"),
+                    new Evidence("Fejl", j.LastFailedStepMessage ?? "–"),
+                    new Evidence("Fejl det seneste døgn", j.FailuresLastDay.ToString(System.Globalization.CultureInfo.InvariantCulture)),
+                ],
+                subject: $"job: {j.Name}",
+                references: Docs.Learn("View job history", "ssms/agent/view-the-job-history"));
+        }));
+    }
+}
+
+/// <summary>
+/// The SQL Agent service is not running, so no scheduled job (backups!) runs.
+/// </summary>
+public sealed class AgentStoppedRule : Rule
+{
+    public override string Id => "SQLI-JOB-002";
+    public override string Title => "SQL Server Agent kører ikke";
+    public override IReadOnlyCollection<Capability> RequiredCapabilities => [Capability.DynamicManagementViews];
+
+    public override RuleEvaluation Evaluate(RuleContext context)
+    {
+        ArgumentNullException.ThrowIfNull(context);
+        var snapshot = context.Data.Get<AgentJobSnapshot>();
+        if (snapshot.IsAgentRunning || context.Instance.EngineEdition == EngineEdition.Express)
+        {
+            return RuleEvaluation.Passed();
+        }
+
+        var scheduled = snapshot.Jobs.Count(j => j.IsEnabled);
+        return RuleEvaluation.Found(CreateFinding(
+            scheduled > 0 ? Severity.High : Severity.Low,
+            scheduled > 0 ? $"SQL Server Agent kører ikke; {scheduled} aktive jobs bliver ikke udført" : Title,
+            "Uden SQL Server Agent kører ingen planlagte jobs: backups, integritetskontrol, indexvedligehold og alarmer.",
+            "Start tjenesten SQL Server Agent, og sæt den til at starte automatisk. Tjek dens log for, hvorfor den er stoppet.",
+            [new Evidence("Aktive jobs", scheduled.ToString(System.Globalization.CultureInfo.InvariantCulture))],
+            references: Docs.Learn("SQL Server Agent", "ssms/agent/sql-server-agent")));
+    }
+}

src/SQLInsight.Rules/BuiltInRules.cs

@@ -1,3 +1,4 @@
+using SQLInsight.Rules.Availability;
 using SQLInsight.Rules.Concurrency;
 using SQLInsight.Rules.Configuration;
 using SQLInsight.Rules.Databases;
@@ -63,6 +64,12 @@ public static class BuiltInRules
         new StaleStatisticsRule(),
         new AutoStatisticsRule(),
 
+        // Availability and jobs
+        new AvailabilityGroupHealthRule(),
+        new AvailabilityGroupLagRule(),
+        new FailedJobRule(),
+        new AgentStoppedRule(),
+
         // Concurrency
         new BlockingRule(),
         new DeadlockRule(),

tests/SQLInsight.Collectors.Tests/SqlSelectionTests.cs

@@ -66,3 +66,14 @@ public class SqlSelectionTests
         Assert.Contains("N'/var/opt/mssql/log/system_health_0_1''2.xel', 12345", sql, StringComparison.Ordinal);
     }
 }
+
+public class AgentJobTimeTests
+{
+    [Theory]
+    [InlineData(20260929, 130507, "2026-09-29 13:05:07")]
+    [InlineData(20260101, 5, "2026-01-01 00:00:05")]
+    public void Converts_msdb_run_date_and_time(int date, int time, string expected)
+    {
+        Assert.Equal(DateTime.Parse(expected, System.Globalization.CultureInfo.InvariantCulture), SQLInsight.Collectors.Instance.AgentJobCollector.ToDateTime(date, time));
+    }
+}

tests/SQLInsight.IntegrationTests/HighAvailabilityTests.cs

@@ -0,0 +1,82 @@
+using Microsoft.Data.SqlClient;
+using SQLInsight.Collectors.Instance;
+using SQLInsight.Core.Data;
+using SQLInsight.Core.Instances;
+
+namespace SQLInsight.IntegrationTests;
+
+[Collection(SqlServerTestGroup.Name)]
+[Trait("Category", "Integration")]
+public class HighAvailabilityTests(SqlServerFixture sql)
+{
+    [Fact]
+    public async Task Availability_group_state_is_read_with_least_privilege()
+    {
+        await sql.ExecuteAsSaAsync("""
+            IF DB_ID(N'AgProbe') IS NULL
+            BEGIN
+                CREATE DATABASE AgProbe;
+            END;
+            """);
+        await sql.ExecuteAsSaAsync("""
+            IF NOT EXISTS (SELECT 1 FROM sys.availability_groups WHERE name = N'ag_probe')
+            BEGIN
+                ALTER DATABASE AgProbe SET RECOVERY FULL;
+                BACKUP DATABASE AgProbe TO DISK = N'/tmp/agprobe.bak' WITH INIT;
+                IF NOT EXISTS (SELECT 1 FROM sys.symmetric_keys WHERE name = N'##MS_DatabaseMasterKey##')
+                    CREATE MASTER KEY ENCRYPTION BY PASSWORD = N'Mk-Pa55word!';
+                IF NOT EXISTS (SELECT 1 FROM sys.certificates WHERE name = N'ag_probe_cert')
+                    CREATE CERTIFICATE ag_probe_cert WITH SUBJECT = N'ag_probe';
+                IF NOT EXISTS (SELECT 1 FROM sys.endpoints WHERE name = N'hadr_endpoint')
+                    CREATE ENDPOINT hadr_endpoint STATE = STARTED AS TCP (LISTENER_PORT = 5022)
+                        FOR DATABASE_MIRRORING (ROLE = ALL, AUTHENTICATION = CERTIFICATE ag_probe_cert, ENCRYPTION = REQUIRED ALGORITHM AES);
+                DECLARE @server sysname = CAST(SERVERPROPERTY('ServerName') AS sysname);
+                EXEC (N'CREATE AVAILABILITY GROUP ag_probe WITH (CLUSTER_TYPE = NONE) FOR DATABASE AgProbe
+                        REPLICA ON N''' + @server + N''' WITH (ENDPOINT_URL = N''tcp://localhost:5022'', AVAILABILITY_MODE = ASYNCHRONOUS_COMMIT,
+                        FAILOVER_MODE = MANUAL, SEEDING_MODE = AUTOMATIC);');
+            END;
+            """);
+
+        await using var session = await AgentSession.OpenAsync(sql);
+        Assert.True(session.Capabilities.Has(Capability.HadrEnabled));
+        var snapshot = await session.CollectAsync<AvailabilityGroupSnapshot>(new AvailabilityGroupCollector());
+
+        var replica = Assert.Single(snapshot.Replicas, r => r.Group == "ag_probe");
+        Assert.True(replica.IsLocal);
+        Assert.Equal("PRIMARY", replica.Role);
+        var database = Assert.Single(snapshot.Databases, d => d.Group == "ag_probe");
+        Assert.Equal("AgProbe", database.Database);
+        Assert.True(database.IsPrimaryReplica);
+    }
+
+    [Fact]
+    public async Task Failing_agent_job_is_reported_with_the_step_message()
+    {
+        await sql.ExecuteAsSaAsync("""
+            IF EXISTS (SELECT 1 FROM msdb.dbo.sysjobs WHERE name = N'SQLInsight probe job')
+                EXEC msdb.dbo.sp_delete_job @job_name = N'SQLInsight probe job';
+            EXEC msdb.dbo.sp_add_job @job_name = N'SQLInsight probe job';
+            EXEC msdb.dbo.sp_add_jobstep @job_name = N'SQLInsight probe job', @step_name = N'Fail on purpose',
+                 @subsystem = N'TSQL', @command = N'RAISERROR(N''sqlinsight-probe-failure'', 16, 1);';
+            EXEC msdb.dbo.sp_add_jobserver @job_name = N'SQLInsight probe job';
+            EXEC msdb.dbo.sp_start_job @job_name = N'SQLInsight probe job';
+            """);
+
+        await using var session = await AgentSession.OpenAsync(sql);
+        AgentJob? job = null;
+        for (var attempt = 0; attempt < 60 && job?.LastRunFailed != true; attempt++)
+        {
+            await Task.Delay(1000);
+            var snapshot = await session.CollectAsync<AgentJobSnapshot>(new AgentJobCollector());
+            Assert.True(snapshot.IsAgentRunning);
+            job = snapshot.Jobs.SingleOrDefault(j => j.Name == "SQLInsight probe job");
+        }
+
+        Assert.NotNull(job);
+        Assert.True(job.LastRunFailed);
+        Assert.Equal("Fail on purpose", job.LastFailedStepName);
+        Assert.Contains("sqlinsight-probe-failure", job.LastFailedStepMessage, StringComparison.Ordinal);
+        Assert.True(job.FailuresLastDay >= 1);
+        Assert.NotNull(job.LastRunAt);
+    }
+}

tests/SQLInsight.IntegrationTests/SqlServerFixture.cs

@@ -16,7 +16,11 @@ public sealed class SqlServerFixture : IAsyncLifetime
 
     public static string Image => Environment.GetEnvironmentVariable("SQLINSIGHT_MSSQL_IMAGE") ?? DefaultImage;
 
-    private readonly MsSqlContainer _container = new MsSqlBuilder(Image).Build();
+    // HADR and SQL Agent on, so availability group and job collectors run against the real DMVs.
+    private readonly MsSqlContainer _container = new MsSqlBuilder(Image)
+        .WithEnvironment("MSSQL_ENABLE_HADR", "1")
+        .WithEnvironment("MSSQL_AGENT_ENABLED", "true")
+        .Build();
 
     public string SaConnectionString => _container.GetConnectionString();
 

tests/SQLInsight.Rules.Tests/AvailabilityRuleTests.cs

@@ -0,0 +1,117 @@
+using SQLInsight.Core.Data;
+using SQLInsight.Core.Findings;
+using SQLInsight.Core.Instances;
+using SQLInsight.Rules.Availability;
+using SQLInsight.Rules.Engine;
+using static SQLInsight.Rules.Tests.TestInstances;
+
+namespace SQLInsight.Rules.Tests;
+
+public class AvailabilityRuleTests
+{
+    private static RuleResult Run(IRule rule, RuleContext context) => Assert.Single(new RuleEngine([rule]).Evaluate(context));
+
+    private static RuleContext Hadr(AvailabilityGroupSnapshot snapshot)
+    {
+        var info = new InstanceInfo { ServerName = "n1", Version = SqlServerVersion.Parse("16.0.4295.3"), Edition = "Enterprise", EngineEdition = EngineEdition.Enterprise, IsHadrEnabled = true };
+        var context = new RuleContext(info, InstanceCapabilities.From(info), Now);
+        context.Data.Set(snapshot);
+        return context;
+    }
+
+    private static DatabaseReplicaState Db(string replica, bool primary, DateTime? commit = null, long redoKb = 0, long redoRate = 1000, bool suspended = false, string health = "HEALTHY") => new()
+    {
+        Group = "ag1", Database = "Sales", Replica = replica, IsPrimaryReplica = primary, LastCommitTime = commit,
+        RedoQueueKb = redoKb, RedoRateKbPerSecond = redoRate, IsSuspended = suspended, SuspendReason = suspended ? "SUSPEND_FROM_USER" : null,
+        SynchronizationHealth = health, SynchronizationState = "SYNCHRONIZED", LogSendQueueKb = 0,
+    };
+
+    [Fact]
+    public void Not_measurable_without_hadr()
+    {
+        Assert.Equal(RuleOutcome.NotMeasurable, Run(new AvailabilityGroupHealthRule(), Context("16.0.4295.3")).Outcome);
+    }
+
+    [Fact]
+    public void Healthy_group_passes()
+    {
+        var snapshot = new AvailabilityGroupSnapshot
+        {
+            CapturedAt = Now,
+            Replicas = [new ReplicaState { Group = "ag1", Replica = "n1", Role = "PRIMARY", ConnectedState = "CONNECTED", SynchronizationHealth = "HEALTHY" }],
+            Databases = [Db("n1", true, Now.UtcDateTime), Db("n2", false, Now.UtcDateTime)],
+        };
+
+        Assert.Equal(RuleOutcome.Passed, Run(new AvailabilityGroupHealthRule(), Hadr(snapshot)).Outcome);
+        Assert.Equal(RuleOutcome.Passed, Run(new AvailabilityGroupLagRule(), Hadr(snapshot)).Outcome);
+    }
+
+    [Fact]
+    public void Disconnected_replica_and_suspended_database_are_high()
+    {
+        var snapshot = new AvailabilityGroupSnapshot
+        {
+            CapturedAt = Now,
+            Replicas = [new ReplicaState { Group = "ag1", Replica = "n2", Role = "SECONDARY", ConnectedState = "DISCONNECTED", SynchronizationHealth = "NOT_HEALTHY" }],
+            Databases = [Db("n2", false, suspended: true, health: "NOT_HEALTHY")],
+        };
+
+        var findings = Run(new AvailabilityGroupHealthRule(), Hadr(snapshot)).Findings;
+
+        Assert.Equal(2, findings.Count);
+        Assert.All(findings, f => Assert.Equal(Severity.High, f.Severity));
+        Assert.Contains(findings, f => f.FixScript == "ALTER DATABASE [Sales] SET HADR RESUME;");
+    }
+
+    [Fact]
+    public void Lag_reports_data_loss_and_redo_time()
+    {
+        var snapshot = new AvailabilityGroupSnapshot
+        {
+            CapturedAt = Now,
+            Replicas = [],
+            Databases = [Db("n1", true, Now.UtcDateTime), Db("n2", false, Now.UtcDateTime.AddMinutes(-5), redoKb: 2_000_000, redoRate: 1000)],
+        };
+
+        var finding = Assert.Single(Run(new AvailabilityGroupLagRule(), Hadr(snapshot)).Findings);
+
+        Assert.Contains("300 s datatab", finding.Title, StringComparison.Ordinal);
+        Assert.Contains("redo ca. 33 min", finding.Title, StringComparison.Ordinal);
+    }
+
+    private static AgentJobSnapshot Jobs(bool running, params AgentJob[] jobs) => new() { CapturedAt = Now, IsAgentRunning = running, Jobs = jobs };
+
+    [Fact]
+    public void Failed_backup_job_is_high_with_step_message()
+    {
+        var job = new AgentJob { Name = "DatabaseBackup - USER_DATABASES - FULL", IsEnabled = true, LastRunStatus = 0, LastFailedStepName = "Backup", LastFailedStepMessage = "Cannot open backup device 'X:\\bak'" };
+
+        var finding = Assert.Single(Run(new FailedJobRule(), With(Jobs(true, job))).Findings);
+
+        Assert.Equal(Severity.High, finding.Severity);
+        Assert.Contains(finding.Evidence, e => e.Value.Contains("Cannot open backup device", StringComparison.Ordinal));
+    }
+
+    [Fact]
+    public void Disabled_or_succeeding_jobs_pass()
+    {
+        var jobs = Jobs(true,
+            new AgentJob { Name = "Old", IsEnabled = false, LastRunStatus = 0 },
+            new AgentJob { Name = "Fine", IsEnabled = true, LastRunStatus = 1 },
+            new AgentJob { Name = "Never run", IsEnabled = true });
+
+        Assert.Equal(RuleOutcome.Passed, Run(new FailedJobRule(), With(jobs)).Outcome);
+    }
+
+    [Fact]
+    public void Stopped_agent_is_high_when_jobs_exist_but_not_on_express()
+    {
+        var jobs = Jobs(false, new AgentJob { Name = "Backup", IsEnabled = true });
+
+        Assert.Equal(Severity.High, Assert.Single(Run(new AgentStoppedRule(), With(jobs)).Findings).Severity);
+
+        var express = Context("16.0.4295.3", edition: EngineEdition.Express);
+        express.Data.Set(jobs);
+        Assert.Equal(RuleOutcome.Passed, Run(new AgentStoppedRule(), express).Outcome);
+    }
+}