From 15907401a9641e270998a15d7b59c79637ccf600 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 07:53:49 +0000 Subject: [PATCH] Fase 2: Always On availability groups and SQL Agent jobs - Availability group collector (replica and database replica states, queues, commit times) and rules: SQLI-AG-001 unhealthy/disconnected replica or suspended data movement (scripts HADR RESUME), SQLI-AG-002 secondary behind (estimated data loss and redo time) - SQL Agent collector using only granted msdb tables (no EXECUTE on msdb functions): SQLI-JOB-001 failed job with the failing step's message (backup/CHECKDB jobs are High), SQLI-JOB-002 SQL Agent not running - Integration tests create a real availability group (CLUSTER_TYPE NONE) and a failing job in the test container (HADR and Agent enabled) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01E3EExwndy8Z22EK2Qpt6BK --- .../BuiltInCollectors.cs | 2 + .../Instance/AgentJobCollector.cs | 57 ++++++ .../Instance/AvailabilityGroupCollector.cs | 66 +++++++ src/SQLInsight.Core/Data/HighAvailability.cs | 85 +++++++++ .../Availability/AvailabilityRules.cs | 177 ++++++++++++++++++ src/SQLInsight.Rules/BuiltInRules.cs | 7 + .../SqlSelectionTests.cs | 11 ++ .../HighAvailabilityTests.cs | 82 ++++++++ .../SqlServerFixture.cs | 6 +- .../AvailabilityRuleTests.cs | 117 ++++++++++++ 10 files changed, 609 insertions(+), 1 deletion(-) create mode 100644 src/SQLInsight.Collectors/Instance/AgentJobCollector.cs create mode 100644 src/SQLInsight.Collectors/Instance/AvailabilityGroupCollector.cs create mode 100644 src/SQLInsight.Core/Data/HighAvailability.cs create mode 100644 src/SQLInsight.Rules/Availability/AvailabilityRules.cs create mode 100644 tests/SQLInsight.IntegrationTests/HighAvailabilityTests.cs create mode 100644 tests/SQLInsight.Rules.Tests/AvailabilityRuleTests.cs diff --git a/src/SQLInsight.Collectors/BuiltInCollectors.cs b/src/SQLInsight.Collectors/BuiltInCollectors.cs index 0ee060e..c2df45f 100644 --- a/src/SQLInsight.Collectors/BuiltInCollectors.cs +++ b/src/SQLInsight.Collectors/BuiltInCollectors.cs @@ -24,5 +24,7 @@ public static class BuiltInCollectors new QueryStoreCollector(), new PlanAnalysisCollector(), new IndexCollector(), + new AvailabilityGroupCollector(), + new AgentJobCollector(), ]; } diff --git a/src/SQLInsight.Collectors/Instance/AgentJobCollector.cs b/src/SQLInsight.Collectors/Instance/AgentJobCollector.cs new file mode 100644 index 0000000..606e51f --- /dev/null +++ b/src/SQLInsight.Collectors/Instance/AgentJobCollector.cs @@ -0,0 +1,57 @@ +using System.Globalization; +using SQLInsight.Core.Data; +using SQLInsight.Core.Instances; + +namespace SQLInsight.Collectors.Instance; + +/// +/// SQL Agent jobs from msdb. Uses only the tables granted by the least-privilege script (no msdb functions, which +/// would need EXECUTE); run_date/run_time integers are converted here. +/// +public sealed class AgentJobCollector : Collector +{ + internal const string RunningSql = "SELECT COUNT(*) FROM sys.dm_exec_sessions WHERE program_name LIKE N'SQLAgent - %'"; + + internal const string JobsSql = """ + DECLARE @since int = CONVERT(int, CONVERT(char(8), DATEADD(day, -1, GETDATE()), 112)); + SELECT j.name, j.enabled, last_run.run_status, last_run.run_date, last_run.run_time, + failed_step.step_name, LEFT(failed_step.message, 2000), failures.n + FROM msdb.dbo.sysjobs j + OUTER APPLY (SELECT TOP (1) h.run_status, h.run_date, h.run_time FROM msdb.dbo.sysjobhistory h + WHERE h.job_id = j.job_id AND h.step_id = 0 ORDER BY h.instance_id DESC) last_run + OUTER APPLY (SELECT TOP (1) h.step_name, h.message FROM msdb.dbo.sysjobhistory h + WHERE h.job_id = j.job_id AND h.step_id > 0 AND h.run_status = 0 ORDER BY h.instance_id DESC) failed_step + OUTER APPLY (SELECT COUNT(*) AS n FROM msdb.dbo.sysjobhistory h + WHERE h.job_id = j.job_id AND h.step_id = 0 AND h.run_status = 0 AND h.run_date >= @since) failures + """; + + public override string Name => "agent-jobs"; + public override TimeSpan DefaultInterval => TimeSpan.FromMinutes(5); + public override IReadOnlyCollection RequiredCapabilities => [Capability.DynamicManagementViews]; + + protected override async Task CollectAsync(CollectorContext context, AgentJobSnapshot? previous) + { + var ct = context.CancellationToken; + var sessions = await Db.QueryAsync(context.Connection, RunningSql, r => r.Int(0), ct).ConfigureAwait(false); + var jobs = await Db.QueryAsync(context.Connection, JobsSql, r => new AgentJob + { + Name = r.String(0), + IsEnabled = r.Bool(1), + LastRunStatus = r.NullableInt(2), + LastRunAt = r.IsNull(3) ? null : ToDateTime(r.Int(3), r.Int(4)), + LastFailedStepName = r.NullableString(5), + LastFailedStepMessage = r.NullableString(6), + FailuresLastDay = r.NullableInt(7) ?? 0, + }, ct).ConfigureAwait(false); + + return new AgentJobSnapshot { CapturedAt = context.Now, IsAgentRunning = sessions[0] > 0, Jobs = jobs }; + } + + /// msdb stores 2026-09-29 13:05:07 as run_date 20260929 and run_time 130507 (server local time). + internal static DateTime? ToDateTime(int runDate, int runTime) => + DateTime.TryParseExact( + runDate.ToString("00000000", CultureInfo.InvariantCulture) + runTime.ToString("000000", CultureInfo.InvariantCulture), + "yyyyMMddHHmmss", CultureInfo.InvariantCulture, DateTimeStyles.None, out var value) + ? value + : null; +} diff --git a/src/SQLInsight.Collectors/Instance/AvailabilityGroupCollector.cs b/src/SQLInsight.Collectors/Instance/AvailabilityGroupCollector.cs new file mode 100644 index 0000000..8e05dfa --- /dev/null +++ b/src/SQLInsight.Collectors/Instance/AvailabilityGroupCollector.cs @@ -0,0 +1,66 @@ +using SQLInsight.Core.Data; +using SQLInsight.Core.Instances; + +namespace SQLInsight.Collectors.Instance; + +public sealed class AvailabilityGroupCollector : Collector +{ + internal const string ReplicasSql = """ + SELECT ag.name, ar.replica_server_name, ars.is_local, ars.role_desc, ars.connected_state_desc, ars.synchronization_health_desc, + ar.availability_mode_desc, ar.failover_mode_desc + FROM sys.availability_groups ag + JOIN sys.availability_replicas ar ON ar.group_id = ag.group_id + LEFT JOIN sys.dm_hadr_availability_replica_states ars ON ars.replica_id = ar.replica_id + """; + + internal const string DatabasesSql = """ + SELECT ag.name, DB_NAME(drs.database_id), ar.replica_server_name, drs.is_local, + CASE WHEN ars.role = 1 THEN 1 ELSE 0 END, + drs.synchronization_state_desc, drs.synchronization_health_desc, drs.is_suspended, drs.suspend_reason_desc, + drs.log_send_queue_size, drs.log_send_rate, drs.redo_queue_size, drs.redo_rate, drs.last_commit_time + FROM sys.dm_hadr_database_replica_states drs + JOIN sys.availability_replicas ar ON ar.replica_id = drs.replica_id + JOIN sys.availability_groups ag ON ag.group_id = drs.group_id + LEFT JOIN sys.dm_hadr_availability_replica_states ars ON ars.replica_id = drs.replica_id + """; + + public override string Name => "availability-groups"; + public override TimeSpan DefaultInterval => TimeSpan.FromMinutes(1); + public override IReadOnlyCollection RequiredCapabilities => [Capability.HadrEnabled]; + + protected override async Task CollectAsync(CollectorContext context, AvailabilityGroupSnapshot? previous) + { + var ct = context.CancellationToken; + var replicas = await Db.QueryAsync(context.Connection, ReplicasSql, r => new ReplicaState + { + Group = r.String(0), + Replica = r.String(1), + IsLocal = r.Bool(2), + Role = r.NullableString(3), + ConnectedState = r.NullableString(4), + SynchronizationHealth = r.NullableString(5), + AvailabilityMode = r.NullableString(6), + FailoverMode = r.NullableString(7), + }, ct).ConfigureAwait(false); + + var databases = await Db.QueryAsync(context.Connection, DatabasesSql, r => new DatabaseReplicaState + { + Group = r.String(0), + Database = r.NullableString(1) ?? "?", + Replica = r.String(2), + IsLocal = r.Bool(3), + IsPrimaryReplica = r.Bool(4), + SynchronizationState = r.NullableString(5), + SynchronizationHealth = r.NullableString(6), + IsSuspended = r.Bool(7), + SuspendReason = r.NullableString(8), + LogSendQueueKb = r.NullableLong(9), + LogSendRateKbPerSecond = r.NullableLong(10), + RedoQueueKb = r.NullableLong(11), + RedoRateKbPerSecond = r.NullableLong(12), + LastCommitTime = r.NullableDateTime(13), + }, ct).ConfigureAwait(false); + + return new AvailabilityGroupSnapshot { CapturedAt = context.Now, Replicas = replicas, Databases = databases }; + } +} diff --git a/src/SQLInsight.Core/Data/HighAvailability.cs b/src/SQLInsight.Core/Data/HighAvailability.cs new file mode 100644 index 0000000..3568429 --- /dev/null +++ b/src/SQLInsight.Core/Data/HighAvailability.cs @@ -0,0 +1,85 @@ +namespace SQLInsight.Core.Data; + +/// +/// Always On availability group state as seen from this instance. A primary sees all replicas; a secondary only itself. +/// +public sealed record AvailabilityGroupSnapshot +{ + public required DateTimeOffset CapturedAt { get; init; } + public required IReadOnlyList Replicas { get; init; } + public required IReadOnlyList Databases { get; init; } +} + +public sealed record ReplicaState +{ + public required string Group { get; init; } + public required string Replica { get; init; } + public bool IsLocal { get; init; } + + /// PRIMARY, SECONDARY or RESOLVING; null when this instance cannot see the replica's state. + public string? Role { get; init; } + + public string? ConnectedState { get; init; } + + /// HEALTHY, PARTIALLY_HEALTHY or NOT_HEALTHY. + public string? SynchronizationHealth { get; init; } + + public string? AvailabilityMode { get; init; } + public string? FailoverMode { get; init; } +} + +public sealed record DatabaseReplicaState +{ + public required string Group { get; init; } + public required string Database { get; init; } + public required string Replica { get; init; } + public bool IsLocal { get; init; } + public bool IsPrimaryReplica { get; init; } + public string? SynchronizationState { get; init; } + public string? SynchronizationHealth { get; init; } + public bool IsSuspended { get; init; } + public string? SuspendReason { get; init; } + + /// Log not yet sent to this secondary, in KB. + public long? LogSendQueueKb { get; init; } + + public long? LogSendRateKbPerSecond { get; init; } + + /// Log received but not yet redone on this secondary, in KB. + public long? RedoQueueKb { get; init; } + + public long? RedoRateKbPerSecond { get; init; } + public DateTime? LastCommitTime { get; init; } +} + +/// +/// SQL Server Agent: whether it runs, and the latest outcome of each job. +/// +public sealed record AgentJobSnapshot +{ + public required DateTimeOffset CapturedAt { get; init; } + + /// False when no SQL Agent session exists (the service is stopped). + public required bool IsAgentRunning { get; init; } + + public required IReadOnlyList Jobs { get; init; } +} + +public sealed record AgentJob +{ + public required string Name { get; init; } + public bool IsEnabled { get; init; } + + /// 0 failed, 1 succeeded, 2 retry, 3 canceled, 4 in progress; null if never run. + public int? LastRunStatus { get; init; } + + public DateTime? LastRunAt { get; init; } + + /// The message of the most recent failed step, which says why (the job-level message does not). + public string? LastFailedStepMessage { get; init; } + + public string? LastFailedStepName { get; init; } + public int FailuresLastDay { get; init; } + + public bool LastRunFailed => LastRunStatus == 0; +} diff --git a/src/SQLInsight.Rules/Availability/AvailabilityRules.cs b/src/SQLInsight.Rules/Availability/AvailabilityRules.cs new file mode 100644 index 0000000..9f33735 --- /dev/null +++ b/src/SQLInsight.Rules/Availability/AvailabilityRules.cs @@ -0,0 +1,177 @@ +using SQLInsight.Core.Data; +using SQLInsight.Core.Findings; +using SQLInsight.Core.Instances; +using SQLInsight.Rules.Engine; + +namespace SQLInsight.Rules.Availability; + +/// +/// Availability replicas or databases that are not healthy: disconnected, not synchronizing or suspended. +/// +public sealed class AvailabilityGroupHealthRule : Rule +{ + public override string Id => "SQLI-AG-001"; + public override string Title => "Always On availability group er ikke sund"; + public override IReadOnlyCollection RequiredCapabilities => [Capability.HadrEnabled]; + + private static readonly Reference Docs = Engine.Docs.Learn("Monitor availability groups", "database-engine/availability-groups/windows/monitoring-of-availability-groups-sql-server"); + + public override RuleEvaluation Evaluate(RuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var snapshot = context.Data.Get(); + var findings = new List(); + + foreach (var r in snapshot.Replicas.Where(r => r.ConnectedState == "DISCONNECTED" || r.SynchronizationHealth is "NOT_HEALTHY" or "PARTIALLY_HEALTHY")) + { + findings.Add(CreateFinding( + r.ConnectedState == "DISCONNECTED" || r.SynchronizationHealth == "NOT_HEALTHY" ? Severity.High : Severity.Medium, + $"{r.Group}: replika {r.Replica} er {(r.ConnectedState == "DISCONNECTED" ? "afbrudt" : r.SynchronizationHealth)}", + "Replikaen synkroniserer ikke normalt. Så længe det står på, er en failover til den enten umulig eller vil miste data, " + + "og på den primære kan transaktionsloggen ikke genbruges, fordi loggen skal gemmes til replikaen.", + "Tjek netværk og endpoint mellem replikaerne, SQL Server error log og AlwaysOn_health-sessionen på begge noder. " + + "Tjek også, om databaserne er suspenderet (se de enkelte databasefund).", + [new Evidence("Rolle", r.Role ?? "ukendt"), new Evidence("Forbindelse", r.ConnectedState ?? "ukendt"), + new Evidence("Sundhed", r.SynchronizationHealth ?? "ukendt"), new Evidence("Tilstand", $"{r.AvailabilityMode}, {r.FailoverMode}")], + subject: $"replica: {r.Group}/{r.Replica}", + references: Docs)); + } + + foreach (var d in snapshot.Databases.Where(d => d.IsSuspended || d.SynchronizationHealth is "NOT_HEALTHY" or "PARTIALLY_HEALTHY")) + { + findings.Add(CreateFinding( + d.IsSuspended || d.SynchronizationHealth == "NOT_HEALTHY" ? Severity.High : Severity.Medium, + d.IsSuspended + ? $"{d.Group}: data movement for {d.Database} på {d.Replica} er suspenderet ({d.SuspendReason})" + : $"{d.Group}: {d.Database} på {d.Replica} er {d.SynchronizationHealth}", + "Databasen synkroniserer ikke til replikaen. Den primære skal gemme al log, til synkroniseringen genoptages, så logfilen vokser, " + + "og en failover vil miste de ændringer, der ikke er nået frem.", + d.IsSuspended + ? "Find årsagen (suspend_reason og error log), ret den, og genoptag: ALTER DATABASE … SET HADR RESUME. " + + "Er den suspenderet længe, kan det være hurtigere at gen-seede databasen." + : "Tjek synkroniseringstilstand, netværk og redo på den sekundære. Se AlwaysOn_health-sessionen.", + [new Evidence("Synkronisering", d.SynchronizationState ?? "ukendt"), new Evidence("Sundhed", d.SynchronizationHealth ?? "ukendt"), + new Evidence("Log send queue", d.LogSendQueueKb is { } q ? $"{Format.Number(q / 1024.0)} MB" : "–")], + subject: $"ag-database: {d.Group}/{d.Replica}/{d.Database}", + fixScript: d.IsSuspended ? $"ALTER DATABASE [{d.Database.Replace("]", "]]", StringComparison.Ordinal)}] SET HADR RESUME;" : null, + references: Docs)); + } + + return Result(findings); + } +} + +/// +/// Secondaries that are far behind: potential data loss (send queue / commit lag) or long failover time (redo queue). +/// +public sealed class AvailabilityGroupLagRule : Rule +{ + public const double MaxDataLossSeconds = 60; + public const double MaxRecoverySeconds = 600; + + public override string Id => "SQLI-AG-002"; + public override string Title => "Sekundær replika er langt bagud"; + public override IReadOnlyCollection RequiredCapabilities => [Capability.HadrEnabled]; + + public override RuleEvaluation Evaluate(RuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var snapshot = context.Data.Get(); + var findings = new List(); + foreach (var secondary in snapshot.Databases.Where(d => !d.IsPrimaryReplica && !d.IsSuspended)) + { + var primary = snapshot.Databases.FirstOrDefault(p => p.IsPrimaryReplica && p.Group == secondary.Group && p.Database == secondary.Database); + double? dataLoss = primary?.LastCommitTime is { } p && secondary.LastCommitTime is { } s ? Math.Max(0, (p - s).TotalSeconds) : null; + double? recovery = secondary.RedoQueueKb is { } redo && secondary.RedoRateKbPerSecond is > 0 ? (double)redo / secondary.RedoRateKbPerSecond.Value : null; + + if ((dataLoss ?? 0) < MaxDataLossSeconds && (recovery ?? 0) < MaxRecoverySeconds) + { + continue; + } + + findings.Add(CreateFinding( + Severity.Medium, + $"{secondary.Group}: {secondary.Database} på {secondary.Replica} er bagud" + + (dataLoss >= MaxDataLossSeconds ? $" (op til {Format.Number(dataLoss!.Value)} s datatab)" : string.Empty) + + (recovery >= MaxRecoverySeconds ? $" (redo ca. {Format.Number(recovery!.Value / 60)} min)" : string.Empty), + "En stor send queue betyder, at ændringer endnu ikke er nået til replikaen; ved en failover nu mistes de. En stor redo queue " + + "betyder, at replikaen har loggen, men mangler at anvende den; en failover tager så tilsvarende længere tid.", + "Tjek netværksbåndbredde og latens mellem replikaerne, og I/O på den sekundære (redo er ofte begrænset af skrivelatens eller " + + "læsetrafik på en readable secondary). Store indexvedligehold og bulk loads giver midlertidige køer.", + [ + new Evidence("Log send queue", secondary.LogSendQueueKb is { } q ? $"{Format.Number(q / 1024.0)} MB" : "–"), + new Evidence("Redo queue", secondary.RedoQueueKb is { } r ? $"{Format.Number(r / 1024.0)} MB" : "–"), + new Evidence("Redo rate", secondary.RedoRateKbPerSecond is { } rr ? $"{Format.Number(rr / 1024.0, 1)} MB/s" : "–"), + ], + subject: $"ag-database: {secondary.Group}/{secondary.Replica}/{secondary.Database}", + references: Docs.Learn("Monitor performance for availability groups", "database-engine/availability-groups/windows/monitor-performance-for-always-on-availability-groups"))); + } + + return Result(findings); + } +} + +/// +/// SQL Agent jobs whose most recent run failed. +/// +public sealed class FailedJobRule : Rule +{ + private static readonly string[] Critical = ["backup", "checkdb", "integrity", "dbcc"]; + + public override string Id => "SQLI-JOB-001"; + public override string Title => "SQL Agent-job fejler"; + public override IReadOnlyCollection RequiredCapabilities => [Capability.DynamicManagementViews]; + + public override RuleEvaluation Evaluate(RuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + return Result(context.Data.Get().Jobs.Where(j => j.IsEnabled && j.LastRunFailed).Select(j => + { + var critical = Critical.Any(c => j.Name.Contains(c, StringComparison.OrdinalIgnoreCase)); + return CreateFinding( + critical ? Severity.High : Severity.Medium, + $"Jobbet '{j.Name}' fejlede senest {Format.Date(j.LastRunAt)}", + critical + ? "Jobbet ser ud til at stå for backup eller integritetskontrol. Så længe det fejler, bliver der ikke taget backup eller kørt CHECKDB." + : "Seneste kørsel af jobbet fejlede. Det, jobbet skulle gøre, er ikke sket.", + "Læs fejlbeskeden fra det fejlende trin (nedenfor eller i jobhistorikken), ret årsagen, og kør jobbet igen. " + + "Sørg for, at fejlende jobs sender en notifikation til en operator.", + [ + new Evidence("Fejlende trin", j.LastFailedStepName ?? "–"), + new Evidence("Fejl", j.LastFailedStepMessage ?? "–"), + new Evidence("Fejl det seneste døgn", j.FailuresLastDay.ToString(System.Globalization.CultureInfo.InvariantCulture)), + ], + subject: $"job: {j.Name}", + references: Docs.Learn("View job history", "ssms/agent/view-the-job-history")); + })); + } +} + +/// +/// The SQL Agent service is not running, so no scheduled job (backups!) runs. +/// +public sealed class AgentStoppedRule : Rule +{ + public override string Id => "SQLI-JOB-002"; + public override string Title => "SQL Server Agent kører ikke"; + public override IReadOnlyCollection RequiredCapabilities => [Capability.DynamicManagementViews]; + + public override RuleEvaluation Evaluate(RuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var snapshot = context.Data.Get(); + if (snapshot.IsAgentRunning || context.Instance.EngineEdition == EngineEdition.Express) + { + return RuleEvaluation.Passed(); + } + + var scheduled = snapshot.Jobs.Count(j => j.IsEnabled); + return RuleEvaluation.Found(CreateFinding( + scheduled > 0 ? Severity.High : Severity.Low, + scheduled > 0 ? $"SQL Server Agent kører ikke; {scheduled} aktive jobs bliver ikke udført" : Title, + "Uden SQL Server Agent kører ingen planlagte jobs: backups, integritetskontrol, indexvedligehold og alarmer.", + "Start tjenesten SQL Server Agent, og sæt den til at starte automatisk. Tjek dens log for, hvorfor den er stoppet.", + [new Evidence("Aktive jobs", scheduled.ToString(System.Globalization.CultureInfo.InvariantCulture))], + references: Docs.Learn("SQL Server Agent", "ssms/agent/sql-server-agent"))); + } +} diff --git a/src/SQLInsight.Rules/BuiltInRules.cs b/src/SQLInsight.Rules/BuiltInRules.cs index 158bffb..2fde34d 100644 --- a/src/SQLInsight.Rules/BuiltInRules.cs +++ b/src/SQLInsight.Rules/BuiltInRules.cs @@ -1,3 +1,4 @@ +using SQLInsight.Rules.Availability; using SQLInsight.Rules.Concurrency; using SQLInsight.Rules.Configuration; using SQLInsight.Rules.Databases; @@ -63,6 +64,12 @@ public static class BuiltInRules new StaleStatisticsRule(), new AutoStatisticsRule(), + // Availability and jobs + new AvailabilityGroupHealthRule(), + new AvailabilityGroupLagRule(), + new FailedJobRule(), + new AgentStoppedRule(), + // Concurrency new BlockingRule(), new DeadlockRule(), diff --git a/tests/SQLInsight.Collectors.Tests/SqlSelectionTests.cs b/tests/SQLInsight.Collectors.Tests/SqlSelectionTests.cs index 1be0af8..82b76e4 100644 --- a/tests/SQLInsight.Collectors.Tests/SqlSelectionTests.cs +++ b/tests/SQLInsight.Collectors.Tests/SqlSelectionTests.cs @@ -66,3 +66,14 @@ public class SqlSelectionTests Assert.Contains("N'/var/opt/mssql/log/system_health_0_1''2.xel', 12345", sql, StringComparison.Ordinal); } } + +public class AgentJobTimeTests +{ + [Theory] + [InlineData(20260929, 130507, "2026-09-29 13:05:07")] + [InlineData(20260101, 5, "2026-01-01 00:00:05")] + public void Converts_msdb_run_date_and_time(int date, int time, string expected) + { + Assert.Equal(DateTime.Parse(expected, System.Globalization.CultureInfo.InvariantCulture), SQLInsight.Collectors.Instance.AgentJobCollector.ToDateTime(date, time)); + } +} diff --git a/tests/SQLInsight.IntegrationTests/HighAvailabilityTests.cs b/tests/SQLInsight.IntegrationTests/HighAvailabilityTests.cs new file mode 100644 index 0000000..0ad9c36 --- /dev/null +++ b/tests/SQLInsight.IntegrationTests/HighAvailabilityTests.cs @@ -0,0 +1,82 @@ +using Microsoft.Data.SqlClient; +using SQLInsight.Collectors.Instance; +using SQLInsight.Core.Data; +using SQLInsight.Core.Instances; + +namespace SQLInsight.IntegrationTests; + +[Collection(SqlServerTestGroup.Name)] +[Trait("Category", "Integration")] +public class HighAvailabilityTests(SqlServerFixture sql) +{ + [Fact] + public async Task Availability_group_state_is_read_with_least_privilege() + { + await sql.ExecuteAsSaAsync(""" + IF DB_ID(N'AgProbe') IS NULL + BEGIN + CREATE DATABASE AgProbe; + END; + """); + await sql.ExecuteAsSaAsync(""" + IF NOT EXISTS (SELECT 1 FROM sys.availability_groups WHERE name = N'ag_probe') + BEGIN + ALTER DATABASE AgProbe SET RECOVERY FULL; + BACKUP DATABASE AgProbe TO DISK = N'/tmp/agprobe.bak' WITH INIT; + IF NOT EXISTS (SELECT 1 FROM sys.symmetric_keys WHERE name = N'##MS_DatabaseMasterKey##') + CREATE MASTER KEY ENCRYPTION BY PASSWORD = N'Mk-Pa55word!'; + IF NOT EXISTS (SELECT 1 FROM sys.certificates WHERE name = N'ag_probe_cert') + CREATE CERTIFICATE ag_probe_cert WITH SUBJECT = N'ag_probe'; + IF NOT EXISTS (SELECT 1 FROM sys.endpoints WHERE name = N'hadr_endpoint') + CREATE ENDPOINT hadr_endpoint STATE = STARTED AS TCP (LISTENER_PORT = 5022) + FOR DATABASE_MIRRORING (ROLE = ALL, AUTHENTICATION = CERTIFICATE ag_probe_cert, ENCRYPTION = REQUIRED ALGORITHM AES); + DECLARE @server sysname = CAST(SERVERPROPERTY('ServerName') AS sysname); + EXEC (N'CREATE AVAILABILITY GROUP ag_probe WITH (CLUSTER_TYPE = NONE) FOR DATABASE AgProbe + REPLICA ON N''' + @server + N''' WITH (ENDPOINT_URL = N''tcp://localhost:5022'', AVAILABILITY_MODE = ASYNCHRONOUS_COMMIT, + FAILOVER_MODE = MANUAL, SEEDING_MODE = AUTOMATIC);'); + END; + """); + + await using var session = await AgentSession.OpenAsync(sql); + Assert.True(session.Capabilities.Has(Capability.HadrEnabled)); + var snapshot = await session.CollectAsync(new AvailabilityGroupCollector()); + + var replica = Assert.Single(snapshot.Replicas, r => r.Group == "ag_probe"); + Assert.True(replica.IsLocal); + Assert.Equal("PRIMARY", replica.Role); + var database = Assert.Single(snapshot.Databases, d => d.Group == "ag_probe"); + Assert.Equal("AgProbe", database.Database); + Assert.True(database.IsPrimaryReplica); + } + + [Fact] + public async Task Failing_agent_job_is_reported_with_the_step_message() + { + await sql.ExecuteAsSaAsync(""" + IF EXISTS (SELECT 1 FROM msdb.dbo.sysjobs WHERE name = N'SQLInsight probe job') + EXEC msdb.dbo.sp_delete_job @job_name = N'SQLInsight probe job'; + EXEC msdb.dbo.sp_add_job @job_name = N'SQLInsight probe job'; + EXEC msdb.dbo.sp_add_jobstep @job_name = N'SQLInsight probe job', @step_name = N'Fail on purpose', + @subsystem = N'TSQL', @command = N'RAISERROR(N''sqlinsight-probe-failure'', 16, 1);'; + EXEC msdb.dbo.sp_add_jobserver @job_name = N'SQLInsight probe job'; + EXEC msdb.dbo.sp_start_job @job_name = N'SQLInsight probe job'; + """); + + await using var session = await AgentSession.OpenAsync(sql); + AgentJob? job = null; + for (var attempt = 0; attempt < 60 && job?.LastRunFailed != true; attempt++) + { + await Task.Delay(1000); + var snapshot = await session.CollectAsync(new AgentJobCollector()); + Assert.True(snapshot.IsAgentRunning); + job = snapshot.Jobs.SingleOrDefault(j => j.Name == "SQLInsight probe job"); + } + + Assert.NotNull(job); + Assert.True(job.LastRunFailed); + Assert.Equal("Fail on purpose", job.LastFailedStepName); + Assert.Contains("sqlinsight-probe-failure", job.LastFailedStepMessage, StringComparison.Ordinal); + Assert.True(job.FailuresLastDay >= 1); + Assert.NotNull(job.LastRunAt); + } +} diff --git a/tests/SQLInsight.IntegrationTests/SqlServerFixture.cs b/tests/SQLInsight.IntegrationTests/SqlServerFixture.cs index eb4f9ab..9e571e0 100644 --- a/tests/SQLInsight.IntegrationTests/SqlServerFixture.cs +++ b/tests/SQLInsight.IntegrationTests/SqlServerFixture.cs @@ -16,7 +16,11 @@ public sealed class SqlServerFixture : IAsyncLifetime public static string Image => Environment.GetEnvironmentVariable("SQLINSIGHT_MSSQL_IMAGE") ?? DefaultImage; - private readonly MsSqlContainer _container = new MsSqlBuilder(Image).Build(); + // HADR and SQL Agent on, so availability group and job collectors run against the real DMVs. + private readonly MsSqlContainer _container = new MsSqlBuilder(Image) + .WithEnvironment("MSSQL_ENABLE_HADR", "1") + .WithEnvironment("MSSQL_AGENT_ENABLED", "true") + .Build(); public string SaConnectionString => _container.GetConnectionString(); diff --git a/tests/SQLInsight.Rules.Tests/AvailabilityRuleTests.cs b/tests/SQLInsight.Rules.Tests/AvailabilityRuleTests.cs new file mode 100644 index 0000000..62a92e6 --- /dev/null +++ b/tests/SQLInsight.Rules.Tests/AvailabilityRuleTests.cs @@ -0,0 +1,117 @@ +using SQLInsight.Core.Data; +using SQLInsight.Core.Findings; +using SQLInsight.Core.Instances; +using SQLInsight.Rules.Availability; +using SQLInsight.Rules.Engine; +using static SQLInsight.Rules.Tests.TestInstances; + +namespace SQLInsight.Rules.Tests; + +public class AvailabilityRuleTests +{ + private static RuleResult Run(IRule rule, RuleContext context) => Assert.Single(new RuleEngine([rule]).Evaluate(context)); + + private static RuleContext Hadr(AvailabilityGroupSnapshot snapshot) + { + var info = new InstanceInfo { ServerName = "n1", Version = SqlServerVersion.Parse("16.0.4295.3"), Edition = "Enterprise", EngineEdition = EngineEdition.Enterprise, IsHadrEnabled = true }; + var context = new RuleContext(info, InstanceCapabilities.From(info), Now); + context.Data.Set(snapshot); + return context; + } + + private static DatabaseReplicaState Db(string replica, bool primary, DateTime? commit = null, long redoKb = 0, long redoRate = 1000, bool suspended = false, string health = "HEALTHY") => new() + { + Group = "ag1", Database = "Sales", Replica = replica, IsPrimaryReplica = primary, LastCommitTime = commit, + RedoQueueKb = redoKb, RedoRateKbPerSecond = redoRate, IsSuspended = suspended, SuspendReason = suspended ? "SUSPEND_FROM_USER" : null, + SynchronizationHealth = health, SynchronizationState = "SYNCHRONIZED", LogSendQueueKb = 0, + }; + + [Fact] + public void Not_measurable_without_hadr() + { + Assert.Equal(RuleOutcome.NotMeasurable, Run(new AvailabilityGroupHealthRule(), Context("16.0.4295.3")).Outcome); + } + + [Fact] + public void Healthy_group_passes() + { + var snapshot = new AvailabilityGroupSnapshot + { + CapturedAt = Now, + Replicas = [new ReplicaState { Group = "ag1", Replica = "n1", Role = "PRIMARY", ConnectedState = "CONNECTED", SynchronizationHealth = "HEALTHY" }], + Databases = [Db("n1", true, Now.UtcDateTime), Db("n2", false, Now.UtcDateTime)], + }; + + Assert.Equal(RuleOutcome.Passed, Run(new AvailabilityGroupHealthRule(), Hadr(snapshot)).Outcome); + Assert.Equal(RuleOutcome.Passed, Run(new AvailabilityGroupLagRule(), Hadr(snapshot)).Outcome); + } + + [Fact] + public void Disconnected_replica_and_suspended_database_are_high() + { + var snapshot = new AvailabilityGroupSnapshot + { + CapturedAt = Now, + Replicas = [new ReplicaState { Group = "ag1", Replica = "n2", Role = "SECONDARY", ConnectedState = "DISCONNECTED", SynchronizationHealth = "NOT_HEALTHY" }], + Databases = [Db("n2", false, suspended: true, health: "NOT_HEALTHY")], + }; + + var findings = Run(new AvailabilityGroupHealthRule(), Hadr(snapshot)).Findings; + + Assert.Equal(2, findings.Count); + Assert.All(findings, f => Assert.Equal(Severity.High, f.Severity)); + Assert.Contains(findings, f => f.FixScript == "ALTER DATABASE [Sales] SET HADR RESUME;"); + } + + [Fact] + public void Lag_reports_data_loss_and_redo_time() + { + var snapshot = new AvailabilityGroupSnapshot + { + CapturedAt = Now, + Replicas = [], + Databases = [Db("n1", true, Now.UtcDateTime), Db("n2", false, Now.UtcDateTime.AddMinutes(-5), redoKb: 2_000_000, redoRate: 1000)], + }; + + var finding = Assert.Single(Run(new AvailabilityGroupLagRule(), Hadr(snapshot)).Findings); + + Assert.Contains("300 s datatab", finding.Title, StringComparison.Ordinal); + Assert.Contains("redo ca. 33 min", finding.Title, StringComparison.Ordinal); + } + + private static AgentJobSnapshot Jobs(bool running, params AgentJob[] jobs) => new() { CapturedAt = Now, IsAgentRunning = running, Jobs = jobs }; + + [Fact] + public void Failed_backup_job_is_high_with_step_message() + { + var job = new AgentJob { Name = "DatabaseBackup - USER_DATABASES - FULL", IsEnabled = true, LastRunStatus = 0, LastFailedStepName = "Backup", LastFailedStepMessage = "Cannot open backup device 'X:\\bak'" }; + + var finding = Assert.Single(Run(new FailedJobRule(), With(Jobs(true, job))).Findings); + + Assert.Equal(Severity.High, finding.Severity); + Assert.Contains(finding.Evidence, e => e.Value.Contains("Cannot open backup device", StringComparison.Ordinal)); + } + + [Fact] + public void Disabled_or_succeeding_jobs_pass() + { + var jobs = Jobs(true, + new AgentJob { Name = "Old", IsEnabled = false, LastRunStatus = 0 }, + new AgentJob { Name = "Fine", IsEnabled = true, LastRunStatus = 1 }, + new AgentJob { Name = "Never run", IsEnabled = true }); + + Assert.Equal(RuleOutcome.Passed, Run(new FailedJobRule(), With(jobs)).Outcome); + } + + [Fact] + public void Stopped_agent_is_high_when_jobs_exist_but_not_on_express() + { + var jobs = Jobs(false, new AgentJob { Name = "Backup", IsEnabled = true }); + + Assert.Equal(Severity.High, Assert.Single(Run(new AgentStoppedRule(), With(jobs)).Findings).Severity); + + var express = Context("16.0.4295.3", edition: EngineEdition.Express); + express.Data.Set(jobs); + Assert.Equal(RuleOutcome.Passed, Run(new AgentStoppedRule(), express).Outcome); + } +}