diff --git a/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs b/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs index 8afb51e..2742f43 100644 --- a/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs +++ b/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs @@ -23,6 +23,10 @@ public sealed partial class InstanceMonitor( private readonly CollectedData _data = new(); private readonly Dictionary _previous = []; private readonly Dictionary _nextDue = []; + private DateTimeOffset _nextBaseline = DateTimeOffset.MinValue; + + /// Metrics a baseline is kept for. Must match what the repository stores (see ). + public static readonly string[] BaselineMetrics = [Metrics.CpuSqlServer, Metrics.BatchRequestsPerSecond, Metrics.PageLifeExpectancy]; public string Name => instance.Name; @@ -75,6 +79,7 @@ public sealed partial class InstanceMonitor( await CollectAsync(collector, opened, info, capabilities, instanceId, now, cancellationToken).ConfigureAwait(false); } + await RefreshBaselineAsync(instanceId, now, cancellationToken).ConfigureAwait(false); var results = ruleEngine.Evaluate(new RuleContext(info, capabilities, now, _data)); await repository.SaveRuleResultsAsync(instanceId, results, now, cancellationToken).ConfigureAwait(false); var findings = 0; @@ -120,6 +125,22 @@ public sealed partial class InstanceMonitor( } } + /// + /// The baseline for the current hour of the week, from the repository's hourly history. Refreshed at each new hour. + /// + private async Task RefreshBaselineAsync(int instanceId, DateTimeOffset now, CancellationToken cancellationToken) + { + if (now < _nextBaseline) + { + return; + } + + var history = await repository.GetHourlyHistoryAsync(instanceId, BaselineMetrics, now, cancellationToken: cancellationToken).ConfigureAwait(false); + _data.Set(Core.Data.BaselineCalculator.Calculate(history, now)); + var hour = new DateTimeOffset(now.UtcDateTime.Year, now.UtcDateTime.Month, now.UtcDateTime.Day, now.UtcDateTime.Hour, 0, 0, TimeSpan.Zero); + _nextBaseline = hour.AddHours(1); + } + /// /// State a collector would otherwise lose when the agent restarts. Deadlocks continue after the newest one /// already stored; the others simply start over (cumulative collectors need one extra sample). diff --git a/src/SQLInsight.Core/Data/Baseline.cs b/src/SQLInsight.Core/Data/Baseline.cs new file mode 100644 index 0000000..0ae9a3d --- /dev/null +++ b/src/SQLInsight.Core/Data/Baseline.cs @@ -0,0 +1,53 @@ +namespace SQLInsight.Core.Data; + +/// +/// What is normal for a metric in the current hour of the week, learned from earlier weeks. +/// +public sealed record BaselineSnapshot +{ + public required DateTimeOffset CapturedAt { get; init; } + + /// Hour of the week (0 = Monday 00:00–01:00 UTC). + public required int HourOfWeek { get; init; } + + public required IReadOnlyDictionary Metrics { get; init; } +} + +/// How many earlier weeks the baseline is based on. +public sealed record MetricBaseline(double Mean, double StandardDeviation, int Samples); + +public sealed record HourlyValue(string Metric, DateTime Hour, double Value); + +/// +/// Computes the baseline for one hour of the week from hourly averages. The same hour in earlier weeks is used, plus +/// the hour before and after, so three weeks of history already give nine points. +/// +public static class BaselineCalculator +{ + public const int MinSamples = 6; + + public static int HourOfWeek(DateTime utc) => ((int)utc.DayOfWeek + 6) % 7 * 24 + utc.Hour; + + public static BaselineSnapshot Calculate(IEnumerable history, DateTimeOffset now) + { + ArgumentNullException.ThrowIfNull(history); + var slot = HourOfWeek(now.UtcDateTime); + var neighbours = new HashSet { slot, (slot + 167) % 168, (slot + 1) % 168 }; + var currentHour = new DateTime(now.UtcDateTime.Year, now.UtcDateTime.Month, now.UtcDateTime.Day, now.UtcDateTime.Hour, 0, 0, DateTimeKind.Utc); + + var metrics = history + .Where(h => h.Hour < currentHour.AddHours(-2) && neighbours.Contains(HourOfWeek(h.Hour))) + .GroupBy(h => h.Metric) + .ToDictionary( + g => g.Key, + g => + { + var values = g.Select(h => h.Value).ToList(); + var mean = values.Average(); + var variance = values.Count > 1 ? values.Sum(v => (v - mean) * (v - mean)) / (values.Count - 1) : 0; + return new MetricBaseline(mean, Math.Sqrt(variance), values.Count); + }); + + return new BaselineSnapshot { CapturedAt = now, HourOfWeek = slot, Metrics = metrics }; + } +} diff --git a/src/SQLInsight.Repository/InsightRepository.Read.cs b/src/SQLInsight.Repository/InsightRepository.Read.cs index 3a3623c..09fe712 100644 --- a/src/SQLInsight.Repository/InsightRepository.Read.cs +++ b/src/SQLInsight.Repository/InsightRepository.Read.cs @@ -256,3 +256,17 @@ public sealed partial class InsightRepository public DateTime? ResolvedAt { get; init; } } } + +public sealed partial class InsightRepository +{ + /// Hourly averages for the given metrics over the last days, for baselines. + public async Task> GetHourlyHistoryAsync(int instanceId, IReadOnlyCollection metrics, DateTimeOffset now, int days = 28, CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false); + var rows = await connection.QueryAsync<(string Metric, DateTime Hour, double Value)>(""" + SELECT Metric, [Hour], AvgValue FROM insight.MetricHourly + WHERE InstanceId = @instanceId AND [Hour] >= @from AND Metric IN (SELECT value FROM OPENJSON(@metrics)); + """, new { instanceId, from = now.UtcDateTime.AddDays(-days), metrics = JsonSerializer.Serialize(metrics, Json) }).ConfigureAwait(false); + return rows.Select(r => new HourlyValue(r.Metric, DateTime.SpecifyKind(r.Hour, DateTimeKind.Utc), r.Value)).ToList(); + } +} diff --git a/src/SQLInsight.Rules/BuiltInRules.cs b/src/SQLInsight.Rules/BuiltInRules.cs index 2fde34d..f331fb6 100644 --- a/src/SQLInsight.Rules/BuiltInRules.cs +++ b/src/SQLInsight.Rules/BuiltInRules.cs @@ -47,6 +47,7 @@ public static class BuiltInRules new LogFileWriteLatencyRule(), new CpuRule(), new MemoryGrantsPendingRule(), + new AnomalyRule(), new PlanRegressionRule(), new FailedForcedPlanRule(), new ParameterSensitivityRule(), diff --git a/src/SQLInsight.Rules/Performance/AnomalyRule.cs b/src/SQLInsight.Rules/Performance/AnomalyRule.cs new file mode 100644 index 0000000..d7babd5 --- /dev/null +++ b/src/SQLInsight.Rules/Performance/AnomalyRule.cs @@ -0,0 +1,95 @@ +using SQLInsight.Core.Collection; +using SQLInsight.Core.Data; +using SQLInsight.Core.Findings; +using SQLInsight.Rules.Engine; + +namespace SQLInsight.Rules.Performance; + +/// +/// Compares the current load with what is normal for this server at this hour of the week. +/// Fixed thresholds miss that 80 % CPU is normal at month-end but alarming on a Sunday night; a baseline does not. +/// +public sealed class AnomalyRule : Rule +{ + public const string CpuMetric = "cpu.sqlserver"; + public const string BatchMetric = "workload.batch_requests"; + public const string PleMetric = "memory.ple"; + + public override string Id => "SQLI-ANOM-001"; + public override string Title => "Usædvanlig belastning i forhold til det normale"; + + public override RuleEvaluation Evaluate(RuleContext context) + { + ArgumentNullException.ThrowIfNull(context); + var baseline = context.Data.Get(); + var usable = baseline.Metrics.Where(m => m.Value.Samples >= BaselineCalculator.MinSamples).ToDictionary(m => m.Key, m => m.Value); + if (usable.Count == 0) + { + throw new DataUnavailableException("Baseline kræver data fra mindst to-tre uger for denne time på ugen; der er endnu ikke nok historik."); + } + + var findings = new List(); + var slot = $"{DayName(baseline.HourOfWeek / 24)} kl. {baseline.HourOfWeek % 24:00}–{(baseline.HourOfWeek % 24 + 1) % 24:00} UTC"; + + if (usable.TryGetValue(CpuMetric, out var cpuBase) && context.Data.TryGet(out var cpu) && cpu.Samples.Count >= 5) + { + var current = cpu.Samples.TakeLast(15).Average(s => s.SqlServerPercent); + if (current >= 50 && current >= cpuBase.Mean + Math.Max(3 * cpuBase.StandardDeviation, 20)) + { + findings.Add(Anomaly("cpu", Severity.Medium, $"SQL Server bruger {Format.Number(current)} % CPU; normalt {Format.Number(cpuBase.Mean)} % på dette tidspunkt", + "Belastningen er markant højere end den plejer at være på netop dette tidspunkt af ugen. Det tyder på en ændring: " + + "en ny eller ændret forespørgsel, en planregression, et job der kører på et nyt tidspunkt, eller mere trafik.", + "Se Query Store (Top Resource Consuming Queries) for den seneste time sammenlignet med samme tid sidste uge, " + + "og tjek planregressioner (SQLI-QS-001) og nye deployments.", slot, current, cpuBase, "%")); + } + } + + if (usable.TryGetValue(BatchMetric, out var batchBase) + && context.Data.TryGet>(out var counters) + && counters.Delta?.BatchRequestsPerSecond is { } batches) + { + if (batches >= 100 && batches >= batchBase.Mean + Math.Max(3 * batchBase.StandardDeviation, batchBase.Mean)) + { + findings.Add(Anomaly("batch-requests", Severity.Low, $"{Format.Number(batches)} batch requests/s; normalt {Format.Number(batchBase.Mean)} på dette tidspunkt", + "Serveren modtager langt flere forespørgsler end normalt på dette tidspunkt. Det kan være forventet (kampagne, månedsafslutning) " + + "eller en fejl, fx en applikation i en genforsøgs-løkke eller en cache der ikke virker.", + "Find de programmer og forespørgsler, der står for stigningen (sys.dm_exec_sessions efter program_name, Query Store efter antal udførelser).", + slot, batches, batchBase, "/s")); + } + else if (batchBase.Mean >= 50 && batches <= batchBase.Mean * 0.3) + { + findings.Add(Anomaly("batch-requests", Severity.Medium, $"Kun {Format.Number(batches)} batch requests/s; normalt {Format.Number(batchBase.Mean)} på dette tidspunkt", + "Serveren modtager langt færre forespørgsler end normalt på dette tidspunkt. Det tyder på, at en applikation er nede, " + + "ikke kan forbinde, eller at trafikken er flyttet til en anden server.", + "Tjek applikationerne, der bruger serveren, og om de kan forbinde (login-fejl i error log, netværk, DNS, failover).", + slot, batches, batchBase, "/s")); + } + } + + if (usable.TryGetValue(PleMetric, out var pleBase) + && context.Data.TryGet>(out var pc) + && pc.Snapshot.PageLifeExpectancySeconds is { } ple && pleBase.Mean >= 300 && ple <= pleBase.Mean * 0.3) + { + findings.Add(Anomaly("ple", Severity.Medium, $"Page life expectancy er {Format.Number(ple)} s; normalt {Format.Number(pleBase.Mean)} s på dette tidspunkt", + "Sider bliver smidt ud af buffer pool langt hurtigere end normalt. Noget læser meget mere data end det plejer, typisk en " + + "forespørgsel, der er begyndt at scanne en stor tabel, eller et vedligeholdelsesjob.", + "Find forespørgslerne med flest læsninger den seneste time (Query Store eller sys.dm_exec_query_stats efter total_logical_reads).", + slot, ple, pleBase, " s")); + } + + return Result(findings); + } + + private Finding Anomaly(string what, Severity severity, string title, string explanation, string recommendation, string slot, double current, MetricBaseline baseline, string unit) => + CreateFinding(severity, title, explanation, recommendation, + [ + new Evidence("Nu", $"{Format.Number(current, 1)}{unit}"), + new Evidence("Normalt", $"{Format.Number(baseline.Mean, 1)}{unit} ± {Format.Number(baseline.StandardDeviation, 1)} ({slot}, {baseline.Samples} målepunkter)"), + ], + subject: $"anomaly: {what}"); + + private static string DayName(int day) => day switch + { + 0 => "mandag", 1 => "tirsdag", 2 => "onsdag", 3 => "torsdag", 4 => "fredag", 5 => "lørdag", _ => "søndag", + }; +} diff --git a/tests/SQLInsight.Core.Tests/BaselineTests.cs b/tests/SQLInsight.Core.Tests/BaselineTests.cs new file mode 100644 index 0000000..2077574 --- /dev/null +++ b/tests/SQLInsight.Core.Tests/BaselineTests.cs @@ -0,0 +1,51 @@ +using SQLInsight.Core.Data; + +namespace SQLInsight.Core.Tests; + +public class BaselineTests +{ + [Theory] + [InlineData("2026-09-28T00:30:00Z", 0)] // Monday + [InlineData("2026-09-29T13:10:00Z", 37)] // Tuesday 13 + [InlineData("2026-10-04T23:59:00Z", 167)] // Sunday 23 + public void Hour_of_week_starts_monday(string utc, int expected) + { + Assert.Equal(expected, BaselineCalculator.HourOfWeek(DateTime.Parse(utc, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal))); + } + + [Fact] + public void Uses_the_same_hour_and_its_neighbours_in_earlier_weeks() + { + var now = new DateTimeOffset(2026, 9, 29, 13, 20, 0, TimeSpan.Zero); // Tuesday 13 + var history = new List(); + for (var week = 1; week <= 3; week++) + { + var sameHour = new DateTime(2026, 9, 29, 13, 0, 0, DateTimeKind.Utc).AddDays(-7 * week); + history.Add(new HourlyValue("cpu", sameHour.AddHours(-1), 20)); + history.Add(new HourlyValue("cpu", sameHour, 30)); + history.Add(new HourlyValue("cpu", sameHour.AddHours(1), 40)); + history.Add(new HourlyValue("cpu", sameHour.AddHours(5), 99)); // other hour: ignored + } + + history.Add(new HourlyValue("cpu", new DateTime(2026, 9, 29, 12, 0, 0, DateTimeKind.Utc), 99)); // today, too recent + + var baseline = BaselineCalculator.Calculate(history, now); + + var cpu = baseline.Metrics["cpu"]; + Assert.Equal(37, baseline.HourOfWeek); + Assert.Equal(9, cpu.Samples); + Assert.Equal(30, cpu.Mean, 6); + Assert.Equal(Math.Sqrt(75), cpu.StandardDeviation, 6); // (3×100 + 3×0 + 3×100) / 8 + } + + [Fact] + public void Neighbour_hours_wrap_around_the_week() + { + var now = new DateTimeOffset(2026, 9, 28, 0, 30, 0, TimeSpan.Zero); // Monday 00 + var sundayLate = new DateTime(2026, 9, 20, 23, 0, 0, DateTimeKind.Utc); + + var baseline = BaselineCalculator.Calculate([new HourlyValue("cpu", sundayLate, 10)], now); + + Assert.Equal(1, baseline.Metrics["cpu"].Samples); + } +} diff --git a/tests/SQLInsight.IntegrationTests/RepositoryTests.cs b/tests/SQLInsight.IntegrationTests/RepositoryTests.cs index dd59ffb..1a84392 100644 --- a/tests/SQLInsight.IntegrationTests/RepositoryTests.cs +++ b/tests/SQLInsight.IntegrationTests/RepositoryTests.cs @@ -175,6 +175,29 @@ public class RepositoryTests(SqlServerFixture sql) : IAsyncLifetime Assert.Equal(3000, top[0].WaitTimeMs); } + [Fact] + public async Task Hourly_history_feeds_the_baseline_and_metric_names_match_the_anomaly_rule() + { + Assert.Equal(SQLInsight.Rules.Performance.AnomalyRule.CpuMetric, Metrics.CpuSqlServer); + Assert.Equal(SQLInsight.Rules.Performance.AnomalyRule.BatchMetric, Metrics.BatchRequestsPerSecond); + Assert.Equal(SQLInsight.Rules.Performance.AnomalyRule.PleMetric, Metrics.PageLifeExpectancy); + + var id = await _repository.UpsertInstanceAsync("sql09", Info, "SqlClient", false, Now); + var now = DateTimeOffset.UtcNow; + await ExecuteAsync($""" + INSERT insight.MetricHourly (InstanceId, Metric, [Hour], AvgValue, MaxValue, Samples) + VALUES ({id}, 'cpu.sqlserver', DATEADD(day, -7, SYSUTCDATETIME()), 30, 40, 60), + ({id}, 'cpu.sqlserver', DATEADD(day, -40, SYSUTCDATETIME()), 99, 99, 60), + ({id}, 'connections.user', DATEADD(day, -7, SYSUTCDATETIME()), 5, 5, 60); + """); + + var history = await _repository.GetHourlyHistoryAsync(id, [Metrics.CpuSqlServer], now); + + var point = Assert.Single(history); + Assert.Equal(30, point.Value); + Assert.Equal(DateTimeKind.Utc, point.Hour.Kind); + } + [Fact] public async Task Maintenance_deletes_data_past_retention() { diff --git a/tests/SQLInsight.Rules.Tests/AnomalyRuleTests.cs b/tests/SQLInsight.Rules.Tests/AnomalyRuleTests.cs new file mode 100644 index 0000000..7dbf841 --- /dev/null +++ b/tests/SQLInsight.Rules.Tests/AnomalyRuleTests.cs @@ -0,0 +1,70 @@ +using SQLInsight.Core.Data; +using SQLInsight.Core.Findings; +using SQLInsight.Rules.Engine; +using SQLInsight.Rules.Performance; +using static SQLInsight.Rules.Tests.TestInstances; + +namespace SQLInsight.Rules.Tests; + +public class AnomalyRuleTests +{ + private static RuleResult Run(RuleContext context) => Assert.Single(new RuleEngine([new AnomalyRule()]).Evaluate(context)); + + private static BaselineSnapshot Baseline(params (string Metric, double Mean, double Stdev, int Samples)[] metrics) => new() + { + CapturedAt = Now, + HourOfWeek = 37, + Metrics = metrics.ToDictionary(m => m.Metric, m => new MetricBaseline(m.Mean, m.Stdev, m.Samples)), + }; + + private static CpuUtilization Cpu(int percent) => new() + { + CapturedAt = Now, + Samples = Enumerable.Range(0, 15).Select(i => new CpuSample(Now.AddMinutes(i - 15), percent, 100 - percent)).ToList(), + }; + + private static CumulativeSample Counters(double batches, long ple) => new( + new PerformanceCounterSnapshot { CapturedAt = Now, SqlServerStartTime = Now.UtcDateTime.AddDays(-1), PageLifeExpectancySeconds = ple }, + new PerformanceCounterRates { From = Now.AddMinutes(-5), To = Now, BatchRequestsPerSecond = batches }); + + [Fact] + public void Too_little_history_is_not_measurable() + { + var result = Run(With(Baseline((AnomalyRule.CpuMetric, 20, 5, 3)), Cpu(90))); + + Assert.Equal(RuleOutcome.NotMeasurable, result.Outcome); + Assert.Contains("uger", result.Reason, StringComparison.Ordinal); + } + + [Fact] + public void Cpu_far_above_normal_for_this_hour() + { + var finding = Assert.Single(Run(With(Baseline((AnomalyRule.CpuMetric, 20, 5, 9)), Cpu(75))).Findings); + + Assert.Equal("anomaly: cpu", finding.Subject); + Assert.Contains("tirsdag kl. 13–14 UTC", finding.Evidence[1].Value, StringComparison.Ordinal); + } + + [Fact] + public void High_cpu_that_is_normal_for_this_hour_passes() + { + Assert.Equal(RuleOutcome.Passed, Run(With(Baseline((AnomalyRule.CpuMetric, 80, 8, 9)), Cpu(88))).Outcome); + } + + [Fact] + public void Traffic_drop_suggests_an_application_is_down() + { + var finding = Assert.Single(Run(With(Baseline((AnomalyRule.BatchMetric, 800, 100, 9)), Counters(batches: 40, ple: 5000))).Findings); + + Assert.Equal(Severity.Medium, finding.Severity); + Assert.Contains("færre", finding.Explanation, StringComparison.Ordinal); + } + + [Fact] + public void Ple_collapse() + { + var finding = Assert.Single(Run(With(Baseline((AnomalyRule.PleMetric, 4000, 500, 9)), Counters(batches: 0, ple: 300))).Findings); + + Assert.Equal("anomaly: ple", finding.Subject); + } +}