Fase 2: baselines and anomaly detection

Baselines per hour of the week (with neighbouring hours) from four weeks
of hourly aggregates, computed in C# (independent of DATEFIRST) and
refreshed by the agent each hour. SQLI-ANOM-001 compares current load
with what is normal at this time: CPU far above normal, batch requests
far above or collapsed (application down?), page life expectancy
collapse. Not measurable until enough history exists.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01E3EExwndy8Z22EK2Qpt6BK

src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs

@@ -23,6 +23,10 @@ public sealed partial class InstanceMonitor(
     private readonly CollectedData _data = new();
     private readonly Dictionary<string, object> _previous = [];
     private readonly Dictionary<string, DateTimeOffset> _nextDue = [];
+    private DateTimeOffset _nextBaseline = DateTimeOffset.MinValue;
+
+    /// <summary>Metrics a baseline is kept for. Must match what the repository stores (see <see cref="Metrics"/>).</summary>
+    public static readonly string[] BaselineMetrics = [Metrics.CpuSqlServer, Metrics.BatchRequestsPerSecond, Metrics.PageLifeExpectancy];
 
     public string Name => instance.Name;
 
@@ -75,6 +79,7 @@ public sealed partial class InstanceMonitor(
                 await CollectAsync(collector, opened, info, capabilities, instanceId, now, cancellationToken).ConfigureAwait(false);
             }
 
+            await RefreshBaselineAsync(instanceId, now, cancellationToken).ConfigureAwait(false);
             var results = ruleEngine.Evaluate(new RuleContext(info, capabilities, now, _data));
             await repository.SaveRuleResultsAsync(instanceId, results, now, cancellationToken).ConfigureAwait(false);
             var findings = 0;
@@ -120,6 +125,22 @@ public sealed partial class InstanceMonitor(
         }
     }
 
+    /// <summary>
+    /// The baseline for the current hour of the week, from the repository's hourly history. Refreshed at each new hour.
+    /// </summary>
+    private async Task RefreshBaselineAsync(int instanceId, DateTimeOffset now, CancellationToken cancellationToken)
+    {
+        if (now < _nextBaseline)
+        {
+            return;
+        }
+
+        var history = await repository.GetHourlyHistoryAsync(instanceId, BaselineMetrics, now, cancellationToken: cancellationToken).ConfigureAwait(false);
+        _data.Set(Core.Data.BaselineCalculator.Calculate(history, now));
+        var hour = new DateTimeOffset(now.UtcDateTime.Year, now.UtcDateTime.Month, now.UtcDateTime.Day, now.UtcDateTime.Hour, 0, 0, TimeSpan.Zero);
+        _nextBaseline = hour.AddHours(1);
+    }
+
     /// <summary>
     /// State a collector would otherwise lose when the agent restarts. Deadlocks continue after the newest one
     /// already stored; the others simply start over (cumulative collectors need one extra sample).

src/SQLInsight.Core/Data/Baseline.cs

@@ -0,0 +1,53 @@
+namespace SQLInsight.Core.Data;
+
+/// <summary>
+/// What is normal for a metric in the current hour of the week, learned from earlier weeks.
+/// </summary>
+public sealed record BaselineSnapshot
+{
+    public required DateTimeOffset CapturedAt { get; init; }
+
+    /// <summary>Hour of the week (0 = Monday 00:00–01:00 UTC).</summary>
+    public required int HourOfWeek { get; init; }
+
+    public required IReadOnlyDictionary<string, MetricBaseline> Metrics { get; init; }
+}
+
+/// <param name="Samples">How many earlier weeks the baseline is based on.</param>
+public sealed record MetricBaseline(double Mean, double StandardDeviation, int Samples);
+
+public sealed record HourlyValue(string Metric, DateTime Hour, double Value);
+
+/// <summary>
+/// Computes the baseline for one hour of the week from hourly averages. The same hour in earlier weeks is used, plus
+/// the hour before and after, so three weeks of history already give nine points.
+/// </summary>
+public static class BaselineCalculator
+{
+    public const int MinSamples = 6;
+
+    public static int HourOfWeek(DateTime utc) => ((int)utc.DayOfWeek + 6) % 7 * 24 + utc.Hour;
+
+    public static BaselineSnapshot Calculate(IEnumerable<HourlyValue> history, DateTimeOffset now)
+    {
+        ArgumentNullException.ThrowIfNull(history);
+        var slot = HourOfWeek(now.UtcDateTime);
+        var neighbours = new HashSet<int> { slot, (slot + 167) % 168, (slot + 1) % 168 };
+        var currentHour = new DateTime(now.UtcDateTime.Year, now.UtcDateTime.Month, now.UtcDateTime.Day, now.UtcDateTime.Hour, 0, 0, DateTimeKind.Utc);
+
+        var metrics = history
+            .Where(h => h.Hour < currentHour.AddHours(-2) && neighbours.Contains(HourOfWeek(h.Hour)))
+            .GroupBy(h => h.Metric)
+            .ToDictionary(
+                g => g.Key,
+                g =>
+                {
+                    var values = g.Select(h => h.Value).ToList();
+                    var mean = values.Average();
+                    var variance = values.Count > 1 ? values.Sum(v => (v - mean) * (v - mean)) / (values.Count - 1) : 0;
+                    return new MetricBaseline(mean, Math.Sqrt(variance), values.Count);
+                });
+
+        return new BaselineSnapshot { CapturedAt = now, HourOfWeek = slot, Metrics = metrics };
+    }
+}

src/SQLInsight.Repository/InsightRepository.Read.cs

@@ -256,3 +256,17 @@ public sealed partial class InsightRepository
         public DateTime? ResolvedAt { get; init; }
     }
 }
+
+public sealed partial class InsightRepository
+{
+    /// <summary>Hourly averages for the given metrics over the last <paramref name="days"/> days, for baselines.</summary>
+    public async Task<IReadOnlyList<HourlyValue>> GetHourlyHistoryAsync(int instanceId, IReadOnlyCollection<string> metrics, DateTimeOffset now, int days = 28, CancellationToken cancellationToken = default)
+    {
+        await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
+        var rows = await connection.QueryAsync<(string Metric, DateTime Hour, double Value)>("""
+            SELECT Metric, [Hour], AvgValue FROM insight.MetricHourly
+            WHERE InstanceId = @instanceId AND [Hour] >= @from AND Metric IN (SELECT value FROM OPENJSON(@metrics));
+            """, new { instanceId, from = now.UtcDateTime.AddDays(-days), metrics = JsonSerializer.Serialize(metrics, Json) }).ConfigureAwait(false);
+        return rows.Select(r => new HourlyValue(r.Metric, DateTime.SpecifyKind(r.Hour, DateTimeKind.Utc), r.Value)).ToList();
+    }
+}

src/SQLInsight.Rules/BuiltInRules.cs

@@ -47,6 +47,7 @@ public static class BuiltInRules
         new LogFileWriteLatencyRule(),
         new CpuRule(),
         new MemoryGrantsPendingRule(),
+        new AnomalyRule(),
         new PlanRegressionRule(),
         new FailedForcedPlanRule(),
         new ParameterSensitivityRule(),

src/SQLInsight.Rules/Performance/AnomalyRule.cs

@@ -0,0 +1,95 @@
+using SQLInsight.Core.Collection;
+using SQLInsight.Core.Data;
+using SQLInsight.Core.Findings;
+using SQLInsight.Rules.Engine;
+
+namespace SQLInsight.Rules.Performance;
+
+/// <summary>
+/// Compares the current load with what is normal for this server at this hour of the week.
+/// Fixed thresholds miss that 80 % CPU is normal at month-end but alarming on a Sunday night; a baseline does not.
+/// </summary>
+public sealed class AnomalyRule : Rule
+{
+    public const string CpuMetric = "cpu.sqlserver";
+    public const string BatchMetric = "workload.batch_requests";
+    public const string PleMetric = "memory.ple";
+
+    public override string Id => "SQLI-ANOM-001";
+    public override string Title => "Usædvanlig belastning i forhold til det normale";
+
+    public override RuleEvaluation Evaluate(RuleContext context)
+    {
+        ArgumentNullException.ThrowIfNull(context);
+        var baseline = context.Data.Get<BaselineSnapshot>();
+        var usable = baseline.Metrics.Where(m => m.Value.Samples >= BaselineCalculator.MinSamples).ToDictionary(m => m.Key, m => m.Value);
+        if (usable.Count == 0)
+        {
+            throw new DataUnavailableException("Baseline kræver data fra mindst to-tre uger for denne time på ugen; der er endnu ikke nok historik.");
+        }
+
+        var findings = new List<Finding>();
+        var slot = $"{DayName(baseline.HourOfWeek / 24)} kl. {baseline.HourOfWeek % 24:00}–{(baseline.HourOfWeek % 24 + 1) % 24:00} UTC";
+
+        if (usable.TryGetValue(CpuMetric, out var cpuBase) && context.Data.TryGet<CpuUtilization>(out var cpu) && cpu.Samples.Count >= 5)
+        {
+            var current = cpu.Samples.TakeLast(15).Average(s => s.SqlServerPercent);
+            if (current >= 50 && current >= cpuBase.Mean + Math.Max(3 * cpuBase.StandardDeviation, 20))
+            {
+                findings.Add(Anomaly("cpu", Severity.Medium, $"SQL Server bruger {Format.Number(current)} % CPU; normalt {Format.Number(cpuBase.Mean)} % på dette tidspunkt",
+                    "Belastningen er markant højere end den plejer at være på netop dette tidspunkt af ugen. Det tyder på en ændring: "
+                    + "en ny eller ændret forespørgsel, en planregression, et job der kører på et nyt tidspunkt, eller mere trafik.",
+                    "Se Query Store (Top Resource Consuming Queries) for den seneste time sammenlignet med samme tid sidste uge, "
+                    + "og tjek planregressioner (SQLI-QS-001) og nye deployments.", slot, current, cpuBase, "%"));
+            }
+        }
+
+        if (usable.TryGetValue(BatchMetric, out var batchBase)
+            && context.Data.TryGet<CumulativeSample<PerformanceCounterSnapshot, PerformanceCounterRates>>(out var counters)
+            && counters.Delta?.BatchRequestsPerSecond is { } batches)
+        {
+            if (batches >= 100 && batches >= batchBase.Mean + Math.Max(3 * batchBase.StandardDeviation, batchBase.Mean))
+            {
+                findings.Add(Anomaly("batch-requests", Severity.Low, $"{Format.Number(batches)} batch requests/s; normalt {Format.Number(batchBase.Mean)} på dette tidspunkt",
+                    "Serveren modtager langt flere forespørgsler end normalt på dette tidspunkt. Det kan være forventet (kampagne, månedsafslutning) "
+                    + "eller en fejl, fx en applikation i en genforsøgs-løkke eller en cache der ikke virker.",
+                    "Find de programmer og forespørgsler, der står for stigningen (sys.dm_exec_sessions efter program_name, Query Store efter antal udførelser).",
+                    slot, batches, batchBase, "/s"));
+            }
+            else if (batchBase.Mean >= 50 && batches <= batchBase.Mean * 0.3)
+            {
+                findings.Add(Anomaly("batch-requests", Severity.Medium, $"Kun {Format.Number(batches)} batch requests/s; normalt {Format.Number(batchBase.Mean)} på dette tidspunkt",
+                    "Serveren modtager langt færre forespørgsler end normalt på dette tidspunkt. Det tyder på, at en applikation er nede, "
+                    + "ikke kan forbinde, eller at trafikken er flyttet til en anden server.",
+                    "Tjek applikationerne, der bruger serveren, og om de kan forbinde (login-fejl i error log, netværk, DNS, failover).",
+                    slot, batches, batchBase, "/s"));
+            }
+        }
+
+        if (usable.TryGetValue(PleMetric, out var pleBase)
+            && context.Data.TryGet<CumulativeSample<PerformanceCounterSnapshot, PerformanceCounterRates>>(out var pc)
+            && pc.Snapshot.PageLifeExpectancySeconds is { } ple && pleBase.Mean >= 300 && ple <= pleBase.Mean * 0.3)
+        {
+            findings.Add(Anomaly("ple", Severity.Medium, $"Page life expectancy er {Format.Number(ple)} s; normalt {Format.Number(pleBase.Mean)} s på dette tidspunkt",
+                "Sider bliver smidt ud af buffer pool langt hurtigere end normalt. Noget læser meget mere data end det plejer, typisk en "
+                + "forespørgsel, der er begyndt at scanne en stor tabel, eller et vedligeholdelsesjob.",
+                "Find forespørgslerne med flest læsninger den seneste time (Query Store eller sys.dm_exec_query_stats efter total_logical_reads).",
+                slot, ple, pleBase, " s"));
+        }
+
+        return Result(findings);
+    }
+
+    private Finding Anomaly(string what, Severity severity, string title, string explanation, string recommendation, string slot, double current, MetricBaseline baseline, string unit) =>
+        CreateFinding(severity, title, explanation, recommendation,
+        [
+            new Evidence("Nu", $"{Format.Number(current, 1)}{unit}"),
+            new Evidence("Normalt", $"{Format.Number(baseline.Mean, 1)}{unit} ± {Format.Number(baseline.StandardDeviation, 1)} ({slot}, {baseline.Samples} målepunkter)"),
+        ],
+        subject: $"anomaly: {what}");
+
+    private static string DayName(int day) => day switch
+    {
+        0 => "mandag", 1 => "tirsdag", 2 => "onsdag", 3 => "torsdag", 4 => "fredag", 5 => "lørdag", _ => "søndag",
+    };
+}

tests/SQLInsight.Core.Tests/BaselineTests.cs

@@ -0,0 +1,51 @@
+using SQLInsight.Core.Data;
+
+namespace SQLInsight.Core.Tests;
+
+public class BaselineTests
+{
+    [Theory]
+    [InlineData("2026-09-28T00:30:00Z", 0)]     // Monday
+    [InlineData("2026-09-29T13:10:00Z", 37)]    // Tuesday 13
+    [InlineData("2026-10-04T23:59:00Z", 167)]   // Sunday 23
+    public void Hour_of_week_starts_monday(string utc, int expected)
+    {
+        Assert.Equal(expected, BaselineCalculator.HourOfWeek(DateTime.Parse(utc, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal)));
+    }
+
+    [Fact]
+    public void Uses_the_same_hour_and_its_neighbours_in_earlier_weeks()
+    {
+        var now = new DateTimeOffset(2026, 9, 29, 13, 20, 0, TimeSpan.Zero); // Tuesday 13
+        var history = new List<HourlyValue>();
+        for (var week = 1; week <= 3; week++)
+        {
+            var sameHour = new DateTime(2026, 9, 29, 13, 0, 0, DateTimeKind.Utc).AddDays(-7 * week);
+            history.Add(new HourlyValue("cpu", sameHour.AddHours(-1), 20));
+            history.Add(new HourlyValue("cpu", sameHour, 30));
+            history.Add(new HourlyValue("cpu", sameHour.AddHours(1), 40));
+            history.Add(new HourlyValue("cpu", sameHour.AddHours(5), 99)); // other hour: ignored
+        }
+
+        history.Add(new HourlyValue("cpu", new DateTime(2026, 9, 29, 12, 0, 0, DateTimeKind.Utc), 99)); // today, too recent
+
+        var baseline = BaselineCalculator.Calculate(history, now);
+
+        var cpu = baseline.Metrics["cpu"];
+        Assert.Equal(37, baseline.HourOfWeek);
+        Assert.Equal(9, cpu.Samples);
+        Assert.Equal(30, cpu.Mean, 6);
+        Assert.Equal(Math.Sqrt(75), cpu.StandardDeviation, 6); // (3×100 + 3×0 + 3×100) / 8
+    }
+
+    [Fact]
+    public void Neighbour_hours_wrap_around_the_week()
+    {
+        var now = new DateTimeOffset(2026, 9, 28, 0, 30, 0, TimeSpan.Zero); // Monday 00
+        var sundayLate = new DateTime(2026, 9, 20, 23, 0, 0, DateTimeKind.Utc);
+
+        var baseline = BaselineCalculator.Calculate([new HourlyValue("cpu", sundayLate, 10)], now);
+
+        Assert.Equal(1, baseline.Metrics["cpu"].Samples);
+    }
+}

tests/SQLInsight.IntegrationTests/RepositoryTests.cs

@@ -175,6 +175,29 @@ public class RepositoryTests(SqlServerFixture sql) : IAsyncLifetime
         Assert.Equal(3000, top[0].WaitTimeMs);
     }
 
+    [Fact]
+    public async Task Hourly_history_feeds_the_baseline_and_metric_names_match_the_anomaly_rule()
+    {
+        Assert.Equal(SQLInsight.Rules.Performance.AnomalyRule.CpuMetric, Metrics.CpuSqlServer);
+        Assert.Equal(SQLInsight.Rules.Performance.AnomalyRule.BatchMetric, Metrics.BatchRequestsPerSecond);
+        Assert.Equal(SQLInsight.Rules.Performance.AnomalyRule.PleMetric, Metrics.PageLifeExpectancy);
+
+        var id = await _repository.UpsertInstanceAsync("sql09", Info, "SqlClient", false, Now);
+        var now = DateTimeOffset.UtcNow;
+        await ExecuteAsync($"""
+            INSERT insight.MetricHourly (InstanceId, Metric, [Hour], AvgValue, MaxValue, Samples)
+            VALUES ({id}, 'cpu.sqlserver', DATEADD(day, -7, SYSUTCDATETIME()), 30, 40, 60),
+                   ({id}, 'cpu.sqlserver', DATEADD(day, -40, SYSUTCDATETIME()), 99, 99, 60),
+                   ({id}, 'connections.user', DATEADD(day, -7, SYSUTCDATETIME()), 5, 5, 60);
+            """);
+
+        var history = await _repository.GetHourlyHistoryAsync(id, [Metrics.CpuSqlServer], now);
+
+        var point = Assert.Single(history);
+        Assert.Equal(30, point.Value);
+        Assert.Equal(DateTimeKind.Utc, point.Hour.Kind);
+    }
+
     [Fact]
     public async Task Maintenance_deletes_data_past_retention()
     {

tests/SQLInsight.Rules.Tests/AnomalyRuleTests.cs

@@ -0,0 +1,70 @@
+using SQLInsight.Core.Data;
+using SQLInsight.Core.Findings;
+using SQLInsight.Rules.Engine;
+using SQLInsight.Rules.Performance;
+using static SQLInsight.Rules.Tests.TestInstances;
+
+namespace SQLInsight.Rules.Tests;
+
+public class AnomalyRuleTests
+{
+    private static RuleResult Run(RuleContext context) => Assert.Single(new RuleEngine([new AnomalyRule()]).Evaluate(context));
+
+    private static BaselineSnapshot Baseline(params (string Metric, double Mean, double Stdev, int Samples)[] metrics) => new()
+    {
+        CapturedAt = Now,
+        HourOfWeek = 37,
+        Metrics = metrics.ToDictionary(m => m.Metric, m => new MetricBaseline(m.Mean, m.Stdev, m.Samples)),
+    };
+
+    private static CpuUtilization Cpu(int percent) => new()
+    {
+        CapturedAt = Now,
+        Samples = Enumerable.Range(0, 15).Select(i => new CpuSample(Now.AddMinutes(i - 15), percent, 100 - percent)).ToList(),
+    };
+
+    private static CumulativeSample<PerformanceCounterSnapshot, PerformanceCounterRates> Counters(double batches, long ple) => new(
+        new PerformanceCounterSnapshot { CapturedAt = Now, SqlServerStartTime = Now.UtcDateTime.AddDays(-1), PageLifeExpectancySeconds = ple },
+        new PerformanceCounterRates { From = Now.AddMinutes(-5), To = Now, BatchRequestsPerSecond = batches });
+
+    [Fact]
+    public void Too_little_history_is_not_measurable()
+    {
+        var result = Run(With(Baseline((AnomalyRule.CpuMetric, 20, 5, 3)), Cpu(90)));
+
+        Assert.Equal(RuleOutcome.NotMeasurable, result.Outcome);
+        Assert.Contains("uger", result.Reason, StringComparison.Ordinal);
+    }
+
+    [Fact]
+    public void Cpu_far_above_normal_for_this_hour()
+    {
+        var finding = Assert.Single(Run(With(Baseline((AnomalyRule.CpuMetric, 20, 5, 9)), Cpu(75))).Findings);
+
+        Assert.Equal("anomaly: cpu", finding.Subject);
+        Assert.Contains("tirsdag kl. 13–14 UTC", finding.Evidence[1].Value, StringComparison.Ordinal);
+    }
+
+    [Fact]
+    public void High_cpu_that_is_normal_for_this_hour_passes()
+    {
+        Assert.Equal(RuleOutcome.Passed, Run(With(Baseline((AnomalyRule.CpuMetric, 80, 8, 9)), Cpu(88))).Outcome);
+    }
+
+    [Fact]
+    public void Traffic_drop_suggests_an_application_is_down()
+    {
+        var finding = Assert.Single(Run(With(Baseline((AnomalyRule.BatchMetric, 800, 100, 9)), Counters(batches: 40, ple: 5000))).Findings);
+
+        Assert.Equal(Severity.Medium, finding.Severity);
+        Assert.Contains("færre", finding.Explanation, StringComparison.Ordinal);
+    }
+
+    [Fact]
+    public void Ple_collapse()
+    {
+        var finding = Assert.Single(Run(With(Baseline((AnomalyRule.PleMetric, 4000, 500, 9)), Counters(batches: 0, ple: 300))).Findings);
+
+        Assert.Equal("anomaly: ple", finding.Subject);
+    }
+}