seo/sqlinsight-pro
1
0
Fork 0
Code Issues Pull requests Projects Releases 11 Packages Wiki Activity Actions

Top SQL: collect what the heaviest statements use every 5 minutes

Browse source 
New collector query-stats reads sys.dm_exec_query_stats for the statements
that ran since the previous collection, grouped by query_hash (SQL Server
2008+) or sql_handle + offsets (2005, via legacy ODBC; literals only). Only
the 50 heaviest by CPU, duration, reads and executions, text cut to 4000
characters, no plan XML. The database comes from the plan's dbid attribute,
because ad hoc batches (as Dynamics AX sends) have none in sql_text.

QueryStatsDelta diffs against the previous collection: a statement seen
before with the same plan gets the difference, a plan compiled since then
counts in full, a recompiled or evicted plan or counters that went
backwards start over (never negative), a restart gives no delta. Up to 2000
statements are remembered between collections.

Repository migration 0006: insight.QueryStatSamples (deltas, top 100 per
collection) and insight.QueryTexts (text once per statement), kept for
Retention:RawDays. The text is customer data: never stored as a JSON
snapshot and never part of hub submissions. Window queries for metrics,
waits (raw and hourly), file I/O, blocking, deadlocks, findings active in a
window and top SQL by CPU/duration/reads/executions.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Søren Eggert Lundsteen Olsen 2026-10-01 15:46:45 +02:00
parent 2c49a1e1b1
commit bb3b38e77f
8 changed files with 647 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/SQLInsight.Collectors/BuiltInCollectors.cs
View file

@ -18,6 +18,7 @@ public static class BuiltInCollectors
new PlanCacheCollector(), new PlanCacheCollector(),
new WaitStatsCollector(), new WaitStatsCollector(),
new FileIoCollector(), new FileIoCollector(),
new QueryStatsCollector(),
new PerformanceCounterCollector(), new PerformanceCounterCollector(),
new CpuCollector(), new CpuCollector(),
new BlockingCollector(), new BlockingCollector(),

131
src/SQLInsight.Collectors/Performance/QueryStatsCollector.cs Normal file
View file

@ -0,0 +1,131 @@
using System.Globalization;
using SQLInsight.Core.Data;
using SQLInsight.Core.Instances;
namespace SQLInsight.Collectors.Performance;
/// <summary>
/// Top SQL: cumulative statistics from <c>sys.dm_exec_query_stats</c> for the statements that ran since the previous
/// collection, so the time window analysis can show which statements used the CPU, time and reads. Kept light: only
/// the heaviest <see cref="TopPerMeasure"/> statements by CPU, duration, reads and executions, their text cut to
/// <see cref="QueryStat.MaxTextLength"/> characters, and no plan XML.
/// </summary>
public sealed class QueryStatsCollector : Collector<CumulativeSample<QueryStatsSnapshot, QueryStatsDelta>>
{
public const int TopPerMeasure = 50;
/// <summary>
/// Statements are grouped by query_hash, so the same statement with different literals counts as one. The database
/// is the batch's own (procedures) or else the one it ran in (plan attribute dbid): ad hoc batches, as AX sends, have none.
/// </summary>
internal const string Sql2008 = """
WITH s AS (
SELECT qs.query_hash AS k,
SUM(qs.execution_count) AS execs, SUM(qs.total_worker_time) AS cpu, SUM(qs.total_elapsed_time) AS elapsed,
SUM(qs.total_logical_reads) AS lreads, SUM(qs.total_physical_reads) AS preads, SUM(qs.total_logical_writes) AS writes,
MIN(qs.creation_time) AS created, MAX(qs.last_execution_time) AS last_exec
FROM sys.dm_exec_query_stats qs
GROUP BY qs.query_hash
HAVING MAX(qs.last_execution_time) >= {since}),
t AS (
SELECT k FROM (SELECT TOP ({top}) k FROM s ORDER BY cpu DESC) a
UNION SELECT k FROM (SELECT TOP ({top}) k FROM s ORDER BY elapsed DESC) b
UNION SELECT k FROM (SELECT TOP ({top}) k FROM s ORDER BY lreads DESC) c
UNION SELECT k FROM (SELECT TOP ({top}) k FROM s ORDER BY execs DESC) d)
SELECT CONVERT(varchar(20), s.k, 1), s.execs, s.cpu, s.elapsed, s.lreads, s.preads, s.writes, s.created, s.last_exec,
DB_NAME(COALESCE(st.dbid, pa.dbid)),
CAST(SUBSTRING(st.text, r.statement_start_offset / 2 + 1,
CASE WHEN r.statement_end_offset = -1 THEN {length}
ELSE (r.statement_end_offset - r.statement_start_offset) / 2 + 1 END) AS nvarchar({length})),
GETDATE()
FROM s JOIN t ON t.k = s.k
OUTER APPLY (SELECT TOP (1) q.sql_handle, q.plan_handle, q.statement_start_offset, q.statement_end_offset
FROM sys.dm_exec_query_stats q WHERE q.query_hash = s.k ORDER BY q.last_execution_time DESC) r
OUTER APPLY sys.dm_exec_sql_text(r.sql_handle) st
OUTER APPLY (SELECT CAST(a.value AS int) AS dbid FROM sys.dm_exec_plan_attributes(r.plan_handle) a WHERE a.attribute = N'dbid') pa
""";
/// <summary>SQL Server 2005 has no query_hash: a statement is its batch (sql_handle) and its offsets in it.</summary>
internal const string Sql2005 = """
WITH s AS (
SELECT qs.sql_handle AS h, qs.statement_start_offset AS so, qs.statement_end_offset AS eo,
SUM(qs.execution_count) AS execs, SUM(qs.total_worker_time) AS cpu, SUM(qs.total_elapsed_time) AS elapsed,
SUM(qs.total_logical_reads) AS lreads, SUM(qs.total_physical_reads) AS preads, SUM(qs.total_logical_writes) AS writes,
MIN(qs.creation_time) AS created, MAX(qs.last_execution_time) AS last_exec
FROM sys.dm_exec_query_stats qs
GROUP BY qs.sql_handle, qs.statement_start_offset, qs.statement_end_offset
HAVING MAX(qs.last_execution_time) >= {since}),
t AS (
SELECT h, so, eo FROM (SELECT TOP ({top}) h, so, eo FROM s ORDER BY cpu DESC) a
UNION SELECT h, so, eo FROM (SELECT TOP ({top}) h, so, eo FROM s ORDER BY elapsed DESC) b
UNION SELECT h, so, eo FROM (SELECT TOP ({top}) h, so, eo FROM s ORDER BY lreads DESC) c
UNION SELECT h, so, eo FROM (SELECT TOP ({top}) h, so, eo FROM s ORDER BY execs DESC) d)
SELECT sys.fn_varbintohexstr(s.h) + ':' + CAST(s.so AS varchar(12)) + ':' + CAST(s.eo AS varchar(12)),
s.execs, s.cpu, s.elapsed, s.lreads, s.preads, s.writes, s.created, s.last_exec,
DB_NAME(COALESCE(st.dbid, pa.dbid)),
CAST(SUBSTRING(st.text, s.so / 2 + 1, CASE WHEN s.eo = -1 THEN {length} ELSE (s.eo - s.so) / 2 + 1 END) AS nvarchar({length})),
GETDATE()
FROM s JOIN t ON t.h = s.h AND t.so = s.so AND t.eo = s.eo
OUTER APPLY sys.dm_exec_sql_text(s.h) st
OUTER APPLY (SELECT TOP (1) q.plan_handle FROM sys.dm_exec_query_stats q
WHERE q.sql_handle = s.h AND q.statement_start_offset = s.so AND q.statement_end_offset = s.eo) r
OUTER APPLY (SELECT CAST(a.value AS int) AS dbid FROM sys.dm_exec_plan_attributes(r.plan_handle) a WHERE a.attribute = N'dbid') pa
""";
public override string Name => "query-stats";
public override TimeSpan DefaultInterval => TimeSpan.FromMinutes(5);
public override IReadOnlyCollection<Capability> RequiredCapabilities => [Capability.DynamicManagementViews];
protected override async Task<CumulativeSample<QueryStatsSnapshot, QueryStatsDelta>> CollectAsync(
CollectorContext context, CumulativeSample<QueryStatsSnapshot, QueryStatsDelta>? previous)
{
var version = context.Instance.Version;
var ct = context.CancellationToken;
var startTime = await Db.StartTimeAsync(context.Connection, version, ct).ConfigureAwait(false);
var sql = SqlFor(version, previous?.Snapshot is { } last && last.SqlServerStartTime == startTime ? last.ServerTime : null);
DateTime? serverTime = null;
var rows = await Db.QueryAsync(context.Connection, sql, r =>
{
serverTime ??= r.DateTime(11);
var text = r.NullableString(10);
return new QueryStat
{
Key = r.String(0),
Executions = r.Long(1),
WorkerTimeUs = r.Long(2),
ElapsedTimeUs = r.Long(3),
LogicalReads = r.Long(4),
PhysicalReads = r.Long(5),
LogicalWrites = r.Long(6),
CreatedAt = r.DateTime(7),
LastExecutionAt = r.DateTime(8),
DatabaseName = r.NullableString(9),
Text = text is { Length: > QueryStat.MaxTextLength } ? text[..QueryStat.MaxTextLength] : text,
};
}, ct).ConfigureAwait(false);
// No statement ran since the previous collection: the server clock still has to move forward.
serverTime ??= (await Db.QueryAsync(context.Connection, "SELECT GETDATE()", r => r.DateTime(0), ct).ConfigureAwait(false))[0];
var fetched = new QueryStatsSnapshot { CapturedAt = context.Now, ServerTime = serverTime.Value, SqlServerStartTime = startTime, Queries = rows };
var delta = QueryStatsDelta.Compute(previous?.Snapshot, fetched);
var kept = fetched with { Queries = QueryStatsDelta.Remember(previous?.Snapshot, rows, startTime) };
return new(kept, delta);
}
/// <summary>
/// The query for the version. <paramref name="since"/> is the server clock at the previous collection; the first
/// time, the last 15 minutes. A literal, not a parameter: ODBC (SQL Server 2005) only knows positional parameters.
/// </summary>
internal static string SqlFor(SqlServerVersion version, DateTime? since)
{
var sinceSql = since is { } value
? "CONVERT(datetime, '" + value.ToString("yyyy-MM-ddTHH:mm:ss.fff", CultureInfo.InvariantCulture) + "', 126)"
: "DATEADD(minute, -15, GETDATE())";
return (version.Major >= 10 ? Sql2008 : Sql2005)
.Replace("{since}", sinceSql, StringComparison.Ordinal)
.Replace("{top}", TopPerMeasure.ToString(CultureInfo.InvariantCulture), StringComparison.Ordinal)
.Replace("{length}", QueryStat.MaxTextLength.ToString(CultureInfo.InvariantCulture), StringComparison.Ordinal);
}
}

156
src/SQLInsight.Core/Data/QueryStats.cs Normal file
View file

@ -0,0 +1,156 @@
namespace SQLInsight.Core.Data;
/// <summary>
/// Cumulative statistics for the statements that ran since the previous collection (<c>sys.dm_exec_query_stats</c>,
/// grouped per statement), plus statements remembered from earlier collections so their totals can still be diffed.
/// </summary>
public sealed record QueryStatsSnapshot
{
/// <summary>Most statements remembered between collections; the least recently executed are forgotten first.</summary>
public const int MaxRemembered = 2000;
public required DateTimeOffset CapturedAt { get; init; }
/// <summary>The server's own clock (GETDATE()) at capture: <c>creation_time</c> and <c>last_execution_time</c> are in it.</summary>
public required DateTime ServerTime { get; init; }
public required DateTime SqlServerStartTime { get; init; }
public required IReadOnlyList<QueryStat> Queries { get; init; }
}
/// <summary>
/// One statement's totals since its first plan in cache was compiled. Times are in microseconds, as the DMV gives them.
/// </summary>
public sealed record QueryStat
{
/// <summary>query_hash as hex (SQL Server 2008+), or sql_handle:start:end on SQL Server 2005.</summary>
public required string Key { get; init; }
public string? DatabaseName { get; init; }
/// <summary>The statement, cut to <see cref="MaxTextLength"/> characters. Customer data: never sent to the hub.</summary>
public string? Text { get; init; }
public required long Executions { get; init; }
public required long WorkerTimeUs { get; init; }
public required long ElapsedTimeUs { get; init; }
public required long LogicalReads { get; init; }
public required long PhysicalReads { get; init; }
public required long LogicalWrites { get; init; }
/// <summary>The oldest cached plan's creation_time (server clock). A newer value means the plan was recompiled.</summary>
public required DateTime CreatedAt { get; init; }
/// <summary>last_execution_time (server clock).</summary>
public required DateTime LastExecutionAt { get; init; }
public const int MaxTextLength = 4000;
}
/// <summary>
/// What each statement used between two collections.
/// </summary>
public sealed record QueryStatsDelta
{
public required DateTimeOffset From { get; init; }
public required DateTimeOffset To { get; init; }
public required IReadOnlyList<QueryStatDelta> Queries { get; init; }
/// <summary>
/// The change per statement. A statement only gets a delta when its starting point is known: it was seen before
/// with the same plan, or its plan was compiled after the previous collection (then all of its totals are new).
/// A plan recompiled or evicted in between, or counters that went backwards, start over: no negative numbers.
/// After an instance restart there is no delta at all.
/// </summary>
public static QueryStatsDelta? Compute(QueryStatsSnapshot? previous, QueryStatsSnapshot current)
{
ArgumentNullException.ThrowIfNull(current);
if (previous is null || previous.SqlServerStartTime != current.SqlServerStartTime || current.CapturedAt <= previous.CapturedAt)
{
return null;
}
var before = previous.Queries.ToDictionary(q => q.Key, StringComparer.Ordinal);
var changes = new List<QueryStatDelta>();
foreach (var now in current.Queries)
{
// Remembered from an earlier collection but not run since: nothing new.
if (now.LastExecutionAt < previous.ServerTime && before.ContainsKey(now.Key))
{
continue;
}
var newSinceLast = now.CreatedAt >= previous.ServerTime;
QueryStatDelta? delta;
if (before.TryGetValue(now.Key, out var then) && then.CreatedAt == now.CreatedAt && !WentBackwards(then, now))
{
delta = Difference(now, then);
}
else
{
delta = newSinceLast ? Difference(now, null) : null;
}
if (delta is { Executions: > 0 } or { WorkerTimeUs: > 0 })
{
changes.Add(delta);
}
}
return new QueryStatsDelta
{
From = previous.CapturedAt,
To = current.CapturedAt,
Queries = changes.OrderByDescending(q => q.WorkerTimeUs).ThenByDescending(q => q.ElapsedTimeUs).ToList(),
};
}
/// <summary>
/// The snapshot to keep for the next collection: this collection's statements plus the remembered ones not seen
/// now, at most <see cref="QueryStatsSnapshot.MaxRemembered"/>. After a restart the old ones are dropped.
/// </summary>
public static IReadOnlyList<QueryStat> Remember(QueryStatsSnapshot? previous, IReadOnlyList<QueryStat> current, DateTime sqlServerStartTime)
{
ArgumentNullException.ThrowIfNull(current);
var keys = current.Select(q => q.Key).ToHashSet(StringComparer.Ordinal);
var older = previous is null || previous.SqlServerStartTime != sqlServerStartTime
? []
: previous.Queries.Where(q => !keys.Contains(q.Key));
return current.Concat(older.OrderByDescending(q => q.LastExecutionAt))
.Take(QueryStatsSnapshot.MaxRemembered)
.ToList();
}
private static bool WentBackwards(QueryStat then, QueryStat now) =>
now.Executions < then.Executions || now.WorkerTimeUs < then.WorkerTimeUs || now.ElapsedTimeUs < then.ElapsedTimeUs
|| now.LogicalReads < then.LogicalReads || now.PhysicalReads < then.PhysicalReads || now.LogicalWrites < then.LogicalWrites;
private static QueryStatDelta Difference(QueryStat now, QueryStat? then) => new()
{
Key = now.Key,
DatabaseName = now.DatabaseName,
Text = now.Text,
Executions = now.Executions - (then?.Executions ?? 0),
WorkerTimeUs = now.WorkerTimeUs - (then?.WorkerTimeUs ?? 0),
ElapsedTimeUs = now.ElapsedTimeUs - (then?.ElapsedTimeUs ?? 0),
LogicalReads = now.LogicalReads - (then?.LogicalReads ?? 0),
PhysicalReads = now.PhysicalReads - (then?.PhysicalReads ?? 0),
LogicalWrites = now.LogicalWrites - (then?.LogicalWrites ?? 0),
};
}
public sealed record QueryStatDelta
{
public required string Key { get; init; }
public string? DatabaseName { get; init; }
/// <summary>The statement. Customer data: shown only in the local web UI, never sent to the hub.</summary>
public string? Text { get; init; }
public required long Executions { get; init; }
public required long WorkerTimeUs { get; init; }
public required long ElapsedTimeUs { get; init; }
public required long LogicalReads { get; init; }
public required long PhysicalReads { get; init; }
public required long LogicalWrites { get; init; }
}

189
src/SQLInsight.Repository/InsightRepository.Analysis.cs Normal file
View file

@ -0,0 +1,189 @@
using System.Text.Json;
using Dapper;
using Microsoft.Data.SqlClient;
using SQLInsight.Core.Data;
namespace SQLInsight.Repository;
/// <summary>A statement's use summed over a time window. <see cref="Text"/> is customer data (local UI only).</summary>
public sealed record TopQuery
{
public required string QueryKey { get; init; }
public string? DatabaseName { get; init; }
public string? Text { get; init; }
public long Executions { get; init; }
public long WorkerTimeUs { get; init; }
public long ElapsedTimeUs { get; init; }
public long LogicalReads { get; init; }
public long PhysicalReads { get; init; }
public long LogicalWrites { get; init; }
}
/// <summary>What a top-SQL list is sorted by.</summary>
public enum TopQueryOrder
{
Cpu,
Duration,
Reads,
Executions,
}
/// <summary>Wait time summed over a span of whole hours (hourly history).</summary>
public sealed record HourlyWaitTotal(string WaitType, long WaitTimeMs, long WaitingTasks, int Hours);
public sealed partial class InsightRepository
{
/// <summary>At most this many statements are stored per collection: the heaviest by CPU.</summary>
public const int MaxQueriesPerSample = 100;
private static async Task SaveQueryStatsAsync(SqlConnection connection, int instanceId, QueryStatsDelta delta)
{
var queries = delta.Queries.Take(MaxQueriesPerSample).ToList();
if (queries.Count == 0)
{
return;
}
await connection.ExecuteAsync("""
INSERT insight.QueryStatSamples (InstanceId, SampledAt, IntervalSeconds, QueryKey, Executions, WorkerTimeUs, ElapsedTimeUs, LogicalReads, PhysicalReads, LogicalWrites)
SELECT @instanceId, @at, @interval, j.[Key], j.Executions, j.WorkerTimeUs, j.ElapsedTimeUs, j.LogicalReads, j.PhysicalReads, j.LogicalWrites
FROM OPENJSON(@json) WITH ([Key] varchar(200), Executions bigint, WorkerTimeUs bigint, ElapsedTimeUs bigint,
LogicalReads bigint, PhysicalReads bigint, LogicalWrites bigint) j
WHERE NOT EXISTS (SELECT 1 FROM insight.QueryStatSamples s WHERE s.InstanceId = @instanceId AND s.SampledAt = @at AND s.QueryKey = j.[Key]);
MERGE insight.QueryTexts AS t
USING (SELECT [Key], MAX(DatabaseName) AS DatabaseName, MAX([Text]) AS [Text]
FROM OPENJSON(@json) WITH ([Key] varchar(200), DatabaseName nvarchar(128), [Text] nvarchar(4000)) GROUP BY [Key]) AS s
ON t.InstanceId = @instanceId AND t.QueryKey = s.[Key]
WHEN MATCHED THEN UPDATE SET LastSeenAt = @at, QueryText = COALESCE(t.QueryText, s.[Text]), DatabaseName = COALESCE(t.DatabaseName, s.DatabaseName)
WHEN NOT MATCHED THEN INSERT (InstanceId, QueryKey, DatabaseName, QueryText, FirstSeenAt, LastSeenAt) VALUES (@instanceId, s.[Key], s.DatabaseName, s.[Text], @at, @at);
""", new
{
instanceId,
at = Utc(delta.To),
interval = (int)(delta.To - delta.From).TotalSeconds,
json = JsonSerializer.Serialize(queries, Json),
}).ConfigureAwait(false);
}
/// <summary>The statements that used the most in [from, to), with their text.</summary>
public async Task<IReadOnlyList<TopQuery>> GetTopQueriesAsync(int instanceId, DateTimeOffset from, DateTimeOffset to, TopQueryOrder order = TopQueryOrder.Cpu, int top = 25, CancellationToken cancellationToken = default)
{
var orderBy = order switch
{
TopQueryOrder.Duration => "SUM(s.ElapsedTimeUs)",
TopQueryOrder.Reads => "SUM(s.LogicalReads)",
TopQueryOrder.Executions => "SUM(s.Executions)",
_ => "SUM(s.WorkerTimeUs)",
};
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<TopQuery>($"""
SELECT TOP (@top) s.QueryKey, MAX(t.DatabaseName) AS DatabaseName, MAX(t.QueryText) AS [Text],
SUM(s.Executions) AS Executions, SUM(s.WorkerTimeUs) AS WorkerTimeUs, SUM(s.ElapsedTimeUs) AS ElapsedTimeUs,
SUM(s.LogicalReads) AS LogicalReads, SUM(s.PhysicalReads) AS PhysicalReads, SUM(s.LogicalWrites) AS LogicalWrites
FROM insight.QueryStatSamples s
LEFT JOIN insight.QueryTexts t ON t.InstanceId = s.InstanceId AND t.QueryKey = s.QueryKey
WHERE s.InstanceId = @instanceId AND s.SampledAt > @from AND s.SampledAt <= @to
GROUP BY s.QueryKey
ORDER BY {orderBy} DESC;
""", new { instanceId, from = from.UtcDateTime, to = to.UtcDateTime, top }).ConfigureAwait(false);
return rows.ToList();
}
/// <summary>Raw 5-minute samples of a metric in [from, to].</summary>
public async Task<IReadOnlyList<MetricPoint>> GetMetricSamplesAsync(int instanceId, string metric, DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken = default)
{
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<MetricPoint>(
"SELECT SampledAt AS At, Value FROM insight.MetricSamples WHERE InstanceId = @instanceId AND Metric = @metric AND SampledAt >= @from AND SampledAt <= @to ORDER BY SampledAt;",
new { instanceId, metric, from = from.UtcDateTime, to = to.UtcDateTime }).ConfigureAwait(false);
return rows.ToList();
}
/// <summary>Hourly averages of a metric for the hours overlapping [from, to).</summary>
public async Task<IReadOnlyList<MetricPoint>> GetMetricHourlyAsync(int instanceId, string metric, DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken = default)
{
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<MetricPoint>(
"SELECT [Hour] AS At, AvgValue AS Value FROM insight.MetricHourly WHERE InstanceId = @instanceId AND Metric = @metric AND [Hour] >= @from AND [Hour] < @to ORDER BY [Hour];",
new { instanceId, metric, from = FloorHour(from), to = to.UtcDateTime }).ConfigureAwait(false);
return rows.ToList();
}
/// <summary>Waits from the 5-minute samples taken in (from, to].</summary>
public async Task<IReadOnlyList<WaitTotal>> GetWaitsAsync(int instanceId, DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken = default)
{
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<WaitTotal>("""
SELECT WaitType, SUM(WaitingTasks) AS WaitingTasks, SUM(WaitTimeMs) AS WaitTimeMs, SUM(SignalWaitTimeMs) AS SignalWaitTimeMs
FROM insight.WaitSamples WHERE InstanceId = @instanceId AND SampledAt > @from AND SampledAt <= @to
GROUP BY WaitType ORDER BY SUM(WaitTimeMs) DESC;
""", new { instanceId, from = from.UtcDateTime, to = to.UtcDateTime }).ConfigureAwait(false);
return rows.ToList();
}
/// <summary>Waits from the hourly history for the hours overlapping [from, to), with how many hours had data.</summary>
public async Task<IReadOnlyList<HourlyWaitTotal>> GetWaitsHourlyAsync(int instanceId, DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken = default)
{
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<HourlyWaitTotal>("""
SELECT w.WaitType, SUM(w.WaitTimeMs) AS WaitTimeMs, SUM(w.WaitingTasks) AS WaitingTasks,
(SELECT COUNT(DISTINCT h.[Hour]) FROM insight.WaitHourly h WHERE h.InstanceId = @instanceId AND h.[Hour] >= @from AND h.[Hour] < @to) AS Hours
FROM insight.WaitHourly w WHERE w.InstanceId = @instanceId AND w.[Hour] >= @from AND w.[Hour] < @to
GROUP BY w.WaitType;
""", new { instanceId, from = FloorHour(from), to = to.UtcDateTime }).ConfigureAwait(false);
return rows.ToList();
}
public async Task<IReadOnlyList<FileLatency>> GetFileLatencyAsync(int instanceId, DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken = default)
{
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<FileLatency>("""
SELECT DatabaseName, FileId, MAX(FileType) AS FileType, SUM(Reads) AS Reads, SUM(ReadStallMs) AS ReadStallMs, SUM(Writes) AS Writes, SUM(WriteStallMs) AS WriteStallMs
FROM insight.FileIoSamples WHERE InstanceId = @instanceId AND SampledAt > @from AND SampledAt <= @to
GROUP BY DatabaseName, FileId
ORDER BY SUM(ReadStallMs) + SUM(WriteStallMs) DESC;
""", new { instanceId, from = from.UtcDateTime, to = to.UtcDateTime }).ConfigureAwait(false);
return rows.ToList();
}
public async Task<IReadOnlyList<StoredBlocking>> GetBlockingAsync(int instanceId, DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken = default)
{
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<(DateTime SampledAt, int HeadSessionId, int BlockedCount, long MaxWaitMs, string SessionsJson)>("""
SELECT TOP (200) SampledAt, HeadSessionId, BlockedCount, MaxWaitMs, SessionsJson FROM insight.BlockingEvents
WHERE InstanceId = @instanceId AND SampledAt >= @from AND SampledAt <= @to ORDER BY SampledAt;
""", new { instanceId, from = from.UtcDateTime, to = to.UtcDateTime }).ConfigureAwait(false);
return rows.Select(r => new StoredBlocking(r.SampledAt, r.HeadSessionId, r.BlockedCount, r.MaxWaitMs,
JsonSerializer.Deserialize<List<SessionInfo>>(r.SessionsJson, Json) ?? [])).ToList();
}
public async Task<IReadOnlyList<StoredDeadlock>> GetDeadlocksAsync(int instanceId, DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken = default)
{
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<(long DeadlockId, DateTime OccurredAt, string ProcessesJson, string GraphXml)>("""
SELECT TOP (200) DeadlockId, OccurredAt, ProcessesJson, GraphXml FROM insight.Deadlocks
WHERE InstanceId = @instanceId AND OccurredAt >= @from AND OccurredAt <= @to ORDER BY OccurredAt;
""", new { instanceId, from = from.UtcDateTime, to = to.UtcDateTime }).ConfigureAwait(false);
return rows.Select(r => new StoredDeadlock(r.DeadlockId, r.OccurredAt, JsonSerializer.Deserialize<List<DeadlockProcess>>(r.ProcessesJson, Json) ?? [], r.GraphXml)).ToList();
}
/// <summary>Findings that were active at some point in [from, to] (first seen before the end, not resolved before the start).</summary>
public async Task<IReadOnlyList<StoredFinding>> GetFindingsActiveDuringAsync(int instanceId, DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken = default)
{
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<FindingRow>($"""
SELECT TOP (200) {FindingColumns}
FROM insight.Findings f JOIN insight.Instances i ON i.InstanceId = f.InstanceId
WHERE f.InstanceId = @instanceId AND f.FirstSeenAt <= @to AND (f.ResolvedAt IS NULL OR f.ResolvedAt >= @from)
ORDER BY f.Severity DESC, f.FirstSeenAt;
""", new { instanceId, from = from.UtcDateTime, to = to.UtcDateTime }).ConfigureAwait(false);
return rows.Select(Map).ToList();
}
private static DateTime FloorHour(DateTimeOffset value)
{
var utc = value.UtcDateTime;
return new DateTime(utc.Year, utc.Month, utc.Day, utc.Hour, 0, 0, DateTimeKind.Utc);
}
}

2
src/SQLInsight.Repository/InsightRepository.Maintenance.cs
View file

@ -63,6 +63,8 @@ public sealed partial class InsightRepository
await DeleteInBatchesAsync(connection, "insight.MetricSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.MetricSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.WaitSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.WaitSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.FileIoSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.FileIoSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.QueryStatSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.QueryTexts", "LastSeenAt < @cutoff", raw).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.MetricHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.MetricHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.WaitHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.WaitHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.BlockingEvents", "SampledAt < @cutoff", events).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.BlockingEvents", "SampledAt < @cutoff", events).ConfigureAwait(false);

6
src/SQLInsight.Repository/InsightRepository.cs
View file

@ -198,6 +198,12 @@ public sealed partial class InsightRepository(string connectionString)
case DeadlockSnapshot deadlocks: case DeadlockSnapshot deadlocks:
await SaveDeadlocksAsync(connection, instanceId, deadlocks).ConfigureAwait(false); await SaveDeadlocksAsync(connection, instanceId, deadlocks).ConfigureAwait(false);
return; return;
case CumulativeSample<QueryStatsSnapshot, QueryStatsDelta> { Delta: { } queries }:
await SaveQueryStatsAsync(connection, instanceId, queries).ConfigureAwait(false);
return;
case CumulativeSample<QueryStatsSnapshot, QueryStatsDelta>:
// Never as a JSON snapshot: it holds statement texts.
return;
default: default:
await connection.ExecuteAsync(""" await connection.ExecuteAsync("""
MERGE insight.Snapshots AS t MERGE insight.Snapshots AS t

28
src/SQLInsight.Repository/Migrations/0006_query_stats.sql Normal file
View file

@ -0,0 +1,28 @@
/* Top SQL: what the heaviest statements used per collection (deltas of sys.dm_exec_query_stats), for the time window
analysis. Kept as long as the other raw samples (Retention:RawDays). */
CREATE TABLE insight.QueryStatSamples
(
InstanceId int NOT NULL,
SampledAt datetime2(0) NOT NULL,
IntervalSeconds int NOT NULL,
QueryKey varchar(200) NOT NULL,
Executions bigint NOT NULL,
WorkerTimeUs bigint NOT NULL,
ElapsedTimeUs bigint NOT NULL,
LogicalReads bigint NOT NULL,
PhysicalReads bigint NOT NULL,
LogicalWrites bigint NOT NULL,
CONSTRAINT PK_QueryStatSamples PRIMARY KEY (InstanceId, SampledAt, QueryKey)
);
/* The statement text once per statement. Customer data: shown in the local web UI only, never sent to the hub. */
CREATE TABLE insight.QueryTexts
(
InstanceId int NOT NULL,
QueryKey varchar(200) NOT NULL,
DatabaseName nvarchar(128) NULL,
QueryText nvarchar(4000) NULL,
FirstSeenAt datetime2(0) NOT NULL,
LastSeenAt datetime2(0) NOT NULL,
CONSTRAINT PK_QueryTexts PRIMARY KEY (InstanceId, QueryKey)
);

134
tests/SQLInsight.Core.Tests/QueryStatsDeltaTests.cs Normal file
View file

@ -0,0 +1,134 @@
using SQLInsight.Core.Data;
namespace SQLInsight.Core.Tests;
public class QueryStatsDeltaTests
{
private static readonly DateTime Start = new(2026, 9, 20, 0, 0, 0);
private static readonly DateTimeOffset T0 = new(2026, 9, 20, 3, 0, 0, TimeSpan.Zero);
private static QueryStat Stat(string key, long executions, long cpu, DateTime created, DateTime lastRun, long reads = 0) => new()
{
Key = key,
Text = "SELECT " + key,
Executions = executions,
WorkerTimeUs = cpu,
ElapsedTimeUs = cpu * 2,
LogicalReads = reads,
PhysicalReads = 0,
LogicalWrites = 0,
CreatedAt = created,
LastExecutionAt = lastRun,
};
private static QueryStatsSnapshot Snapshot(DateTimeOffset at, DateTime serverTime, params QueryStat[] queries) => new()
{
CapturedAt = at,
ServerTime = serverTime,
SqlServerStartTime = Start,
Queries = queries,
};
[Fact]
public void Statement_seen_before_with_the_same_plan_gets_the_difference()
{
var created = new DateTime(2026, 9, 20, 1, 0, 0);
var before = Snapshot(T0, new DateTime(2026, 9, 20, 3, 0, 0), Stat("0xA", 100, 5_000, created, created.AddHours(1), reads: 1_000));
var now = Snapshot(T0.AddMinutes(5), new DateTime(2026, 9, 20, 3, 5, 0), Stat("0xA", 130, 8_000, created, new DateTime(2026, 9, 20, 3, 4, 0), reads: 1_600));
var delta = QueryStatsDelta.Compute(before, now)!;
var q = Assert.Single(delta.Queries);
Assert.Equal(30, q.Executions);
Assert.Equal(3_000, q.WorkerTimeUs);
Assert.Equal(6_000, q.ElapsedTimeUs);
Assert.Equal(600, q.LogicalReads);
Assert.Equal("SELECT 0xA", q.Text);
}
[Fact]
public void Plan_compiled_after_the_previous_collection_counts_in_full()
{
var before = Snapshot(T0, new DateTime(2026, 9, 20, 3, 0, 0));
var now = Snapshot(T0.AddMinutes(5), new DateTime(2026, 9, 20, 3, 5, 0),
Stat("0xB", 4, 900, new DateTime(2026, 9, 20, 3, 2, 0), new DateTime(2026, 9, 20, 3, 4, 0)));
var q = Assert.Single(QueryStatsDelta.Compute(before, now)!.Queries);
Assert.Equal(4, q.Executions);
Assert.Equal(900, q.WorkerTimeUs);
}
[Fact]
public void Old_statement_seen_for_the_first_time_has_no_known_starting_point()
{
var before = Snapshot(T0, new DateTime(2026, 9, 20, 3, 0, 0));
var now = Snapshot(T0.AddMinutes(5), new DateTime(2026, 9, 20, 3, 5, 0),
Stat("0xC", 1_000_000, 9_000_000, new DateTime(2026, 9, 19, 8, 0, 0), new DateTime(2026, 9, 20, 3, 4, 0)));
Assert.Empty(QueryStatsDelta.Compute(before, now)!.Queries);
}
[Fact]
public void Recompiled_plan_starts_over_without_negative_numbers()
{
var oldPlan = new DateTime(2026, 9, 20, 1, 0, 0);
var before = Snapshot(T0, new DateTime(2026, 9, 20, 3, 0, 0), Stat("0xD", 500, 50_000, oldPlan, oldPlan.AddHours(1)));
// Evicted and compiled again at 03:03: smaller totals, newer creation time. Everything since 03:03 is new.
var recompiled = Snapshot(T0.AddMinutes(5), new DateTime(2026, 9, 20, 3, 5, 0),
Stat("0xD", 7, 700, new DateTime(2026, 9, 20, 3, 3, 0), new DateTime(2026, 9, 20, 3, 4, 0)));
var q = Assert.Single(QueryStatsDelta.Compute(before, recompiled)!.Queries);
Assert.Equal(7, q.Executions);
Assert.Equal(700, q.WorkerTimeUs);
// Counters went backwards but the oldest plan is from before: one of several plans was evicted. Skip, never negative.
var partlyEvicted = Snapshot(T0.AddMinutes(5), new DateTime(2026, 9, 20, 3, 5, 0),
Stat("0xD", 300, 30_000, oldPlan, new DateTime(2026, 9, 20, 3, 4, 0)));
Assert.Empty(QueryStatsDelta.Compute(before, partlyEvicted)!.Queries);
}
[Fact]
public void Restart_gives_no_delta_and_forgets_remembered_statements()
{
var before = Snapshot(T0, new DateTime(2026, 9, 20, 3, 0, 0), Stat("0xE", 1, 1, Start, Start));
var after = Snapshot(T0.AddMinutes(5), new DateTime(2026, 9, 20, 3, 5, 0)) with { SqlServerStartTime = Start.AddHours(3) };
Assert.Null(QueryStatsDelta.Compute(before, after));
Assert.Empty(QueryStatsDelta.Remember(before, [], after.SqlServerStartTime));
}
[Fact]
public void Statements_not_run_now_are_remembered_for_later_up_to_a_limit()
{
var created = new DateTime(2026, 9, 20, 1, 0, 0);
var previous = Snapshot(T0, new DateTime(2026, 9, 20, 3, 0, 0),
[.. Enumerable.Range(0, QueryStatsSnapshot.MaxRemembered + 10).Select(i => Stat("0x" + i, 1, 1, created, created.AddSeconds(i)))]);
var current = new[] { Stat("0xNEW", 1, 1, created, created.AddHours(2)) };
var kept = QueryStatsDelta.Remember(previous, current, Start);
Assert.Equal(QueryStatsSnapshot.MaxRemembered, kept.Count);
Assert.Equal("0xNEW", kept[0].Key);
Assert.DoesNotContain(kept, q => q.Key == "0x0"); // least recently run is forgotten first
}
[Fact]
public void Sql_server_2005_key_is_handle_and_offsets()
{
// The key is whatever the collector builds; on 2005 sql_handle:start:end, so the same batch's statements stay apart.
var created = new DateTime(2026, 9, 20, 1, 0, 0);
var first = "0x0200000012ab:0:120";
var second = "0x0200000012ab:122:-1";
var before = Snapshot(T0, new DateTime(2026, 9, 20, 3, 0, 0), Stat(first, 10, 100, created, created), Stat(second, 10, 100, created, created));
var now = Snapshot(T0.AddMinutes(5), new DateTime(2026, 9, 20, 3, 5, 0),
Stat(first, 12, 160, created, new DateTime(2026, 9, 20, 3, 1, 0)), Stat(second, 15, 400, created, new DateTime(2026, 9, 20, 3, 1, 0)));
var delta = QueryStatsDelta.Compute(before, now)!;
Assert.Equal(2, delta.Queries.Count);
Assert.Equal(second, delta.Queries[0].Key); // most CPU first
Assert.Equal(300, delta.Queries[0].WorkerTimeUs);
Assert.Equal(60, delta.Queries[1].WorkerTimeUs);
}
}