Index usage over time and never-updated statistics (8g, F)
The index collector now sums index reads and writes between collections
across restarts of SQL Server (and of the agent, by seeding from the
stored snapshot) and keeps them per day in insight.IndexUsageDaily
(migration 0015, HourlyDays). SQLI-IDX-002 judges an index once it has
been followed for 7 days instead of needing 7 days of uptime.
GET /api/v1/instances/{name}/index-usage lists usage per index.
SQLI-STAT-001 also reports statistics that were never updated on tables
with rows; statistic rows now come from the table's heap or clustered
index, as a never-updated statistic has no row count of its own.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
473a7d0be3
commit
6cc65e3c75
13 changed files with 330 additions and 17 deletions
|
|
@ -132,3 +132,12 @@ Hvert 5. minut læser agenten tempdb's plads (`tempdb.sys.dm_db_file_space_usage
|
||||||
| SQLI-TEMP-001 | Middel | Version store over `SQLInsight:Tempdb:VersionStoreMegabytes` (standard 1.024 MB), eller vokset i hver af de seneste 6 målinger med mindst 256 MB i alt |
|
| SQLI-TEMP-001 | Middel | Version store over `SQLInsight:Tempdb:VersionStoreMegabytes` (standard 1.024 MB), eller vokset i hver af de seneste 6 målinger med mindst 256 MB i alt |
|
||||||
| SQLI-TEMP-002 | Høj | Under 10 % fri plads, og ingen datafil kan vokse |
|
| SQLI-TEMP-002 | Høj | Under 10 % fri plads, og ingen datafil kan vokse |
|
||||||
| SQLI-TEMP-003 | Middel | En session har brugt over `SessionMegabytes` (standard 1.024 MB) tempdb i mindst 15 minutter |
|
| SQLI-TEMP-003 | Middel | En session har brugt over `SessionMegabytes` (standard 1.024 MB) tempdb i mindst 15 minutter |
|
||||||
|
|
||||||
|
## Indexbrug over tid
|
||||||
|
|
||||||
|
SQL Server nulstiller sine tal for indexbrug (`sys.dm_db_index_usage_stats`) ved hver genstart. SQLInsight lægger derfor brugen mellem målingerne sammen hen over genstarter (og agentens egne genstarter) og gemmer læsninger og skrivninger pr. index pr. dag i `insight.IndexUsageDaily` (i `HourlyDays`, standard 400 dage).
|
||||||
|
|
||||||
|
- **SQLI-IDX-002** (ubrugt index) vurderer et index, når det har været fulgt i mindst 7 dage – også hvis SQL Server er genstartet for nylig – og først når det ikke er læst i hele perioden.
|
||||||
|
- API: `GET /api/v1/instances/{navn}/index-usage?days=30` giver læsninger, skrivninger og antal dage med læsninger pr. index, mindst læste først.
|
||||||
|
|
||||||
|
**SQLI-STAT-001** melder nu også statistikker, der **aldrig** er opdateret (`STATS_DATE` er tom) på tabeller med rækker, fx statistikker oprettet med NORECOMPUTE på en tom tabel. De står øverst blandt fundene.
|
||||||
|
|
|
||||||
|
|
@ -258,6 +258,12 @@ public sealed partial class InstanceMonitor(
|
||||||
return await repository.GetSnapshotAsync<Core.Data.TempdbSnapshot>(instanceId, collector.Name, cancellationToken).ConfigureAwait(false);
|
return await repository.GetSnapshotAsync<Core.Data.TempdbSnapshot>(instanceId, collector.Name, cancellationToken).ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Index usage is summed across restarts of SQL Server and of the agent.
|
||||||
|
if (collector is Collectors.Databases.IndexCollector)
|
||||||
|
{
|
||||||
|
return await repository.GetSnapshotAsync<Core.Data.IndexSnapshot>(instanceId, collector.Name, cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
|
||||||
if (collector is not Collectors.Concurrency.DeadlockCollector)
|
if (collector is not Collectors.Concurrency.DeadlockCollector)
|
||||||
{
|
{
|
||||||
return null;
|
return null;
|
||||||
|
|
|
||||||
|
|
@ -56,7 +56,7 @@ public sealed class IndexCollector : Collector<IndexSnapshot>
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
return new IndexSnapshot { CapturedAt = context.Now, SqlServerStartTime = startTime, Databases = databases };
|
return IndexUsageTracking.Advance(previous, new IndexSnapshot { CapturedAt = context.Now, SqlServerStartTime = startTime, Databases = databases });
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async Task<IndexDatabase> ReadAsync(DbConnection connection, string database, bool filtered, CancellationToken ct)
|
private static async Task<IndexDatabase> ReadAsync(DbConnection connection, string database, bool filtered, CancellationToken ct)
|
||||||
|
|
@ -141,12 +141,14 @@ public sealed class IndexCollector : Collector<IndexSnapshot>
|
||||||
""";
|
""";
|
||||||
}
|
}
|
||||||
|
|
||||||
// sys.sysindexes.rowmodctr needs only metadata visibility, unlike sys.dm_db_stats_properties.
|
// sys.sysindexes.rowmodctr needs only metadata visibility, unlike sys.dm_db_stats_properties. Rows are the table's
|
||||||
|
// (heap or clustered index), so a statistic that was never updated (STATS_DATE NULL) still shows the rows it covers.
|
||||||
internal static string BuildStatisticsSql(string database)
|
internal static string BuildStatisticsSql(string database)
|
||||||
{
|
{
|
||||||
var db = Db.Identifier(database);
|
var db = Db.Identifier(database);
|
||||||
return $"""
|
return $"""
|
||||||
SELECT s.name, o.name, si.name, STATS_DATE(si.id, si.indid), si.rowcnt, si.rowmodctr
|
SELECT s.name, o.name, si.name, STATS_DATE(si.id, si.indid),
|
||||||
|
COALESCE((SELECT TOP 1 t.rowcnt FROM {db}.sys.sysindexes t WHERE t.id = si.id AND t.indid IN (0, 1)), si.rowcnt), si.rowmodctr
|
||||||
FROM {db}.sys.sysindexes si
|
FROM {db}.sys.sysindexes si
|
||||||
JOIN {db}.sys.objects o ON o.object_id = si.id AND o.type = 'U' AND o.is_ms_shipped = 0
|
JOIN {db}.sys.objects o ON o.object_id = si.id AND o.type = 'U' AND o.is_ms_shipped = 0
|
||||||
JOIN {db}.sys.schemas s ON s.schema_id = o.schema_id
|
JOIN {db}.sys.schemas s ON s.schema_id = o.schema_id
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,16 @@ public sealed record IndexSnapshot
|
||||||
public required IReadOnlyList<IndexDatabase> Databases { get; init; }
|
public required IReadOnlyList<IndexDatabase> Databases { get; init; }
|
||||||
|
|
||||||
public TimeSpan Uptime => CapturedAt.UtcDateTime - SqlServerStartTime;
|
public TimeSpan Uptime => CapturedAt.UtcDateTime - SqlServerStartTime;
|
||||||
|
|
||||||
|
/// <summary>Reads and writes per index since the previous collection, for the daily history.</summary>
|
||||||
|
public IReadOnlyList<IndexUsageDelta> UsageDeltas { get; init; } = [];
|
||||||
|
|
||||||
|
/// <summary>How long the usage of an index has been followed: across restarts, unlike <see cref="Uptime"/>.</summary>
|
||||||
|
public TimeSpan TrackedFor(IndexInfo index)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(index);
|
||||||
|
return index.TrackedSince is { } since ? CapturedAt - since : Uptime;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed record IndexDatabase
|
public sealed record IndexDatabase
|
||||||
|
|
@ -51,6 +61,17 @@ public sealed record IndexInfo
|
||||||
/// <summary>Heaps only: rows read through a forwarding pointer.</summary>
|
/// <summary>Heaps only: rows read through a forwarding pointer.</summary>
|
||||||
public long ForwardedFetches { get; init; }
|
public long ForwardedFetches { get; init; }
|
||||||
|
|
||||||
|
/// <summary>Reads summed across restarts of SQL Server since <see cref="TrackedSince"/> (index usage is reset at each restart).</summary>
|
||||||
|
public long TrackedReads { get; init; }
|
||||||
|
|
||||||
|
/// <summary>Writes summed the same way.</summary>
|
||||||
|
public long TrackedWrites { get; init; }
|
||||||
|
|
||||||
|
/// <summary>Since when SQLInsight has followed the usage (the restart before the first collection).</summary>
|
||||||
|
public DateTimeOffset? TrackedSince { get; init; }
|
||||||
|
|
||||||
|
public string Key => $"{Schema}001f{Table}001f{IndexId}";
|
||||||
|
|
||||||
public long Reads => UserSeeks + UserScans + UserLookups;
|
public long Reads => UserSeeks + UserScans + UserLookups;
|
||||||
public double SizeMb => UsedPages * 8.0 / 1024;
|
public double SizeMb => UsedPages * 8.0 / 1024;
|
||||||
public string QualifiedTable => $"{Schema}.{Table}";
|
public string QualifiedTable => $"{Schema}.{Table}";
|
||||||
|
|
@ -88,3 +109,46 @@ public sealed record StatisticInfo
|
||||||
/// <summary>Row modifications since the last update (sysindexes.rowmodctr; approximate).</summary>
|
/// <summary>Row modifications since the last update (sysindexes.rowmodctr; approximate).</summary>
|
||||||
public long Modifications { get; init; }
|
public long Modifications { get; init; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>An index's reads and writes since the previous collection (after a restart: since the restart).</summary>
|
||||||
|
public sealed record IndexUsageDelta(string Database, string Schema, string Table, string? Index, long Reads, long Writes);
|
||||||
|
|
||||||
|
public static class IndexUsageTracking
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Adds this collection's usage to the totals carried over from the previous one. After a restart the counters
|
||||||
|
/// started from zero again, so all of the current usage is new.
|
||||||
|
/// </summary>
|
||||||
|
public static IndexSnapshot Advance(IndexSnapshot? previous, IndexSnapshot current)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(current);
|
||||||
|
var restarted = previous is null || previous.SqlServerStartTime != current.SqlServerStartTime;
|
||||||
|
var start = new DateTimeOffset(DateTime.SpecifyKind(current.SqlServerStartTime, DateTimeKind.Utc));
|
||||||
|
var before = (previous?.Databases ?? [])
|
||||||
|
.SelectMany(d => d.Indexes.Select(i => (Key: d.Name + "\u001f" + i.Key, Index: i)))
|
||||||
|
.GroupBy(x => x.Key, StringComparer.OrdinalIgnoreCase)
|
||||||
|
.ToDictionary(g => g.Key, g => g.First().Index, StringComparer.OrdinalIgnoreCase);
|
||||||
|
var deltas = new List<IndexUsageDelta>();
|
||||||
|
var databases = current.Databases.Select(d => d with
|
||||||
|
{
|
||||||
|
Indexes = d.Indexes.Select(i =>
|
||||||
|
{
|
||||||
|
before.TryGetValue(d.Name + "\u001f" + i.Key, out var was);
|
||||||
|
var reads = restarted || was is null ? i.Reads : Math.Max(0, i.Reads - was.Reads);
|
||||||
|
var writes = restarted || was is null ? i.UserUpdates : Math.Max(0, i.UserUpdates - was.UserUpdates);
|
||||||
|
if (reads > 0 || writes > 0)
|
||||||
|
{
|
||||||
|
deltas.Add(new IndexUsageDelta(d.Name, i.Schema, i.Table, i.Name, reads, writes));
|
||||||
|
}
|
||||||
|
|
||||||
|
return i with
|
||||||
|
{
|
||||||
|
TrackedReads = (was?.TrackedReads ?? 0) + reads,
|
||||||
|
TrackedWrites = (was?.TrackedWrites ?? 0) + writes,
|
||||||
|
TrackedSince = was?.TrackedSince ?? start,
|
||||||
|
};
|
||||||
|
}).ToList(),
|
||||||
|
}).ToList();
|
||||||
|
return current with { Databases = databases, UsageDeltas = deltas };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
54
src/SQLInsight.Repository/InsightRepository.IndexUsage.cs
Normal file
54
src/SQLInsight.Repository/InsightRepository.IndexUsage.cs
Normal file
|
|
@ -0,0 +1,54 @@
|
||||||
|
using System.Text.Json;
|
||||||
|
using Dapper;
|
||||||
|
using Microsoft.Data.SqlClient;
|
||||||
|
using SQLInsight.Core.Data;
|
||||||
|
|
||||||
|
namespace SQLInsight.Repository;
|
||||||
|
|
||||||
|
/// <summary>An index's reads and writes summed over a period, and on how many days it was used at all.</summary>
|
||||||
|
public sealed record IndexUsageTotal(string DatabaseName, string SchemaName, string TableName, string IndexName, long Reads, long Writes, int Days, DateTime FirstDay, DateTime LastDay);
|
||||||
|
|
||||||
|
public sealed partial class InsightRepository
|
||||||
|
{
|
||||||
|
private static async Task SaveIndexUsageAsync(SqlConnection connection, int instanceId, IndexSnapshot snapshot)
|
||||||
|
{
|
||||||
|
if (snapshot.UsageDeltas.Count == 0)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await connection.ExecuteAsync("""
|
||||||
|
MERGE insight.IndexUsageDaily AS t
|
||||||
|
USING (SELECT DatabaseName, SchemaName, TableName, IndexName, SUM(Reads) AS Reads, SUM(Writes) AS Writes
|
||||||
|
FROM OPENJSON(@json) WITH (DatabaseName nvarchar(128) '$.Database', SchemaName nvarchar(128) '$.Schema', TableName nvarchar(128) '$.Table',
|
||||||
|
IndexName nvarchar(128) '$.Index', Reads bigint, Writes bigint)
|
||||||
|
GROUP BY DatabaseName, SchemaName, TableName, IndexName) AS s
|
||||||
|
ON t.InstanceId = @instanceId AND t.[Day] = @day AND t.DatabaseName = s.DatabaseName AND t.SchemaName = s.SchemaName
|
||||||
|
AND t.TableName = s.TableName AND t.IndexName = s.IndexName
|
||||||
|
WHEN MATCHED THEN UPDATE SET Reads = t.Reads + s.Reads, Writes = t.Writes + s.Writes
|
||||||
|
WHEN NOT MATCHED THEN INSERT (InstanceId, [Day], DatabaseName, SchemaName, TableName, IndexName, Reads, Writes)
|
||||||
|
VALUES (@instanceId, @day, s.DatabaseName, s.SchemaName, s.TableName, s.IndexName, s.Reads, s.Writes);
|
||||||
|
""", new
|
||||||
|
{
|
||||||
|
instanceId,
|
||||||
|
day = snapshot.CapturedAt.UtcDateTime.Date,
|
||||||
|
json = JsonSerializer.Serialize(snapshot.UsageDeltas.Select(d => d with { Index = d.Index ?? "(heap)" }), Json),
|
||||||
|
}).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Index usage summed over the days in [from, to], least read first.</summary>
|
||||||
|
public async Task<IReadOnlyList<IndexUsageTotal>> GetIndexUsageAsync(int instanceId, DateTimeOffset from, DateTimeOffset to, int top = 500,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
var rows = await connection.QueryAsync<IndexUsageTotal>("""
|
||||||
|
SELECT TOP (@top) DatabaseName, SchemaName, TableName, IndexName, SUM(Reads) AS Reads, SUM(Writes) AS Writes,
|
||||||
|
SUM(CASE WHEN Reads > 0 THEN 1 ELSE 0 END) AS Days, MIN([Day]) AS FirstDay, MAX([Day]) AS LastDay
|
||||||
|
FROM insight.IndexUsageDaily
|
||||||
|
WHERE InstanceId = @instanceId AND [Day] >= @from AND [Day] <= @to
|
||||||
|
GROUP BY DatabaseName, SchemaName, TableName, IndexName
|
||||||
|
ORDER BY SUM(Reads), SUM(Writes) DESC;
|
||||||
|
""", new { instanceId, from = from.UtcDateTime.Date, to = to.UtcDateTime.Date, top }).ConfigureAwait(false);
|
||||||
|
return rows.ToList();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -89,6 +89,7 @@ public sealed partial class InsightRepository
|
||||||
await DeleteInBatchesAsync(connection, "insight.MetricHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
|
await DeleteInBatchesAsync(connection, "insight.MetricHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
|
||||||
await DeleteInBatchesAsync(connection, "insight.WaitHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
|
await DeleteInBatchesAsync(connection, "insight.WaitHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
|
||||||
await DeleteInBatchesAsync(connection, "insight.DiskHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
|
await DeleteInBatchesAsync(connection, "insight.DiskHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
|
||||||
|
await DeleteInBatchesAsync(connection, "insight.IndexUsageDaily", "[Day] < @cutoff", hourly).ConfigureAwait(false);
|
||||||
await DeleteInBatchesAsync(connection, "insight.BlockingEvents", "SampledAt < @cutoff", events).ConfigureAwait(false);
|
await DeleteInBatchesAsync(connection, "insight.BlockingEvents", "SampledAt < @cutoff", events).ConfigureAwait(false);
|
||||||
await DeleteInBatchesAsync(connection, "insight.Deadlocks", "OccurredAt < @cutoff", events).ConfigureAwait(false);
|
await DeleteInBatchesAsync(connection, "insight.Deadlocks", "OccurredAt < @cutoff", events).ConfigureAwait(false);
|
||||||
await DeleteInBatchesAsync(connection, "insight.ErrorLogEvents", "LastAt < @cutoff", events).ConfigureAwait(false);
|
await DeleteInBatchesAsync(connection, "insight.ErrorLogEvents", "LastAt < @cutoff", events).ConfigureAwait(false);
|
||||||
|
|
|
||||||
|
|
@ -280,6 +280,10 @@ public sealed partial class InsightRepository(string connectionString)
|
||||||
]).ConfigureAwait(false);
|
]).ConfigureAwait(false);
|
||||||
await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false);
|
await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false);
|
||||||
return;
|
return;
|
||||||
|
case IndexSnapshot indexes:
|
||||||
|
await SaveIndexUsageAsync(connection, instanceId, indexes).ConfigureAwait(false);
|
||||||
|
await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false);
|
||||||
|
return;
|
||||||
case PrivilegeSnapshot privileges:
|
case PrivilegeSnapshot privileges:
|
||||||
await SavePrivilegeChangesAsync(connection, instanceId, privileges.Changes).ConfigureAwait(false);
|
await SavePrivilegeChangesAsync(connection, instanceId, privileges.Changes).ConfigureAwait(false);
|
||||||
await SaveLatePrivilegeAuthorsAsync(connection, instanceId, privileges.RecentChanges).ConfigureAwait(false);
|
await SaveLatePrivilegeAuthorsAsync(connection, instanceId, privileges.RecentChanges).ConfigureAwait(false);
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
/* Index reads and writes per day (deltas of sys.dm_db_index_usage_stats between collections), so unused indexes can be
|
||||||
|
judged over time, across restarts of SQL Server. Kept as long as the hourly history (HourlyDays). */
|
||||||
|
CREATE TABLE insight.IndexUsageDaily
|
||||||
|
(
|
||||||
|
InstanceId int NOT NULL,
|
||||||
|
[Day] date NOT NULL,
|
||||||
|
DatabaseName nvarchar(128) NOT NULL,
|
||||||
|
SchemaName nvarchar(128) NOT NULL,
|
||||||
|
TableName nvarchar(128) NOT NULL,
|
||||||
|
IndexName nvarchar(128) NOT NULL,
|
||||||
|
Reads bigint NOT NULL,
|
||||||
|
Writes bigint NOT NULL,
|
||||||
|
CONSTRAINT PK_IndexUsageDaily PRIMARY KEY (InstanceId, [Day], DatabaseName, SchemaName, TableName, IndexName)
|
||||||
|
);
|
||||||
|
|
@ -173,25 +173,33 @@ public sealed class UnusedIndexRule : Rule
|
||||||
public override RuleEvaluation Evaluate(RuleContext context)
|
public override RuleEvaluation Evaluate(RuleContext context)
|
||||||
{
|
{
|
||||||
ArgumentNullException.ThrowIfNull(context);
|
ArgumentNullException.ThrowIfNull(context);
|
||||||
var snapshot = IndexText.RequireUptime(context);
|
// Usage is followed across restarts, so an index can be judged once it has been followed for a week.
|
||||||
|
var snapshot = IndexText.Snapshot(context);
|
||||||
|
var followed = snapshot.Databases.SelectMany(db => db.Indexes).Any(i => snapshot.TrackedFor(i) >= IndexText.MinUptimeForUsage);
|
||||||
|
if (!followed)
|
||||||
|
{
|
||||||
|
IndexText.RequireUptime(context);
|
||||||
|
}
|
||||||
|
|
||||||
return Result(snapshot.Databases
|
return Result(snapshot.Databases
|
||||||
.SelectMany(db => db.Indexes.Select(i => (Db: db.Name, Index: i)))
|
.SelectMany(db => db.Indexes.Select(i => (Db: db.Name, Index: i)))
|
||||||
.Where(x => x.Index.Type == "NONCLUSTERED" && !x.Index.IsPrimaryKey && !x.Index.IsUniqueConstraint && !x.Index.IsUnique
|
.Where(x => x.Index.Type == "NONCLUSTERED" && !x.Index.IsPrimaryKey && !x.Index.IsUniqueConstraint && !x.Index.IsUnique
|
||||||
&& !x.Index.IsDisabled && x.Index.Reads == 0 && (x.Index.UserUpdates >= 1000 || x.Index.SizeMb >= 100))
|
&& !x.Index.IsDisabled && snapshot.TrackedFor(x.Index) >= IndexText.MinUptimeForUsage
|
||||||
.OrderByDescending(x => x.Index.UserUpdates + x.Index.UsedPages)
|
&& x.Index.Reads == 0 && x.Index.TrackedReads == 0 && (Math.Max(x.Index.UserUpdates, x.Index.TrackedWrites) >= 1000 || x.Index.SizeMb >= 100))
|
||||||
|
.OrderByDescending(x => Math.Max(x.Index.UserUpdates, x.Index.TrackedWrites) + x.Index.UsedPages)
|
||||||
.Take(IndexText.MaxFindingsPerRule)
|
.Take(IndexText.MaxFindingsPerRule)
|
||||||
.Select(x => CreateFinding(
|
.Select(x => CreateFinding(
|
||||||
x.Index.SizeMb >= 1024 ? Severity.Medium : Severity.Low,
|
x.Index.SizeMb >= 1024 ? Severity.Medium : Severity.Low,
|
||||||
$"{x.Db}.{x.Index.QualifiedTable}: index {x.Index.Name} er ikke læst i {Format.Age(snapshot.Uptime)}",
|
$"{x.Db}.{x.Index.QualifiedTable}: index {x.Index.Name} er ikke læst i {Format.Age(snapshot.TrackedFor(x.Index))}",
|
||||||
"Indexet opdateres ved hver INSERT, UPDATE og DELETE på tabellen, men ingen forespørgsel har læst det siden seneste "
|
"Indexet opdateres ved hver INSERT, UPDATE og DELETE på tabellen, men ingen forespørgsel har læst det, så længe SQLInsight har fulgt det "
|
||||||
+ "genstart. Det koster skrivninger, transaktionslog, backup-størrelse og vedligeholdelsestid.",
|
+ "(også hen over genstarter af SQL Server). Det koster skrivninger, transaktionslog, backup-størrelse og vedligeholdelsestid.",
|
||||||
"Deaktivér indexet (ALTER INDEX … DISABLE) i stedet for at slette det, så definitionen bevares, og vent en hel "
|
"Deaktivér indexet (ALTER INDEX … DISABLE) i stedet for at slette det, så definitionen bevares, og vent en hel "
|
||||||
+ "forretningscyklus (måneds- og kvartalsafslutning, rapporter). Er det stadig ubrugt, kan det slettes. Genopret med ALTER INDEX … REBUILD.",
|
+ "forretningscyklus (måneds- og kvartalsafslutning, rapporter). Er det stadig ubrugt, kan det slettes. Genopret med ALTER INDEX … REBUILD.",
|
||||||
[
|
[
|
||||||
new Evidence("Index", $"{x.Index.Name} ({IndexText.Columns(x.Index.KeyColumns)})"),
|
new Evidence("Index", $"{x.Index.Name} ({IndexText.Columns(x.Index.KeyColumns)})"),
|
||||||
new Evidence("Skrivninger / læsninger", $"{Format.Number(x.Index.UserUpdates)} / 0"),
|
new Evidence("Skrivninger / læsninger", $"{Format.Number(Math.Max(x.Index.UserUpdates, x.Index.TrackedWrites))} / 0"),
|
||||||
new Evidence("Størrelse", $"{Format.Number(x.Index.SizeMb)} MB"),
|
new Evidence("Størrelse", $"{Format.Number(x.Index.SizeMb)} MB"),
|
||||||
new Evidence("Periode", $"siden genstart for {Format.Age(snapshot.Uptime)} siden"),
|
new Evidence("Periode", $"fulgt i {Format.Age(snapshot.TrackedFor(x.Index))} (seneste genstart for {Format.Age(snapshot.Uptime)} siden)"),
|
||||||
],
|
],
|
||||||
subject: $"index: {x.Db}.{x.Index.QualifiedTable}.{x.Index.Name}",
|
subject: $"index: {x.Db}.{x.Index.QualifiedTable}.{x.Index.Name}",
|
||||||
fixScript: $"{IndexText.Use(x.Db)}\nALTER INDEX {IndexText.Quote(x.Index.Name!)} ON {IndexText.Table(x.Index)} DISABLE;",
|
fixScript: $"{IndexText.Use(x.Db)}\nALTER INDEX {IndexText.Quote(x.Index.Name!)} ON {IndexText.Table(x.Index)} DISABLE;",
|
||||||
|
|
@ -330,12 +338,15 @@ public sealed class StaleStatisticsRule : Rule
|
||||||
{
|
{
|
||||||
ArgumentNullException.ThrowIfNull(context);
|
ArgumentNullException.ThrowIfNull(context);
|
||||||
return Result(IndexText.Snapshot(context).Databases
|
return Result(IndexText.Snapshot(context).Databases
|
||||||
.SelectMany(db => db.Statistics.Where(s => s.Rows >= MinRows && s.Modifications >= s.Rows * MinModifiedShare).Select(s => (Db: db.Name, Stat: s)))
|
.SelectMany(db => db.Statistics.Where(s => s.LastUpdated is null && s.Rows > 0 || s.Rows >= MinRows && s.Modifications >= s.Rows * MinModifiedShare)
|
||||||
.OrderByDescending(x => x.Stat.Modifications)
|
.Select(s => (Db: db.Name, Stat: s)))
|
||||||
|
.OrderByDescending(x => x.Stat.LastUpdated is null).ThenByDescending(x => x.Stat.Modifications)
|
||||||
.Take(IndexText.MaxFindingsPerRule)
|
.Take(IndexText.MaxFindingsPerRule)
|
||||||
.Select(x => CreateFinding(
|
.Select(x => CreateFinding(
|
||||||
Severity.Low,
|
Severity.Low,
|
||||||
$"{x.Db}.{x.Stat.Schema}.{x.Stat.Table}: statistikken {x.Stat.Name} er {Format.Number(100.0 * x.Stat.Modifications / x.Stat.Rows)} % ændret",
|
x.Stat.LastUpdated is null
|
||||||
|
? $"{x.Db}.{x.Stat.Schema}.{x.Stat.Table}: statistikken {x.Stat.Name} er aldrig opdateret ({Format.Number(x.Stat.Rows)} rækker)"
|
||||||
|
: $"{x.Db}.{x.Stat.Schema}.{x.Stat.Table}: statistikken {x.Stat.Name} er {Format.Number(100.0 * x.Stat.Modifications / x.Stat.Rows)} % ændret",
|
||||||
"Optimizeren vælger planer ud fra statistikkernes billede af data. Når en stor del af rækkerne er ændret siden seneste "
|
"Optimizeren vælger planer ud fra statistikkernes billede af data. Når en stor del af rækkerne er ændret siden seneste "
|
||||||
+ "opdatering, rammer estimaterne ved siden af: for små memory grants, forkerte join-typer og spild til tempdb.",
|
+ "opdatering, rammer estimaterne ved siden af: for små memory grants, forkerte join-typer og spild til tempdb.",
|
||||||
"Opdatér statistikkerne som en del af det faste vedligehold (fx Ola Hallengrens IndexOptimize med @UpdateStatistics = 'ALL' og "
|
"Opdatér statistikkerne som en del af det faste vedligehold (fx Ola Hallengrens IndexOptimize med @UpdateStatistics = 'ALL' og "
|
||||||
|
|
|
||||||
|
|
@ -49,6 +49,7 @@ public static class ApiV1Endpoints
|
||||||
api.MapGet("/instances/{name}/activity", ActivityAsync).WithSummary("Åbne transaktioner og lange sætninger lige nu / Open transactions and long-running statements");
|
api.MapGet("/instances/{name}/activity", ActivityAsync).WithSummary("Åbne transaktioner og lange sætninger lige nu / Open transactions and long-running statements");
|
||||||
api.MapGet("/instances/{name}/tempdb", TempdbAsync).WithSummary("tempdb og version store lige nu / tempdb and the version store");
|
api.MapGet("/instances/{name}/tempdb", TempdbAsync).WithSummary("tempdb og version store lige nu / tempdb and the version store");
|
||||||
api.MapGet("/instances/{name}/replication", ReplicationAsync).WithSummary("Replikering set fra distributøren / Replication from the distributor");
|
api.MapGet("/instances/{name}/replication", ReplicationAsync).WithSummary("Replikering set fra distributøren / Replication from the distributor");
|
||||||
|
api.MapGet("/instances/{name}/index-usage", IndexUsageAsync).WithSummary("Indexbrug over tid, på tværs af genstarter / Index usage over time (days)");
|
||||||
api.MapGet("/instances/{name}/error-log", ErrorLogAsync).WithSummary("Vigtige hændelser i fejlloggen / Important error log events (days)");
|
api.MapGet("/instances/{name}/error-log", ErrorLogAsync).WithSummary("Vigtige hændelser i fejlloggen / Important error log events (days)");
|
||||||
api.MapGet("/instances/{name}/wait-types/{waitType}", WaitTypeAsync).WithSummary("En ventetype: forklaring, udvikling og sætninger / One wait type");
|
api.MapGet("/instances/{name}/wait-types/{waitType}", WaitTypeAsync).WithSummary("En ventetype: forklaring, udvikling og sætninger / One wait type");
|
||||||
}
|
}
|
||||||
|
|
@ -576,6 +577,19 @@ public static class ApiV1Endpoints
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static async Task<IResult> IndexUsageAsync(string name, InsightRepository repository, CancellationToken ct, int? days = null)
|
||||||
|
{
|
||||||
|
if (await repository.GetInstanceAsync(name, ct).ConfigureAwait(false) is not { } instance)
|
||||||
|
{
|
||||||
|
return Results.NotFound();
|
||||||
|
}
|
||||||
|
|
||||||
|
var span = Math.Clamp(days ?? 30, 1, 400);
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
var usage = await repository.GetIndexUsageAsync(instance.InstanceId, now.AddDays(-span), now, 1000, ct).ConfigureAwait(false);
|
||||||
|
return Results.Ok(new { Instance = instance.Name, Days = span, Indexes = usage });
|
||||||
|
}
|
||||||
|
|
||||||
private static async Task<IResult> ErrorLogAsync(string name, InsightRepository repository, IOptions<WebOptions> options, CancellationToken ct, int? days = null)
|
private static async Task<IResult> ErrorLogAsync(string name, InsightRepository repository, IOptions<WebOptions> options, CancellationToken ct, int? days = null)
|
||||||
{
|
{
|
||||||
if (await repository.GetInstanceAsync(name, ct).ConfigureAwait(false) is not { } instance)
|
if (await repository.GetInstanceAsync(name, ct).ConfigureAwait(false) is not { } instance)
|
||||||
|
|
|
||||||
33
tests/SQLInsight.Core.Tests/IndexUsageTests.cs
Normal file
33
tests/SQLInsight.Core.Tests/IndexUsageTests.cs
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
using SQLInsight.Core.Data;
|
||||||
|
|
||||||
|
namespace SQLInsight.Core.Tests;
|
||||||
|
|
||||||
|
public class IndexUsageTests
|
||||||
|
{
|
||||||
|
private static readonly DateTime Start = new(2026, 9, 1, 6, 0, 0, DateTimeKind.Utc);
|
||||||
|
|
||||||
|
private static IndexSnapshot Snapshot(DateTime start, DateTimeOffset at, long reads, long writes) => new()
|
||||||
|
{
|
||||||
|
CapturedAt = at,
|
||||||
|
SqlServerStartTime = start,
|
||||||
|
Databases = [new IndexDatabase { Name = "AX", Indexes = [new IndexInfo { Schema = "dbo", Table = "INVENTTRANS", IndexId = 5, Name = "I_177ITEMIDX", Type = "NONCLUSTERED", UserSeeks = reads, UserUpdates = writes }] }],
|
||||||
|
};
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Usage_is_summed_across_restarts_and_tracked_from_the_first_restart_seen()
|
||||||
|
{
|
||||||
|
var first = IndexUsageTracking.Advance(null, Snapshot(Start, new DateTimeOffset(Start.AddDays(1)), 10, 100));
|
||||||
|
var second = IndexUsageTracking.Advance(first, Snapshot(Start, new DateTimeOffset(Start.AddDays(2)), 15, 300));
|
||||||
|
var restart = Start.AddDays(3);
|
||||||
|
var afterRestart = IndexUsageTracking.Advance(second, Snapshot(restart, new DateTimeOffset(restart.AddHours(6)), 2, 50));
|
||||||
|
|
||||||
|
var index = afterRestart.Databases[0].Indexes[0];
|
||||||
|
Assert.Equal((17L, 350L), (index.TrackedReads, index.TrackedWrites));
|
||||||
|
Assert.Equal(new DateTimeOffset(Start), index.TrackedSince);
|
||||||
|
Assert.Equal(TimeSpan.FromDays(3.25), afterRestart.TrackedFor(index));
|
||||||
|
Assert.Equal(TimeSpan.FromHours(6), afterRestart.Uptime);
|
||||||
|
var delta = Assert.Single(second.UsageDeltas);
|
||||||
|
Assert.Equal((5L, 200L), (delta.Reads, delta.Writes));
|
||||||
|
Assert.Equal((2L, 50L), (afterRestart.UsageDeltas[0].Reads, afterRestart.UsageDeltas[0].Writes)); // counters started over
|
||||||
|
}
|
||||||
|
}
|
||||||
66
tests/SQLInsight.IntegrationTests/IndexUsageTests.cs
Normal file
66
tests/SQLInsight.IntegrationTests/IndexUsageTests.cs
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
using Microsoft.AspNetCore.Hosting;
|
||||||
|
using Microsoft.AspNetCore.Mvc.Testing;
|
||||||
|
using SQLInsight.Collectors.Databases;
|
||||||
|
using SQLInsight.Core.Data;
|
||||||
|
using SQLInsight.Core.Instances;
|
||||||
|
using SQLInsight.Repository;
|
||||||
|
|
||||||
|
namespace SQLInsight.IntegrationTests;
|
||||||
|
|
||||||
|
/// <summary>Index usage followed over time, and statistics that were never updated.</summary>
|
||||||
|
[Collection(SqlServerTestGroup.Name)]
|
||||||
|
[Trait("Category", "Integration")]
|
||||||
|
public class IndexUsageTests(SqlServerFixture sql)
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task Usage_between_collections_is_stored_per_day_and_never_updated_statistics_are_seen()
|
||||||
|
{
|
||||||
|
var table = "Usage_" + Guid.NewGuid().ToString("N")[..6];
|
||||||
|
await sql.ExecuteAsSaAsync($"""
|
||||||
|
USE [{SqlServerFixture.TestDatabase}];
|
||||||
|
CREATE TABLE dbo.{table} (Id int IDENTITY PRIMARY KEY, Code int NOT NULL, Name nvarchar(20));
|
||||||
|
CREATE INDEX IX_{table}_Code ON dbo.{table} (Code);
|
||||||
|
CREATE STATISTICS ST_{table}_Name ON dbo.{table} (Name) WITH NORECOMPUTE;
|
||||||
|
INSERT dbo.{table} (Code, Name) SELECT TOP (500) object_id % 100, N'x' FROM sys.all_objects;
|
||||||
|
""");
|
||||||
|
var database = "SQLInsightIdx_" + Guid.NewGuid().ToString("N")[..8];
|
||||||
|
await sql.ExecuteAsSaAsync($"CREATE DATABASE [{database}];");
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using var session = await AgentSession.OpenAsync(sql);
|
||||||
|
var first = await session.CollectAsync<IndexSnapshot>(new IndexCollector());
|
||||||
|
await sql.ExecuteAsSaAsync($"USE [{SqlServerFixture.TestDatabase}]; SELECT COUNT(*) FROM dbo.{table} WITH (INDEX(IX_{table}_Code)) WHERE Code = 7;");
|
||||||
|
var second = await session.CollectAsync(new IndexCollector(), first);
|
||||||
|
|
||||||
|
var index = second.Databases.Single(d => d.Name == SqlServerFixture.TestDatabase).Indexes.Single(i => i.Name == $"IX_{table}_Code");
|
||||||
|
Assert.True(index.TrackedReads >= 1);
|
||||||
|
Assert.NotNull(index.TrackedSince);
|
||||||
|
var delta = Assert.Single(second.UsageDeltas, d => d.Index == $"IX_{table}_Code");
|
||||||
|
Assert.True(delta.Reads >= 1);
|
||||||
|
var never = second.Databases.Single(d => d.Name == SqlServerFixture.TestDatabase).Statistics.Single(s => s.Name == $"ST_{table}_Name");
|
||||||
|
Assert.Null(never.LastUpdated);
|
||||||
|
Assert.Equal(500, never.Rows); // the table's rows, not the statistic's
|
||||||
|
|
||||||
|
var repository = new InsightRepository(sql.DatabaseConnectionString(database));
|
||||||
|
await repository.MigrateAsync();
|
||||||
|
var id = await repository.UpsertInstanceAsync("prod-sql01", session.Instance, "SqlClient", false, DateTimeOffset.UtcNow);
|
||||||
|
await repository.SaveCollectedAsync(id, "indexes", second);
|
||||||
|
var usage = await repository.GetIndexUsageAsync(id, DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow);
|
||||||
|
Assert.Contains(usage, u => u.IndexName == $"IX_{table}_Code" && u.Reads >= 1 && u.Days == 1);
|
||||||
|
|
||||||
|
await using var factory = new WebApplicationFactory<Program>().WithWebHostBuilder(b =>
|
||||||
|
{
|
||||||
|
b.UseEnvironment("Development");
|
||||||
|
b.UseSetting("SQLInsight:Repository:ConnectionString", sql.DatabaseConnectionString(database));
|
||||||
|
b.UseSetting("SQLInsight:Web:Authentication", "None");
|
||||||
|
});
|
||||||
|
var api = await factory.CreateClient().GetStringAsync("/api/v1/instances/prod-sql01/index-usage?days=7");
|
||||||
|
Assert.Contains($"IX_{table}_Code", api, StringComparison.Ordinal);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await sql.ExecuteAsSaAsync($"ALTER DATABASE [{database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{database}];");
|
||||||
|
await sql.ExecuteAsSaAsync($"USE [{SqlServerFixture.TestDatabase}]; DROP TABLE dbo.{table};");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -173,9 +173,10 @@ public class IndexRuleTests
|
||||||
Name = "Sales",
|
Name = "Sales",
|
||||||
Statistics =
|
Statistics =
|
||||||
[
|
[
|
||||||
new StatisticInfo { Schema = "dbo", Table = "Orders", Name = "IX_Orders_Date", Rows = 1_000_000, Modifications = 400_000 },
|
new StatisticInfo { Schema = "dbo", Table = "Orders", Name = "IX_Orders_Date", Rows = 1_000_000, Modifications = 400_000, LastUpdated = Now.UtcDateTime.AddDays(-9) },
|
||||||
new StatisticInfo { Schema = "dbo", Table = "Small", Name = "PK_Small", Rows = 500, Modifications = 500 },
|
new StatisticInfo { Schema = "dbo", Table = "Small", Name = "PK_Small", Rows = 500, Modifications = 500, LastUpdated = Now.UtcDateTime.AddDays(-9) },
|
||||||
new StatisticInfo { Schema = "dbo", Table = "Stable", Name = "PK_Stable", Rows = 1_000_000, Modifications = 1_000 },
|
new StatisticInfo { Schema = "dbo", Table = "Stable", Name = "PK_Stable", Rows = 1_000_000, Modifications = 1_000, LastUpdated = Now.UtcDateTime.AddDays(-9) },
|
||||||
|
new StatisticInfo { Schema = "dbo", Table = "Empty", Name = "_WA_Sys_Empty", Rows = 0, Modifications = 0 },
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -184,6 +185,40 @@ public class IndexRuleTests
|
||||||
Assert.Equal("statistic: Sales.dbo.Orders.IX_Orders_Date", finding.Subject);
|
Assert.Equal("statistic: Sales.dbo.Orders.IX_Orders_Date", finding.Subject);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Statistics_never_updated_on_a_table_with_rows_are_found_first()
|
||||||
|
{
|
||||||
|
var db = new IndexDatabase
|
||||||
|
{
|
||||||
|
Name = "AX",
|
||||||
|
Statistics =
|
||||||
|
[
|
||||||
|
new StatisticInfo { Schema = "dbo", Table = "Orders", Name = "IX_Orders_Date", Rows = 1_000_000, Modifications = 400_000, LastUpdated = Now.UtcDateTime.AddDays(-9) },
|
||||||
|
new StatisticInfo { Schema = "dbo", Table = "CUSTTRANS", Name = "_WA_Sys_00000003", Rows = 120 },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
var findings = Run(new StaleStatisticsRule(), Indexes(Month, db)).Findings;
|
||||||
|
|
||||||
|
Assert.Equal(2, findings.Count);
|
||||||
|
Assert.Contains("aldrig opdateret", findings[0].Title, StringComparison.Ordinal);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Unused_index_is_judged_on_usage_followed_across_restarts()
|
||||||
|
{
|
||||||
|
var index = Index(2, "IX_Unused", "NONCLUSTERED", ["Status"], reads: 0, updates: 50_000) with
|
||||||
|
{
|
||||||
|
TrackedReads = 0, TrackedWrites = 900_000, TrackedSince = Now.AddDays(-40),
|
||||||
|
};
|
||||||
|
var db = new IndexDatabase { Name = "Sales", Indexes = [index] };
|
||||||
|
|
||||||
|
var finding = Assert.Single(Run(new UnusedIndexRule(), Indexes(TimeSpan.FromDays(1), db)).Findings); // restarted yesterday
|
||||||
|
|
||||||
|
Assert.Contains("40 dage", finding.Title, StringComparison.Ordinal);
|
||||||
|
Assert.Equal(RuleOutcome.Passed, Run(new UnusedIndexRule(), Indexes(TimeSpan.FromDays(1), db with { Indexes = [index with { TrackedReads = 3 }] })).Outcome);
|
||||||
|
}
|
||||||
[Fact]
|
[Fact]
|
||||||
public void Auto_statistics_off()
|
public void Auto_statistics_off()
|
||||||
{
|
{
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue