diff --git a/docs/analyse.md b/docs/analyse.md index 2d0342a..af42a97 100644 --- a/docs/analyse.md +++ b/docs/analyse.md @@ -132,3 +132,12 @@ Hvert 5. minut læser agenten tempdb's plads (`tempdb.sys.dm_db_file_space_usage | SQLI-TEMP-001 | Middel | Version store over `SQLInsight:Tempdb:VersionStoreMegabytes` (standard 1.024 MB), eller vokset i hver af de seneste 6 målinger med mindst 256 MB i alt | | SQLI-TEMP-002 | Høj | Under 10 % fri plads, og ingen datafil kan vokse | | SQLI-TEMP-003 | Middel | En session har brugt over `SessionMegabytes` (standard 1.024 MB) tempdb i mindst 15 minutter | + +## Indexbrug over tid + +SQL Server nulstiller sine tal for indexbrug (`sys.dm_db_index_usage_stats`) ved hver genstart. SQLInsight lægger derfor brugen mellem målingerne sammen hen over genstarter (og agentens egne genstarter) og gemmer læsninger og skrivninger pr. index pr. dag i `insight.IndexUsageDaily` (i `HourlyDays`, standard 400 dage). + +- **SQLI-IDX-002** (ubrugt index) vurderer et index, når det har været fulgt i mindst 7 dage – også hvis SQL Server er genstartet for nylig – og først når det ikke er læst i hele perioden. +- API: `GET /api/v1/instances/{navn}/index-usage?days=30` giver læsninger, skrivninger og antal dage med læsninger pr. index, mindst læste først. + +**SQLI-STAT-001** melder nu også statistikker, der **aldrig** er opdateret (`STATS_DATE` er tom) på tabeller med rækker, fx statistikker oprettet med NORECOMPUTE på en tom tabel. De står øverst blandt fundene. diff --git a/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs b/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs index e20ded0..f9c254e 100644 --- a/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs +++ b/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs @@ -258,6 +258,12 @@ public sealed partial class InstanceMonitor( return await repository.GetSnapshotAsync(instanceId, collector.Name, cancellationToken).ConfigureAwait(false); } + // Index usage is summed across restarts of SQL Server and of the agent. + if (collector is Collectors.Databases.IndexCollector) + { + return await repository.GetSnapshotAsync(instanceId, collector.Name, cancellationToken).ConfigureAwait(false); + } + if (collector is not Collectors.Concurrency.DeadlockCollector) { return null; diff --git a/src/SQLInsight.Collectors/Databases/IndexCollector.cs b/src/SQLInsight.Collectors/Databases/IndexCollector.cs index 432c155..632ef6f 100644 --- a/src/SQLInsight.Collectors/Databases/IndexCollector.cs +++ b/src/SQLInsight.Collectors/Databases/IndexCollector.cs @@ -56,7 +56,7 @@ public sealed class IndexCollector : Collector }); } - return new IndexSnapshot { CapturedAt = context.Now, SqlServerStartTime = startTime, Databases = databases }; + return IndexUsageTracking.Advance(previous, new IndexSnapshot { CapturedAt = context.Now, SqlServerStartTime = startTime, Databases = databases }); } private static async Task ReadAsync(DbConnection connection, string database, bool filtered, CancellationToken ct) @@ -141,12 +141,14 @@ public sealed class IndexCollector : Collector """; } - // sys.sysindexes.rowmodctr needs only metadata visibility, unlike sys.dm_db_stats_properties. + // sys.sysindexes.rowmodctr needs only metadata visibility, unlike sys.dm_db_stats_properties. Rows are the table's + // (heap or clustered index), so a statistic that was never updated (STATS_DATE NULL) still shows the rows it covers. internal static string BuildStatisticsSql(string database) { var db = Db.Identifier(database); return $""" - SELECT s.name, o.name, si.name, STATS_DATE(si.id, si.indid), si.rowcnt, si.rowmodctr + SELECT s.name, o.name, si.name, STATS_DATE(si.id, si.indid), + COALESCE((SELECT TOP 1 t.rowcnt FROM {db}.sys.sysindexes t WHERE t.id = si.id AND t.indid IN (0, 1)), si.rowcnt), si.rowmodctr FROM {db}.sys.sysindexes si JOIN {db}.sys.objects o ON o.object_id = si.id AND o.type = 'U' AND o.is_ms_shipped = 0 JOIN {db}.sys.schemas s ON s.schema_id = o.schema_id diff --git a/src/SQLInsight.Core/Data/Indexes.cs b/src/SQLInsight.Core/Data/Indexes.cs index 4af39fd..98850d6 100644 --- a/src/SQLInsight.Core/Data/Indexes.cs +++ b/src/SQLInsight.Core/Data/Indexes.cs @@ -11,6 +11,16 @@ public sealed record IndexSnapshot public required IReadOnlyList Databases { get; init; } public TimeSpan Uptime => CapturedAt.UtcDateTime - SqlServerStartTime; + + /// Reads and writes per index since the previous collection, for the daily history. + public IReadOnlyList UsageDeltas { get; init; } = []; + + /// How long the usage of an index has been followed: across restarts, unlike . + public TimeSpan TrackedFor(IndexInfo index) + { + ArgumentNullException.ThrowIfNull(index); + return index.TrackedSince is { } since ? CapturedAt - since : Uptime; + } } public sealed record IndexDatabase @@ -51,6 +61,17 @@ public sealed record IndexInfo /// Heaps only: rows read through a forwarding pointer. public long ForwardedFetches { get; init; } + /// Reads summed across restarts of SQL Server since (index usage is reset at each restart). + public long TrackedReads { get; init; } + + /// Writes summed the same way. + public long TrackedWrites { get; init; } + + /// Since when SQLInsight has followed the usage (the restart before the first collection). + public DateTimeOffset? TrackedSince { get; init; } + + public string Key => $"{Schema}001f{Table}001f{IndexId}"; + public long Reads => UserSeeks + UserScans + UserLookups; public double SizeMb => UsedPages * 8.0 / 1024; public string QualifiedTable => $"{Schema}.{Table}"; @@ -88,3 +109,46 @@ public sealed record StatisticInfo /// Row modifications since the last update (sysindexes.rowmodctr; approximate). public long Modifications { get; init; } } + +/// An index's reads and writes since the previous collection (after a restart: since the restart). +public sealed record IndexUsageDelta(string Database, string Schema, string Table, string? Index, long Reads, long Writes); + +public static class IndexUsageTracking +{ + /// + /// Adds this collection's usage to the totals carried over from the previous one. After a restart the counters + /// started from zero again, so all of the current usage is new. + /// + public static IndexSnapshot Advance(IndexSnapshot? previous, IndexSnapshot current) + { + ArgumentNullException.ThrowIfNull(current); + var restarted = previous is null || previous.SqlServerStartTime != current.SqlServerStartTime; + var start = new DateTimeOffset(DateTime.SpecifyKind(current.SqlServerStartTime, DateTimeKind.Utc)); + var before = (previous?.Databases ?? []) + .SelectMany(d => d.Indexes.Select(i => (Key: d.Name + "\u001f" + i.Key, Index: i))) + .GroupBy(x => x.Key, StringComparer.OrdinalIgnoreCase) + .ToDictionary(g => g.Key, g => g.First().Index, StringComparer.OrdinalIgnoreCase); + var deltas = new List(); + var databases = current.Databases.Select(d => d with + { + Indexes = d.Indexes.Select(i => + { + before.TryGetValue(d.Name + "\u001f" + i.Key, out var was); + var reads = restarted || was is null ? i.Reads : Math.Max(0, i.Reads - was.Reads); + var writes = restarted || was is null ? i.UserUpdates : Math.Max(0, i.UserUpdates - was.UserUpdates); + if (reads > 0 || writes > 0) + { + deltas.Add(new IndexUsageDelta(d.Name, i.Schema, i.Table, i.Name, reads, writes)); + } + + return i with + { + TrackedReads = (was?.TrackedReads ?? 0) + reads, + TrackedWrites = (was?.TrackedWrites ?? 0) + writes, + TrackedSince = was?.TrackedSince ?? start, + }; + }).ToList(), + }).ToList(); + return current with { Databases = databases, UsageDeltas = deltas }; + } +} diff --git a/src/SQLInsight.Repository/InsightRepository.IndexUsage.cs b/src/SQLInsight.Repository/InsightRepository.IndexUsage.cs new file mode 100644 index 0000000..93525a0 --- /dev/null +++ b/src/SQLInsight.Repository/InsightRepository.IndexUsage.cs @@ -0,0 +1,54 @@ +using System.Text.Json; +using Dapper; +using Microsoft.Data.SqlClient; +using SQLInsight.Core.Data; + +namespace SQLInsight.Repository; + +/// An index's reads and writes summed over a period, and on how many days it was used at all. +public sealed record IndexUsageTotal(string DatabaseName, string SchemaName, string TableName, string IndexName, long Reads, long Writes, int Days, DateTime FirstDay, DateTime LastDay); + +public sealed partial class InsightRepository +{ + private static async Task SaveIndexUsageAsync(SqlConnection connection, int instanceId, IndexSnapshot snapshot) + { + if (snapshot.UsageDeltas.Count == 0) + { + return; + } + + await connection.ExecuteAsync(""" + MERGE insight.IndexUsageDaily AS t + USING (SELECT DatabaseName, SchemaName, TableName, IndexName, SUM(Reads) AS Reads, SUM(Writes) AS Writes + FROM OPENJSON(@json) WITH (DatabaseName nvarchar(128) '$.Database', SchemaName nvarchar(128) '$.Schema', TableName nvarchar(128) '$.Table', + IndexName nvarchar(128) '$.Index', Reads bigint, Writes bigint) + GROUP BY DatabaseName, SchemaName, TableName, IndexName) AS s + ON t.InstanceId = @instanceId AND t.[Day] = @day AND t.DatabaseName = s.DatabaseName AND t.SchemaName = s.SchemaName + AND t.TableName = s.TableName AND t.IndexName = s.IndexName + WHEN MATCHED THEN UPDATE SET Reads = t.Reads + s.Reads, Writes = t.Writes + s.Writes + WHEN NOT MATCHED THEN INSERT (InstanceId, [Day], DatabaseName, SchemaName, TableName, IndexName, Reads, Writes) + VALUES (@instanceId, @day, s.DatabaseName, s.SchemaName, s.TableName, s.IndexName, s.Reads, s.Writes); + """, new + { + instanceId, + day = snapshot.CapturedAt.UtcDateTime.Date, + json = JsonSerializer.Serialize(snapshot.UsageDeltas.Select(d => d with { Index = d.Index ?? "(heap)" }), Json), + }).ConfigureAwait(false); + } + + /// Index usage summed over the days in [from, to], least read first. + public async Task> GetIndexUsageAsync(int instanceId, DateTimeOffset from, DateTimeOffset to, int top = 500, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false); + var rows = await connection.QueryAsync(""" + SELECT TOP (@top) DatabaseName, SchemaName, TableName, IndexName, SUM(Reads) AS Reads, SUM(Writes) AS Writes, + SUM(CASE WHEN Reads > 0 THEN 1 ELSE 0 END) AS Days, MIN([Day]) AS FirstDay, MAX([Day]) AS LastDay + FROM insight.IndexUsageDaily + WHERE InstanceId = @instanceId AND [Day] >= @from AND [Day] <= @to + GROUP BY DatabaseName, SchemaName, TableName, IndexName + ORDER BY SUM(Reads), SUM(Writes) DESC; + """, new { instanceId, from = from.UtcDateTime.Date, to = to.UtcDateTime.Date, top }).ConfigureAwait(false); + return rows.ToList(); + } +} diff --git a/src/SQLInsight.Repository/InsightRepository.Maintenance.cs b/src/SQLInsight.Repository/InsightRepository.Maintenance.cs index 8989a41..876f3b5 100644 --- a/src/SQLInsight.Repository/InsightRepository.Maintenance.cs +++ b/src/SQLInsight.Repository/InsightRepository.Maintenance.cs @@ -89,6 +89,7 @@ public sealed partial class InsightRepository await DeleteInBatchesAsync(connection, "insight.MetricHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.WaitHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.DiskHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false); + await DeleteInBatchesAsync(connection, "insight.IndexUsageDaily", "[Day] < @cutoff", hourly).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.BlockingEvents", "SampledAt < @cutoff", events).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.Deadlocks", "OccurredAt < @cutoff", events).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.ErrorLogEvents", "LastAt < @cutoff", events).ConfigureAwait(false); diff --git a/src/SQLInsight.Repository/InsightRepository.cs b/src/SQLInsight.Repository/InsightRepository.cs index c92b590..03a0868 100644 --- a/src/SQLInsight.Repository/InsightRepository.cs +++ b/src/SQLInsight.Repository/InsightRepository.cs @@ -280,6 +280,10 @@ public sealed partial class InsightRepository(string connectionString) ]).ConfigureAwait(false); await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false); return; + case IndexSnapshot indexes: + await SaveIndexUsageAsync(connection, instanceId, indexes).ConfigureAwait(false); + await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false); + return; case PrivilegeSnapshot privileges: await SavePrivilegeChangesAsync(connection, instanceId, privileges.Changes).ConfigureAwait(false); await SaveLatePrivilegeAuthorsAsync(connection, instanceId, privileges.RecentChanges).ConfigureAwait(false); diff --git a/src/SQLInsight.Repository/Migrations/0015_index_usage_daily.sql b/src/SQLInsight.Repository/Migrations/0015_index_usage_daily.sql new file mode 100644 index 0000000..fdb82e1 --- /dev/null +++ b/src/SQLInsight.Repository/Migrations/0015_index_usage_daily.sql @@ -0,0 +1,14 @@ +/* Index reads and writes per day (deltas of sys.dm_db_index_usage_stats between collections), so unused indexes can be + judged over time, across restarts of SQL Server. Kept as long as the hourly history (HourlyDays). */ +CREATE TABLE insight.IndexUsageDaily +( + InstanceId int NOT NULL, + [Day] date NOT NULL, + DatabaseName nvarchar(128) NOT NULL, + SchemaName nvarchar(128) NOT NULL, + TableName nvarchar(128) NOT NULL, + IndexName nvarchar(128) NOT NULL, + Reads bigint NOT NULL, + Writes bigint NOT NULL, + CONSTRAINT PK_IndexUsageDaily PRIMARY KEY (InstanceId, [Day], DatabaseName, SchemaName, TableName, IndexName) +); diff --git a/src/SQLInsight.Rules/Indexes/IndexRules.cs b/src/SQLInsight.Rules/Indexes/IndexRules.cs index b5d1111..f016e14 100644 --- a/src/SQLInsight.Rules/Indexes/IndexRules.cs +++ b/src/SQLInsight.Rules/Indexes/IndexRules.cs @@ -173,25 +173,33 @@ public sealed class UnusedIndexRule : Rule public override RuleEvaluation Evaluate(RuleContext context) { ArgumentNullException.ThrowIfNull(context); - var snapshot = IndexText.RequireUptime(context); + // Usage is followed across restarts, so an index can be judged once it has been followed for a week. + var snapshot = IndexText.Snapshot(context); + var followed = snapshot.Databases.SelectMany(db => db.Indexes).Any(i => snapshot.TrackedFor(i) >= IndexText.MinUptimeForUsage); + if (!followed) + { + IndexText.RequireUptime(context); + } + return Result(snapshot.Databases .SelectMany(db => db.Indexes.Select(i => (Db: db.Name, Index: i))) .Where(x => x.Index.Type == "NONCLUSTERED" && !x.Index.IsPrimaryKey && !x.Index.IsUniqueConstraint && !x.Index.IsUnique - && !x.Index.IsDisabled && x.Index.Reads == 0 && (x.Index.UserUpdates >= 1000 || x.Index.SizeMb >= 100)) - .OrderByDescending(x => x.Index.UserUpdates + x.Index.UsedPages) + && !x.Index.IsDisabled && snapshot.TrackedFor(x.Index) >= IndexText.MinUptimeForUsage + && x.Index.Reads == 0 && x.Index.TrackedReads == 0 && (Math.Max(x.Index.UserUpdates, x.Index.TrackedWrites) >= 1000 || x.Index.SizeMb >= 100)) + .OrderByDescending(x => Math.Max(x.Index.UserUpdates, x.Index.TrackedWrites) + x.Index.UsedPages) .Take(IndexText.MaxFindingsPerRule) .Select(x => CreateFinding( x.Index.SizeMb >= 1024 ? Severity.Medium : Severity.Low, - $"{x.Db}.{x.Index.QualifiedTable}: index {x.Index.Name} er ikke læst i {Format.Age(snapshot.Uptime)}", - "Indexet opdateres ved hver INSERT, UPDATE og DELETE på tabellen, men ingen forespørgsel har læst det siden seneste " - + "genstart. Det koster skrivninger, transaktionslog, backup-størrelse og vedligeholdelsestid.", + $"{x.Db}.{x.Index.QualifiedTable}: index {x.Index.Name} er ikke læst i {Format.Age(snapshot.TrackedFor(x.Index))}", + "Indexet opdateres ved hver INSERT, UPDATE og DELETE på tabellen, men ingen forespørgsel har læst det, så længe SQLInsight har fulgt det " + + "(også hen over genstarter af SQL Server). Det koster skrivninger, transaktionslog, backup-størrelse og vedligeholdelsestid.", "Deaktivér indexet (ALTER INDEX … DISABLE) i stedet for at slette det, så definitionen bevares, og vent en hel " + "forretningscyklus (måneds- og kvartalsafslutning, rapporter). Er det stadig ubrugt, kan det slettes. Genopret med ALTER INDEX … REBUILD.", [ new Evidence("Index", $"{x.Index.Name} ({IndexText.Columns(x.Index.KeyColumns)})"), - new Evidence("Skrivninger / læsninger", $"{Format.Number(x.Index.UserUpdates)} / 0"), + new Evidence("Skrivninger / læsninger", $"{Format.Number(Math.Max(x.Index.UserUpdates, x.Index.TrackedWrites))} / 0"), new Evidence("Størrelse", $"{Format.Number(x.Index.SizeMb)} MB"), - new Evidence("Periode", $"siden genstart for {Format.Age(snapshot.Uptime)} siden"), + new Evidence("Periode", $"fulgt i {Format.Age(snapshot.TrackedFor(x.Index))} (seneste genstart for {Format.Age(snapshot.Uptime)} siden)"), ], subject: $"index: {x.Db}.{x.Index.QualifiedTable}.{x.Index.Name}", fixScript: $"{IndexText.Use(x.Db)}\nALTER INDEX {IndexText.Quote(x.Index.Name!)} ON {IndexText.Table(x.Index)} DISABLE;", @@ -330,12 +338,15 @@ public sealed class StaleStatisticsRule : Rule { ArgumentNullException.ThrowIfNull(context); return Result(IndexText.Snapshot(context).Databases - .SelectMany(db => db.Statistics.Where(s => s.Rows >= MinRows && s.Modifications >= s.Rows * MinModifiedShare).Select(s => (Db: db.Name, Stat: s))) - .OrderByDescending(x => x.Stat.Modifications) + .SelectMany(db => db.Statistics.Where(s => s.LastUpdated is null && s.Rows > 0 || s.Rows >= MinRows && s.Modifications >= s.Rows * MinModifiedShare) + .Select(s => (Db: db.Name, Stat: s))) + .OrderByDescending(x => x.Stat.LastUpdated is null).ThenByDescending(x => x.Stat.Modifications) .Take(IndexText.MaxFindingsPerRule) .Select(x => CreateFinding( Severity.Low, - $"{x.Db}.{x.Stat.Schema}.{x.Stat.Table}: statistikken {x.Stat.Name} er {Format.Number(100.0 * x.Stat.Modifications / x.Stat.Rows)} % ændret", + x.Stat.LastUpdated is null + ? $"{x.Db}.{x.Stat.Schema}.{x.Stat.Table}: statistikken {x.Stat.Name} er aldrig opdateret ({Format.Number(x.Stat.Rows)} rækker)" + : $"{x.Db}.{x.Stat.Schema}.{x.Stat.Table}: statistikken {x.Stat.Name} er {Format.Number(100.0 * x.Stat.Modifications / x.Stat.Rows)} % ændret", "Optimizeren vælger planer ud fra statistikkernes billede af data. Når en stor del af rækkerne er ændret siden seneste " + "opdatering, rammer estimaterne ved siden af: for små memory grants, forkerte join-typer og spild til tempdb.", "Opdatér statistikkerne som en del af det faste vedligehold (fx Ola Hallengrens IndexOptimize med @UpdateStatistics = 'ALL' og " diff --git a/src/SQLInsight.Web/Api/ApiV1Endpoints.cs b/src/SQLInsight.Web/Api/ApiV1Endpoints.cs index 223f542..97febd0 100644 --- a/src/SQLInsight.Web/Api/ApiV1Endpoints.cs +++ b/src/SQLInsight.Web/Api/ApiV1Endpoints.cs @@ -49,6 +49,7 @@ public static class ApiV1Endpoints api.MapGet("/instances/{name}/activity", ActivityAsync).WithSummary("Åbne transaktioner og lange sætninger lige nu / Open transactions and long-running statements"); api.MapGet("/instances/{name}/tempdb", TempdbAsync).WithSummary("tempdb og version store lige nu / tempdb and the version store"); api.MapGet("/instances/{name}/replication", ReplicationAsync).WithSummary("Replikering set fra distributøren / Replication from the distributor"); + api.MapGet("/instances/{name}/index-usage", IndexUsageAsync).WithSummary("Indexbrug over tid, på tværs af genstarter / Index usage over time (days)"); api.MapGet("/instances/{name}/error-log", ErrorLogAsync).WithSummary("Vigtige hændelser i fejlloggen / Important error log events (days)"); api.MapGet("/instances/{name}/wait-types/{waitType}", WaitTypeAsync).WithSummary("En ventetype: forklaring, udvikling og sætninger / One wait type"); } @@ -576,6 +577,19 @@ public static class ApiV1Endpoints }); } + private static async Task IndexUsageAsync(string name, InsightRepository repository, CancellationToken ct, int? days = null) + { + if (await repository.GetInstanceAsync(name, ct).ConfigureAwait(false) is not { } instance) + { + return Results.NotFound(); + } + + var span = Math.Clamp(days ?? 30, 1, 400); + var now = DateTimeOffset.UtcNow; + var usage = await repository.GetIndexUsageAsync(instance.InstanceId, now.AddDays(-span), now, 1000, ct).ConfigureAwait(false); + return Results.Ok(new { Instance = instance.Name, Days = span, Indexes = usage }); + } + private static async Task ErrorLogAsync(string name, InsightRepository repository, IOptions options, CancellationToken ct, int? days = null) { if (await repository.GetInstanceAsync(name, ct).ConfigureAwait(false) is not { } instance) diff --git a/tests/SQLInsight.Core.Tests/IndexUsageTests.cs b/tests/SQLInsight.Core.Tests/IndexUsageTests.cs new file mode 100644 index 0000000..5d2f3ac --- /dev/null +++ b/tests/SQLInsight.Core.Tests/IndexUsageTests.cs @@ -0,0 +1,33 @@ +using SQLInsight.Core.Data; + +namespace SQLInsight.Core.Tests; + +public class IndexUsageTests +{ + private static readonly DateTime Start = new(2026, 9, 1, 6, 0, 0, DateTimeKind.Utc); + + private static IndexSnapshot Snapshot(DateTime start, DateTimeOffset at, long reads, long writes) => new() + { + CapturedAt = at, + SqlServerStartTime = start, + Databases = [new IndexDatabase { Name = "AX", Indexes = [new IndexInfo { Schema = "dbo", Table = "INVENTTRANS", IndexId = 5, Name = "I_177ITEMIDX", Type = "NONCLUSTERED", UserSeeks = reads, UserUpdates = writes }] }], + }; + + [Fact] + public void Usage_is_summed_across_restarts_and_tracked_from_the_first_restart_seen() + { + var first = IndexUsageTracking.Advance(null, Snapshot(Start, new DateTimeOffset(Start.AddDays(1)), 10, 100)); + var second = IndexUsageTracking.Advance(first, Snapshot(Start, new DateTimeOffset(Start.AddDays(2)), 15, 300)); + var restart = Start.AddDays(3); + var afterRestart = IndexUsageTracking.Advance(second, Snapshot(restart, new DateTimeOffset(restart.AddHours(6)), 2, 50)); + + var index = afterRestart.Databases[0].Indexes[0]; + Assert.Equal((17L, 350L), (index.TrackedReads, index.TrackedWrites)); + Assert.Equal(new DateTimeOffset(Start), index.TrackedSince); + Assert.Equal(TimeSpan.FromDays(3.25), afterRestart.TrackedFor(index)); + Assert.Equal(TimeSpan.FromHours(6), afterRestart.Uptime); + var delta = Assert.Single(second.UsageDeltas); + Assert.Equal((5L, 200L), (delta.Reads, delta.Writes)); + Assert.Equal((2L, 50L), (afterRestart.UsageDeltas[0].Reads, afterRestart.UsageDeltas[0].Writes)); // counters started over + } +} diff --git a/tests/SQLInsight.IntegrationTests/IndexUsageTests.cs b/tests/SQLInsight.IntegrationTests/IndexUsageTests.cs new file mode 100644 index 0000000..2c7f2b9 --- /dev/null +++ b/tests/SQLInsight.IntegrationTests/IndexUsageTests.cs @@ -0,0 +1,66 @@ +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using SQLInsight.Collectors.Databases; +using SQLInsight.Core.Data; +using SQLInsight.Core.Instances; +using SQLInsight.Repository; + +namespace SQLInsight.IntegrationTests; + +/// Index usage followed over time, and statistics that were never updated. +[Collection(SqlServerTestGroup.Name)] +[Trait("Category", "Integration")] +public class IndexUsageTests(SqlServerFixture sql) +{ + [Fact] + public async Task Usage_between_collections_is_stored_per_day_and_never_updated_statistics_are_seen() + { + var table = "Usage_" + Guid.NewGuid().ToString("N")[..6]; + await sql.ExecuteAsSaAsync($""" + USE [{SqlServerFixture.TestDatabase}]; + CREATE TABLE dbo.{table} (Id int IDENTITY PRIMARY KEY, Code int NOT NULL, Name nvarchar(20)); + CREATE INDEX IX_{table}_Code ON dbo.{table} (Code); + CREATE STATISTICS ST_{table}_Name ON dbo.{table} (Name) WITH NORECOMPUTE; + INSERT dbo.{table} (Code, Name) SELECT TOP (500) object_id % 100, N'x' FROM sys.all_objects; + """); + var database = "SQLInsightIdx_" + Guid.NewGuid().ToString("N")[..8]; + await sql.ExecuteAsSaAsync($"CREATE DATABASE [{database}];"); + try + { + await using var session = await AgentSession.OpenAsync(sql); + var first = await session.CollectAsync(new IndexCollector()); + await sql.ExecuteAsSaAsync($"USE [{SqlServerFixture.TestDatabase}]; SELECT COUNT(*) FROM dbo.{table} WITH (INDEX(IX_{table}_Code)) WHERE Code = 7;"); + var second = await session.CollectAsync(new IndexCollector(), first); + + var index = second.Databases.Single(d => d.Name == SqlServerFixture.TestDatabase).Indexes.Single(i => i.Name == $"IX_{table}_Code"); + Assert.True(index.TrackedReads >= 1); + Assert.NotNull(index.TrackedSince); + var delta = Assert.Single(second.UsageDeltas, d => d.Index == $"IX_{table}_Code"); + Assert.True(delta.Reads >= 1); + var never = second.Databases.Single(d => d.Name == SqlServerFixture.TestDatabase).Statistics.Single(s => s.Name == $"ST_{table}_Name"); + Assert.Null(never.LastUpdated); + Assert.Equal(500, never.Rows); // the table's rows, not the statistic's + + var repository = new InsightRepository(sql.DatabaseConnectionString(database)); + await repository.MigrateAsync(); + var id = await repository.UpsertInstanceAsync("prod-sql01", session.Instance, "SqlClient", false, DateTimeOffset.UtcNow); + await repository.SaveCollectedAsync(id, "indexes", second); + var usage = await repository.GetIndexUsageAsync(id, DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow); + Assert.Contains(usage, u => u.IndexName == $"IX_{table}_Code" && u.Reads >= 1 && u.Days == 1); + + await using var factory = new WebApplicationFactory().WithWebHostBuilder(b => + { + b.UseEnvironment("Development"); + b.UseSetting("SQLInsight:Repository:ConnectionString", sql.DatabaseConnectionString(database)); + b.UseSetting("SQLInsight:Web:Authentication", "None"); + }); + var api = await factory.CreateClient().GetStringAsync("/api/v1/instances/prod-sql01/index-usage?days=7"); + Assert.Contains($"IX_{table}_Code", api, StringComparison.Ordinal); + } + finally + { + await sql.ExecuteAsSaAsync($"ALTER DATABASE [{database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{database}];"); + await sql.ExecuteAsSaAsync($"USE [{SqlServerFixture.TestDatabase}]; DROP TABLE dbo.{table};"); + } + } +} diff --git a/tests/SQLInsight.Rules.Tests/IndexRuleTests.cs b/tests/SQLInsight.Rules.Tests/IndexRuleTests.cs index 5c66a9d..8e65be2 100644 --- a/tests/SQLInsight.Rules.Tests/IndexRuleTests.cs +++ b/tests/SQLInsight.Rules.Tests/IndexRuleTests.cs @@ -173,9 +173,10 @@ public class IndexRuleTests Name = "Sales", Statistics = [ - new StatisticInfo { Schema = "dbo", Table = "Orders", Name = "IX_Orders_Date", Rows = 1_000_000, Modifications = 400_000 }, - new StatisticInfo { Schema = "dbo", Table = "Small", Name = "PK_Small", Rows = 500, Modifications = 500 }, - new StatisticInfo { Schema = "dbo", Table = "Stable", Name = "PK_Stable", Rows = 1_000_000, Modifications = 1_000 }, + new StatisticInfo { Schema = "dbo", Table = "Orders", Name = "IX_Orders_Date", Rows = 1_000_000, Modifications = 400_000, LastUpdated = Now.UtcDateTime.AddDays(-9) }, + new StatisticInfo { Schema = "dbo", Table = "Small", Name = "PK_Small", Rows = 500, Modifications = 500, LastUpdated = Now.UtcDateTime.AddDays(-9) }, + new StatisticInfo { Schema = "dbo", Table = "Stable", Name = "PK_Stable", Rows = 1_000_000, Modifications = 1_000, LastUpdated = Now.UtcDateTime.AddDays(-9) }, + new StatisticInfo { Schema = "dbo", Table = "Empty", Name = "_WA_Sys_Empty", Rows = 0, Modifications = 0 }, ], }; @@ -184,6 +185,40 @@ public class IndexRuleTests Assert.Equal("statistic: Sales.dbo.Orders.IX_Orders_Date", finding.Subject); } + + [Fact] + public void Statistics_never_updated_on_a_table_with_rows_are_found_first() + { + var db = new IndexDatabase + { + Name = "AX", + Statistics = + [ + new StatisticInfo { Schema = "dbo", Table = "Orders", Name = "IX_Orders_Date", Rows = 1_000_000, Modifications = 400_000, LastUpdated = Now.UtcDateTime.AddDays(-9) }, + new StatisticInfo { Schema = "dbo", Table = "CUSTTRANS", Name = "_WA_Sys_00000003", Rows = 120 }, + ], + }; + + var findings = Run(new StaleStatisticsRule(), Indexes(Month, db)).Findings; + + Assert.Equal(2, findings.Count); + Assert.Contains("aldrig opdateret", findings[0].Title, StringComparison.Ordinal); + } + + [Fact] + public void Unused_index_is_judged_on_usage_followed_across_restarts() + { + var index = Index(2, "IX_Unused", "NONCLUSTERED", ["Status"], reads: 0, updates: 50_000) with + { + TrackedReads = 0, TrackedWrites = 900_000, TrackedSince = Now.AddDays(-40), + }; + var db = new IndexDatabase { Name = "Sales", Indexes = [index] }; + + var finding = Assert.Single(Run(new UnusedIndexRule(), Indexes(TimeSpan.FromDays(1), db)).Findings); // restarted yesterday + + Assert.Contains("40 dage", finding.Title, StringComparison.Ordinal); + Assert.Equal(RuleOutcome.Passed, Run(new UnusedIndexRule(), Indexes(TimeSpan.FromDays(1), db with { Indexes = [index with { TrackedReads = 3 }] })).Outcome); + } [Fact] public void Auto_statistics_off() {