using System.Data.Common;
using SQLInsight.Core.Data;
using SQLInsight.Core.Instances;
namespace SQLInsight.Collectors.Databases;
///
/// Index inventory with usage, missing-index suggestions and statistics for every readable user database.
/// Everything is metadata and DMVs; no table data is read and nothing is scanned (no index_physical_stats).
///
public sealed class IndexCollector : Collector
{
internal const string DatabasesSql = """
SELECT d.name FROM sys.databases d
WHERE d.state = 0 AND d.database_id > 4 AND HAS_DBACCESS(d.name) = 1
AND NOT EXISTS (SELECT 1 FROM sys.database_mirroring m WHERE m.database_id = d.database_id AND m.mirroring_role = 2)
""";
internal const string MissingSql = """
SELECT DB_NAME(d.database_id), d.statement, d.equality_columns, d.inequality_columns, d.included_columns,
gs.user_seeks, gs.user_scans, gs.avg_total_user_cost, gs.avg_user_impact, gs.last_user_seek
FROM sys.dm_db_missing_index_details d
JOIN sys.dm_db_missing_index_groups g ON g.index_handle = d.index_handle
JOIN sys.dm_db_missing_index_group_stats gs ON gs.group_handle = g.index_group_handle
""";
public override string Name => "indexes";
public override TimeSpan DefaultInterval => TimeSpan.FromHours(6);
public override IReadOnlyCollection RequiredCapabilities => [Capability.DynamicManagementViews];
protected override async Task CollectAsync(CollectorContext context, IndexSnapshot? previous)
{
var connection = context.Connection;
var ct = context.CancellationToken;
var startTime = await Db.StartTimeAsync(connection, context.Instance.Version, ct).ConfigureAwait(false);
var names = (await Db.QueryAsync(connection, DatabasesSql, r => r.String(0), ct).ConfigureAwait(false)).Where(context.IncludesDatabase).ToList();
var missing = await Db.QueryAsync(connection, MissingSql, r => (Database: r.NullableString(0), Index: new MissingIndex
{
Table = r.String(1),
EqualityColumns = r.NullableString(2),
InequalityColumns = r.NullableString(3),
IncludedColumns = r.NullableString(4),
Seeks = r.Long(5),
Scans = r.Long(6),
AverageTotalUserCost = r.Double(7),
AverageUserImpact = r.Double(8),
LastUserSeek = r.NullableDateTime(9),
}), ct).ConfigureAwait(false);
var databases = new List(names.Count);
foreach (var name in names)
{
databases.Add(await ReadAsync(connection, name, context.Instance.Version.Major >= 10, ct).ConfigureAwait(false) with
{
MissingIndexes = missing.Where(m => string.Equals(m.Database, name, StringComparison.OrdinalIgnoreCase)).Select(m => m.Index).ToList(),
});
}
return IndexUsageTracking.Advance(previous, new IndexSnapshot { CapturedAt = context.Now, SqlServerStartTime = startTime, Databases = databases });
}
private static async Task ReadAsync(DbConnection connection, string database, bool filtered, CancellationToken ct)
{
try
{
var indexes = await Db.QueryAsync(connection, BuildIndexSql(database, filtered), r => new IndexInfo
{
Schema = r.String(0),
Table = r.String(1),
IndexId = r.Int(2),
Name = r.NullableString(3),
Type = r.String(4),
IsUnique = r.Bool(5),
IsPrimaryKey = r.Bool(6),
IsUniqueConstraint = r.Bool(7),
IsDisabled = r.Bool(8),
Filter = r.NullableString(9),
KeyColumns = Split(r.NullableString(10)),
IncludedColumns = Split(r.NullableString(11)),
Rows = r.NullableLong(12) ?? 0,
UsedPages = r.NullableLong(13) ?? 0,
UserSeeks = r.NullableLong(14) ?? 0,
UserScans = r.NullableLong(15) ?? 0,
UserLookups = r.NullableLong(16) ?? 0,
UserUpdates = r.NullableLong(17) ?? 0,
ForwardedFetches = r.NullableLong(18) ?? 0,
}, ct).ConfigureAwait(false);
var statistics = await Db.QueryAsync(connection, BuildStatisticsSql(database), r => new StatisticInfo
{
Schema = r.String(0),
Table = r.String(1),
Name = r.String(2),
LastUpdated = r.NullableDateTime(3),
Rows = r.NullableLong(4) ?? 0,
Modifications = r.NullableLong(5) ?? 0,
}, ct).ConfigureAwait(false);
return new IndexDatabase { Name = database, Indexes = indexes, Statistics = statistics };
}
catch (DbException ex)
{
return new IndexDatabase { Name = database, Error = ex.Message };
}
}
private static string[] Split(string? columns) =>
string.IsNullOrEmpty(columns) ? [] : columns.Split('\uE000', StringSplitOptions.RemoveEmptyEntries);
// Filtered indexes (filter_definition) exist from 2008.
// Column lists are joined with U+E000 (private use: valid in XML, not found in real names) using FOR XML PATH,
// which works from 2005. Control characters such as NCHAR(1) cannot be serialized by FOR XML.
internal static string BuildIndexSql(string database, bool filteredIndexes = true)
{
var db = Db.Identifier(database);
var id = $"DB_ID({Db.Literal(database)})";
string Columns(string included) => $"""
STUFF((SELECT NCHAR(57344) + c.name + CASE WHEN ic.is_descending_key = 1 THEN N' DESC' ELSE N'' END
FROM {db}.sys.index_columns ic JOIN {db}.sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id
WHERE ic.object_id = i.object_id AND ic.index_id = i.index_id AND ic.is_included_column = {included}
{(included == "0" ? "AND ic.key_ordinal > 0" : string.Empty)}
ORDER BY {(included == "0" ? "ic.key_ordinal" : "c.name")}
FOR XML PATH(''), TYPE).value('.', 'nvarchar(max)'), 1, 1, N'')
""";
return $"""
SELECT s.name, o.name, i.index_id, i.name, i.type_desc, i.is_unique, i.is_primary_key, i.is_unique_constraint, i.is_disabled,
{(filteredIndexes ? "i.filter_definition" : "CAST(NULL AS nvarchar(max))")},
{Columns("0")},
{Columns("1")},
ps.row_count, ps.used_pages,
us.user_seeks, us.user_scans, us.user_lookups, us.user_updates,
fw.forwarded
FROM {db}.sys.indexes i
JOIN {db}.sys.objects o ON o.object_id = i.object_id AND o.type = 'U' AND o.is_ms_shipped = 0
JOIN {db}.sys.schemas s ON s.schema_id = o.schema_id
OUTER APPLY (SELECT SUM(p.row_count) AS row_count, SUM(p.used_page_count) AS used_pages
FROM {db}.sys.dm_db_partition_stats p WHERE p.object_id = i.object_id AND p.index_id = i.index_id) ps
LEFT JOIN sys.dm_db_index_usage_stats us ON us.database_id = {id} AND us.object_id = i.object_id AND us.index_id = i.index_id
OUTER APPLY (SELECT SUM(os.forwarded_fetch_count) AS forwarded
FROM sys.dm_db_index_operational_stats({id}, i.object_id, 0, NULL) os WHERE i.index_id = 0) fw;
""";
}
// sys.sysindexes.rowmodctr needs only metadata visibility, unlike sys.dm_db_stats_properties. Rows are the table's
// (heap or clustered index), so a statistic that was never updated (STATS_DATE NULL) still shows the rows it covers.
internal static string BuildStatisticsSql(string database)
{
var db = Db.Identifier(database);
return $"""
SELECT s.name, o.name, si.name, STATS_DATE(si.id, si.indid),
COALESCE((SELECT TOP 1 t.rowcnt FROM {db}.sys.sysindexes t WHERE t.id = si.id AND t.indid IN (0, 1)), si.rowcnt), si.rowmodctr
FROM {db}.sys.sysindexes si
JOIN {db}.sys.objects o ON o.object_id = si.id AND o.type = 'U' AND o.is_ms_shipped = 0
JOIN {db}.sys.schemas s ON s.schema_id = o.schema_id
WHERE si.indid > 0 AND si.indid < 255;
""";
}
}