sqlinsight-pro/src/SQLInsight.Collectors/Databases/IndexCollector.cs

using System.Data.Common;
using SQLInsight.Core.Data;
using SQLInsight.Core.Instances;

namespace SQLInsight.Collectors.Databases;

/// <summary>
/// Index inventory with usage, missing-index suggestions and statistics for every readable user database.
/// Everything is metadata and DMVs; no table data is read and nothing is scanned (no index_physical_stats).
/// </summary>
public sealed class IndexCollector : Collector<IndexSnapshot>
{
    internal const string DatabasesSql = """
        SELECT d.name FROM sys.databases d
        WHERE d.state = 0 AND d.database_id > 4 AND HAS_DBACCESS(d.name) = 1
          AND NOT EXISTS (SELECT 1 FROM sys.database_mirroring m WHERE m.database_id = d.database_id AND m.mirroring_role = 2)
        """;

    internal const string MissingSql = """
        SELECT DB_NAME(d.database_id), d.statement, d.equality_columns, d.inequality_columns, d.included_columns,
               gs.user_seeks, gs.user_scans, gs.avg_total_user_cost, gs.avg_user_impact, gs.last_user_seek
        FROM sys.dm_db_missing_index_details d
        JOIN sys.dm_db_missing_index_groups g ON g.index_handle = d.index_handle
        JOIN sys.dm_db_missing_index_group_stats gs ON gs.group_handle = g.index_group_handle
        """;

    public override string Name => "indexes";
    public override TimeSpan DefaultInterval => TimeSpan.FromHours(6);
    public override IReadOnlyCollection<Capability> RequiredCapabilities => [Capability.DynamicManagementViews];

    protected override async Task<IndexSnapshot> CollectAsync(CollectorContext context, IndexSnapshot? previous)
    {
        var connection = context.Connection;
        var ct = context.CancellationToken;
        var startTime = await Db.StartTimeAsync(connection, context.Instance.Version, ct).ConfigureAwait(false);
        var names = (await Db.QueryAsync(connection, DatabasesSql, r => r.String(0), ct).ConfigureAwait(false)).Where(context.IncludesDatabase).ToList();
        var missing = await Db.QueryAsync(connection, MissingSql, r => (Database: r.NullableString(0), Index: new MissingIndex
        {
            Table = r.String(1),
            EqualityColumns = r.NullableString(2),
            InequalityColumns = r.NullableString(3),
            IncludedColumns = r.NullableString(4),
            Seeks = r.Long(5),
            Scans = r.Long(6),
            AverageTotalUserCost = r.Double(7),
            AverageUserImpact = r.Double(8),
            LastUserSeek = r.NullableDateTime(9),
        }), ct).ConfigureAwait(false);

        var databases = new List<IndexDatabase>(names.Count);
        foreach (var name in names)
        {
            databases.Add(await ReadAsync(connection, name, context.Instance.Version.Major >= 10, ct).ConfigureAwait(false) with
            {
                MissingIndexes = missing.Where(m => string.Equals(m.Database, name, StringComparison.OrdinalIgnoreCase)).Select(m => m.Index).ToList(),
            });
        }

        return IndexUsageTracking.Advance(previous, new IndexSnapshot { CapturedAt = context.Now, SqlServerStartTime = startTime, Databases = databases });
    }

    private static async Task<IndexDatabase> ReadAsync(DbConnection connection, string database, bool filtered, CancellationToken ct)
    {
        try
        {
            var indexes = await Db.QueryAsync(connection, BuildIndexSql(database, filtered), r => new IndexInfo
            {
                Schema = r.String(0),
                Table = r.String(1),
                IndexId = r.Int(2),
                Name = r.NullableString(3),
                Type = r.String(4),
                IsUnique = r.Bool(5),
                IsPrimaryKey = r.Bool(6),
                IsUniqueConstraint = r.Bool(7),
                IsDisabled = r.Bool(8),
                Filter = r.NullableString(9),
                KeyColumns = Split(r.NullableString(10)),
                IncludedColumns = Split(r.NullableString(11)),
                Rows = r.NullableLong(12) ?? 0,
                UsedPages = r.NullableLong(13) ?? 0,
                UserSeeks = r.NullableLong(14) ?? 0,
                UserScans = r.NullableLong(15) ?? 0,
                UserLookups = r.NullableLong(16) ?? 0,
                UserUpdates = r.NullableLong(17) ?? 0,
                ForwardedFetches = r.NullableLong(18) ?? 0,
            }, ct).ConfigureAwait(false);

            var statistics = await Db.QueryAsync(connection, BuildStatisticsSql(database), r => new StatisticInfo
            {
                Schema = r.String(0),
                Table = r.String(1),
                Name = r.String(2),
                LastUpdated = r.NullableDateTime(3),
                Rows = r.NullableLong(4) ?? 0,
                Modifications = r.NullableLong(5) ?? 0,
            }, ct).ConfigureAwait(false);

            return new IndexDatabase { Name = database, Indexes = indexes, Statistics = statistics };
        }
        catch (DbException ex)
        {
            return new IndexDatabase { Name = database, Error = ex.Message };
        }
    }

    private static string[] Split(string? columns) =>
        string.IsNullOrEmpty(columns) ? [] : columns.Split('\uE000', StringSplitOptions.RemoveEmptyEntries);

    // Filtered indexes (filter_definition) exist from 2008.
    // Column lists are joined with U+E000 (private use: valid in XML, not found in real names) using FOR XML PATH,
    // which works from 2005. Control characters such as NCHAR(1) cannot be serialized by FOR XML.
    internal static string BuildIndexSql(string database, bool filteredIndexes = true)
    {
        var db = Db.Identifier(database);
        var id = $"DB_ID({Db.Literal(database)})";
        string Columns(string included) => $"""
            STUFF((SELECT NCHAR(57344) + c.name + CASE WHEN ic.is_descending_key = 1 THEN N' DESC' ELSE N'' END
                   FROM {db}.sys.index_columns ic JOIN {db}.sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id
                   WHERE ic.object_id = i.object_id AND ic.index_id = i.index_id AND ic.is_included_column = {included}
                     {(included == "0" ? "AND ic.key_ordinal > 0" : string.Empty)}
                   ORDER BY {(included == "0" ? "ic.key_ordinal" : "c.name")}
                   FOR XML PATH(''), TYPE).value('.', 'nvarchar(max)'), 1, 1, N'')
            """;
        return $"""
            SELECT s.name, o.name, i.index_id, i.name, i.type_desc, i.is_unique, i.is_primary_key, i.is_unique_constraint, i.is_disabled,
                   {(filteredIndexes ? "i.filter_definition" : "CAST(NULL AS nvarchar(max))")},
                   {Columns("0")},
                   {Columns("1")},
                   ps.row_count, ps.used_pages,
                   us.user_seeks, us.user_scans, us.user_lookups, us.user_updates,
                   fw.forwarded
            FROM {db}.sys.indexes i
            JOIN {db}.sys.objects o ON o.object_id = i.object_id AND o.type = 'U' AND o.is_ms_shipped = 0
            JOIN {db}.sys.schemas s ON s.schema_id = o.schema_id
            OUTER APPLY (SELECT SUM(p.row_count) AS row_count, SUM(p.used_page_count) AS used_pages
                         FROM {db}.sys.dm_db_partition_stats p WHERE p.object_id = i.object_id AND p.index_id = i.index_id) ps
            LEFT JOIN sys.dm_db_index_usage_stats us ON us.database_id = {id} AND us.object_id = i.object_id AND us.index_id = i.index_id
            OUTER APPLY (SELECT SUM(os.forwarded_fetch_count) AS forwarded
                         FROM sys.dm_db_index_operational_stats({id}, i.object_id, 0, NULL) os WHERE i.index_id = 0) fw;
            """;
    }

    // sys.sysindexes.rowmodctr needs only metadata visibility, unlike sys.dm_db_stats_properties. Rows are the table's
    // (heap or clustered index), so a statistic that was never updated (STATS_DATE NULL) still shows the rows it covers.
    internal static string BuildStatisticsSql(string database)
    {
        var db = Db.Identifier(database);
        return $"""
            SELECT s.name, o.name, si.name, STATS_DATE(si.id, si.indid),
                   COALESCE((SELECT TOP 1 t.rowcnt FROM {db}.sys.sysindexes t WHERE t.id = si.id AND t.indid IN (0, 1)), si.rowcnt), si.rowmodctr
            FROM {db}.sys.sysindexes si
            JOIN {db}.sys.objects o ON o.object_id = si.id AND o.type = 'U' AND o.is_ms_shipped = 0
            JOIN {db}.sys.schemas s ON s.schema_id = o.schema_id
            WHERE si.indid > 0 AND si.indid < 255;
            """;
    }
}