sqlinsight-pro/src/SQLInsight.Repository/InsightRepository.Maintenance.cs

using Dapper;

namespace SQLInsight.Repository;

public sealed class RetentionOptions
{
    /// <summary>Raw samples (5-minute waits, per-minute CPU …).</summary>
    public int RawDays { get; set; } = 14;

    /// <summary>Hourly aggregates, for long trends, and the daily samples of Dynamics AX number sequences and RecIds.</summary>
    public int HourlyDays { get; set; } = 400;

    /// <summary>Blocking events, deadlock graphs and completed Dynamics AX batch runs.</summary>
    public int EventDays { get; set; } = 90;

    /// <summary>Resolved findings.</summary>
    public int ResolvedFindingDays { get; set; } = 365;
}

public sealed partial class InsightRepository
{
    private const int DeleteBatchSize = 5000;

    /// <summary>
    /// Aggregates recent raw samples to hourly values and deletes data past retention. Deletes run in small batches
    /// so the repository's own transaction log never has to hold a huge delete.
    /// </summary>
    public async Task RunMaintenanceAsync(RetentionOptions options, DateTimeOffset now, CancellationToken cancellationToken = default)
    {
        ArgumentNullException.ThrowIfNull(options);
        var utc = now.UtcDateTime;
        await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);

        // Recompute the last two days of complete and current hours; idempotent, so it tolerates missed runs.
        var from = new DateTime(utc.Year, utc.Month, utc.Day, utc.Hour, 0, 0, DateTimeKind.Utc).AddDays(-2);
        await connection.ExecuteAsync("""
            MERGE insight.MetricHourly AS t
            USING (
                SELECT InstanceId, Metric, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0) AS [Hour],
                       AVG(Value) AS AvgValue, MAX(Value) AS MaxValue, COUNT(*) AS Samples
                FROM insight.MetricSamples WHERE SampledAt >= @from
                GROUP BY InstanceId, Metric, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0)
            ) AS s ON t.InstanceId = s.InstanceId AND t.Metric = s.Metric AND t.[Hour] = s.[Hour]
            WHEN MATCHED THEN UPDATE SET AvgValue = s.AvgValue, MaxValue = s.MaxValue, Samples = s.Samples
            WHEN NOT MATCHED THEN INSERT (InstanceId, Metric, [Hour], AvgValue, MaxValue, Samples) VALUES (s.InstanceId, s.Metric, s.[Hour], s.AvgValue, s.MaxValue, s.Samples);

            MERGE insight.WaitHourly AS t
            USING (
                SELECT InstanceId, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0) AS [Hour], WaitType,
                       SUM(WaitingTasks) AS WaitingTasks, SUM(WaitTimeMs) AS WaitTimeMs, SUM(SignalWaitTimeMs) AS SignalWaitTimeMs
                FROM insight.WaitSamples WHERE SampledAt >= @from
                GROUP BY InstanceId, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0), WaitType
            ) AS s ON t.InstanceId = s.InstanceId AND t.[Hour] = s.[Hour] AND t.WaitType = s.WaitType
            WHEN MATCHED THEN UPDATE SET WaitingTasks = s.WaitingTasks, WaitTimeMs = s.WaitTimeMs, SignalWaitTimeMs = s.SignalWaitTimeMs
            WHEN NOT MATCHED THEN INSERT (InstanceId, [Hour], WaitType, WaitingTasks, WaitTimeMs, SignalWaitTimeMs) VALUES (s.InstanceId, s.[Hour], s.WaitType, s.WaitingTasks, s.WaitTimeMs, s.SignalWaitTimeMs);

            MERGE insight.DiskHourly AS t
            USING (
                SELECT InstanceId, [Hour], Volume, SUM(Reads) AS Reads, SUM(Writes) AS Writes, SUM(BytesRead) AS BytesRead, SUM(BytesWritten) AS BytesWritten,
                       SUM(ReadStallMs) AS ReadStallMs, SUM(WriteStallMs) AS WriteStallMs, SUM(Seconds) AS Seconds
                FROM (
                    SELECT InstanceId, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0) AS [Hour], Volume, SampledAt,
                           SUM(Reads) AS Reads, SUM(Writes) AS Writes, SUM(BytesRead) AS BytesRead, SUM(BytesWritten) AS BytesWritten,
                           SUM(ReadStallMs) AS ReadStallMs, SUM(WriteStallMs) AS WriteStallMs, MAX(IntervalSeconds) AS Seconds
                    FROM insight.FileIoSamples WHERE SampledAt >= @from AND Volume IS NOT NULL AND IntervalSeconds IS NOT NULL
                    GROUP BY InstanceId, Volume, SampledAt
                ) AS samples
                GROUP BY InstanceId, [Hour], Volume
            ) AS s ON t.InstanceId = s.InstanceId AND t.[Hour] = s.[Hour] AND t.Volume = s.Volume
            WHEN MATCHED THEN UPDATE SET Reads = s.Reads, Writes = s.Writes, BytesRead = s.BytesRead, BytesWritten = s.BytesWritten,
                ReadStallMs = s.ReadStallMs, WriteStallMs = s.WriteStallMs, Seconds = s.Seconds
            WHEN NOT MATCHED THEN INSERT (InstanceId, [Hour], Volume, Reads, Writes, BytesRead, BytesWritten, ReadStallMs, WriteStallMs, Seconds)
                VALUES (s.InstanceId, s.[Hour], s.Volume, s.Reads, s.Writes, s.BytesRead, s.BytesWritten, s.ReadStallMs, s.WriteStallMs, s.Seconds);
            """, new { from }, commandTimeout: 600).ConfigureAwait(false);

        var raw = utc.AddDays(-options.RawDays);
        var hourly = utc.AddDays(-options.HourlyDays);
        var events = utc.AddDays(-options.EventDays);
        var resolved = utc.AddDays(-options.ResolvedFindingDays);

        await DeleteInBatchesAsync(connection, "insight.MetricSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.WaitSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.FileIoSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.QueryStatSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.ActiveWaitSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.LongOperations", "LastSeenAt < @cutoff", raw).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.LongTransactions", "LastSeenAt < @cutoff", raw).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.QueryTexts", "LastSeenAt < @cutoff", raw).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.MetricHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.WaitHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.DiskHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.IndexUsageDaily", "[Day] < @cutoff", hourly).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.BlockingEvents", "SampledAt < @cutoff", events).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.Deadlocks", "OccurredAt < @cutoff", events).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.AxBatchRuns", "EndedAt < @cutoff", events).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.AxSequenceSamples", "[Day] < @cutoff", hourly).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.ErrorLogEvents", "LastAt < @cutoff", events).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.Findings", "ResolvedAt < @cutoff", resolved).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.ConfigurationChanges", "DetectedAt < @cutoff", resolved).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.PrivilegeChanges", "DetectedAt < @cutoff", resolved).ConfigureAwait(false);
        await DeleteInBatchesAsync(connection, "insight.OutboundSubmissions", "COALESCE(SentAt, GaveUpAt) < @cutoff", utc.AddDays(-7)).ConfigureAwait(false);
    }

    private static async Task DeleteInBatchesAsync(Microsoft.Data.SqlClient.SqlConnection connection, string table, string predicate, DateTime cutoff)
    {
        int deleted;
        do
        {
            deleted = await connection.ExecuteAsync($"DELETE TOP ({DeleteBatchSize}) FROM {table} WHERE {predicate};", new { cutoff }, commandTimeout: 300)
                .ConfigureAwait(false);
        }
        while (deleted == DeleteBatchSize);
    }
}