using Dapper; namespace SQLInsight.Repository; public sealed class RetentionOptions { /// Raw samples (5-minute waits, per-minute CPU …). public int RawDays { get; set; } = 14; /// Hourly aggregates, for long trends, and the daily samples of Dynamics AX number sequences and RecIds. public int HourlyDays { get; set; } = 400; /// Blocking events, deadlock graphs and completed Dynamics AX batch runs. public int EventDays { get; set; } = 90; /// Resolved findings. public int ResolvedFindingDays { get; set; } = 365; } public sealed partial class InsightRepository { private const int DeleteBatchSize = 5000; /// /// Aggregates recent raw samples to hourly values and deletes data past retention. Deletes run in small batches /// so the repository's own transaction log never has to hold a huge delete. /// public async Task RunMaintenanceAsync(RetentionOptions options, DateTimeOffset now, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(options); var utc = now.UtcDateTime; await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false); // Recompute the last two days of complete and current hours; idempotent, so it tolerates missed runs. var from = new DateTime(utc.Year, utc.Month, utc.Day, utc.Hour, 0, 0, DateTimeKind.Utc).AddDays(-2); await connection.ExecuteAsync(""" MERGE insight.MetricHourly AS t USING ( SELECT InstanceId, Metric, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0) AS [Hour], AVG(Value) AS AvgValue, MAX(Value) AS MaxValue, COUNT(*) AS Samples FROM insight.MetricSamples WHERE SampledAt >= @from GROUP BY InstanceId, Metric, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0) ) AS s ON t.InstanceId = s.InstanceId AND t.Metric = s.Metric AND t.[Hour] = s.[Hour] WHEN MATCHED THEN UPDATE SET AvgValue = s.AvgValue, MaxValue = s.MaxValue, Samples = s.Samples WHEN NOT MATCHED THEN INSERT (InstanceId, Metric, [Hour], AvgValue, MaxValue, Samples) VALUES (s.InstanceId, s.Metric, s.[Hour], s.AvgValue, s.MaxValue, s.Samples); MERGE insight.WaitHourly AS t USING ( SELECT InstanceId, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0) AS [Hour], WaitType, SUM(WaitingTasks) AS WaitingTasks, SUM(WaitTimeMs) AS WaitTimeMs, SUM(SignalWaitTimeMs) AS SignalWaitTimeMs FROM insight.WaitSamples WHERE SampledAt >= @from GROUP BY InstanceId, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0), WaitType ) AS s ON t.InstanceId = s.InstanceId AND t.[Hour] = s.[Hour] AND t.WaitType = s.WaitType WHEN MATCHED THEN UPDATE SET WaitingTasks = s.WaitingTasks, WaitTimeMs = s.WaitTimeMs, SignalWaitTimeMs = s.SignalWaitTimeMs WHEN NOT MATCHED THEN INSERT (InstanceId, [Hour], WaitType, WaitingTasks, WaitTimeMs, SignalWaitTimeMs) VALUES (s.InstanceId, s.[Hour], s.WaitType, s.WaitingTasks, s.WaitTimeMs, s.SignalWaitTimeMs); MERGE insight.DiskHourly AS t USING ( SELECT InstanceId, [Hour], Volume, SUM(Reads) AS Reads, SUM(Writes) AS Writes, SUM(BytesRead) AS BytesRead, SUM(BytesWritten) AS BytesWritten, SUM(ReadStallMs) AS ReadStallMs, SUM(WriteStallMs) AS WriteStallMs, SUM(Seconds) AS Seconds FROM ( SELECT InstanceId, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0) AS [Hour], Volume, SampledAt, SUM(Reads) AS Reads, SUM(Writes) AS Writes, SUM(BytesRead) AS BytesRead, SUM(BytesWritten) AS BytesWritten, SUM(ReadStallMs) AS ReadStallMs, SUM(WriteStallMs) AS WriteStallMs, MAX(IntervalSeconds) AS Seconds FROM insight.FileIoSamples WHERE SampledAt >= @from AND Volume IS NOT NULL AND IntervalSeconds IS NOT NULL GROUP BY InstanceId, Volume, SampledAt ) AS samples GROUP BY InstanceId, [Hour], Volume ) AS s ON t.InstanceId = s.InstanceId AND t.[Hour] = s.[Hour] AND t.Volume = s.Volume WHEN MATCHED THEN UPDATE SET Reads = s.Reads, Writes = s.Writes, BytesRead = s.BytesRead, BytesWritten = s.BytesWritten, ReadStallMs = s.ReadStallMs, WriteStallMs = s.WriteStallMs, Seconds = s.Seconds WHEN NOT MATCHED THEN INSERT (InstanceId, [Hour], Volume, Reads, Writes, BytesRead, BytesWritten, ReadStallMs, WriteStallMs, Seconds) VALUES (s.InstanceId, s.[Hour], s.Volume, s.Reads, s.Writes, s.BytesRead, s.BytesWritten, s.ReadStallMs, s.WriteStallMs, s.Seconds); """, new { from }, commandTimeout: 600).ConfigureAwait(false); var raw = utc.AddDays(-options.RawDays); var hourly = utc.AddDays(-options.HourlyDays); var events = utc.AddDays(-options.EventDays); var resolved = utc.AddDays(-options.ResolvedFindingDays); await DeleteInBatchesAsync(connection, "insight.MetricSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.WaitSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.FileIoSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.QueryStatSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.ActiveWaitSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.LongOperations", "LastSeenAt < @cutoff", raw).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.LongTransactions", "LastSeenAt < @cutoff", raw).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.QueryTexts", "LastSeenAt < @cutoff", raw).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.MetricHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.WaitHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.DiskHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.IndexUsageDaily", "[Day] < @cutoff", hourly).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.BlockingEvents", "SampledAt < @cutoff", events).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.Deadlocks", "OccurredAt < @cutoff", events).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.AxBatchRuns", "EndedAt < @cutoff", events).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.AxSequenceSamples", "[Day] < @cutoff", hourly).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.ErrorLogEvents", "LastAt < @cutoff", events).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.Findings", "ResolvedAt < @cutoff", resolved).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.ConfigurationChanges", "DetectedAt < @cutoff", resolved).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.PrivilegeChanges", "DetectedAt < @cutoff", resolved).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.OutboundSubmissions", "COALESCE(SentAt, GaveUpAt) < @cutoff", utc.AddDays(-7)).ConfigureAwait(false); } private static async Task DeleteInBatchesAsync(Microsoft.Data.SqlClient.SqlConnection connection, string table, string predicate, DateTime cutoff) { int deleted; do { deleted = await connection.ExecuteAsync($"DELETE TOP ({DeleteBatchSize}) FROM {table} WHERE {predicate};", new { cutoff }, commandTimeout: 300) .ConfigureAwait(false); } while (deleted == DeleteBatchSize); } }