using Dapper;
namespace SQLInsight.Repository;
public sealed class RetentionOptions
{
/// Raw samples (5-minute waits, per-minute CPU …).
public int RawDays { get; set; } = 14;
/// Hourly aggregates, for long trends, and the daily samples of Dynamics AX number sequences and RecIds.
public int HourlyDays { get; set; } = 400;
/// Blocking events, deadlock graphs and completed Dynamics AX batch runs.
public int EventDays { get; set; } = 90;
/// Resolved findings.
public int ResolvedFindingDays { get; set; } = 365;
}
public sealed partial class InsightRepository
{
private const int DeleteBatchSize = 5000;
///
/// Aggregates recent raw samples to hourly values and deletes data past retention. Deletes run in small batches
/// so the repository's own transaction log never has to hold a huge delete.
///
public async Task RunMaintenanceAsync(RetentionOptions options, DateTimeOffset now, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
var utc = now.UtcDateTime;
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
// Recompute the last two days of complete and current hours; idempotent, so it tolerates missed runs.
var from = new DateTime(utc.Year, utc.Month, utc.Day, utc.Hour, 0, 0, DateTimeKind.Utc).AddDays(-2);
await connection.ExecuteAsync("""
MERGE insight.MetricHourly AS t
USING (
SELECT InstanceId, Metric, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0) AS [Hour],
AVG(Value) AS AvgValue, MAX(Value) AS MaxValue, COUNT(*) AS Samples
FROM insight.MetricSamples WHERE SampledAt >= @from
GROUP BY InstanceId, Metric, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0)
) AS s ON t.InstanceId = s.InstanceId AND t.Metric = s.Metric AND t.[Hour] = s.[Hour]
WHEN MATCHED THEN UPDATE SET AvgValue = s.AvgValue, MaxValue = s.MaxValue, Samples = s.Samples
WHEN NOT MATCHED THEN INSERT (InstanceId, Metric, [Hour], AvgValue, MaxValue, Samples) VALUES (s.InstanceId, s.Metric, s.[Hour], s.AvgValue, s.MaxValue, s.Samples);
MERGE insight.WaitHourly AS t
USING (
SELECT InstanceId, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0) AS [Hour], WaitType,
SUM(WaitingTasks) AS WaitingTasks, SUM(WaitTimeMs) AS WaitTimeMs, SUM(SignalWaitTimeMs) AS SignalWaitTimeMs
FROM insight.WaitSamples WHERE SampledAt >= @from
GROUP BY InstanceId, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0), WaitType
) AS s ON t.InstanceId = s.InstanceId AND t.[Hour] = s.[Hour] AND t.WaitType = s.WaitType
WHEN MATCHED THEN UPDATE SET WaitingTasks = s.WaitingTasks, WaitTimeMs = s.WaitTimeMs, SignalWaitTimeMs = s.SignalWaitTimeMs
WHEN NOT MATCHED THEN INSERT (InstanceId, [Hour], WaitType, WaitingTasks, WaitTimeMs, SignalWaitTimeMs) VALUES (s.InstanceId, s.[Hour], s.WaitType, s.WaitingTasks, s.WaitTimeMs, s.SignalWaitTimeMs);
MERGE insight.DiskHourly AS t
USING (
SELECT InstanceId, [Hour], Volume, SUM(Reads) AS Reads, SUM(Writes) AS Writes, SUM(BytesRead) AS BytesRead, SUM(BytesWritten) AS BytesWritten,
SUM(ReadStallMs) AS ReadStallMs, SUM(WriteStallMs) AS WriteStallMs, SUM(Seconds) AS Seconds
FROM (
SELECT InstanceId, DATEADD(hour, DATEDIFF(hour, 0, SampledAt), 0) AS [Hour], Volume, SampledAt,
SUM(Reads) AS Reads, SUM(Writes) AS Writes, SUM(BytesRead) AS BytesRead, SUM(BytesWritten) AS BytesWritten,
SUM(ReadStallMs) AS ReadStallMs, SUM(WriteStallMs) AS WriteStallMs, MAX(IntervalSeconds) AS Seconds
FROM insight.FileIoSamples WHERE SampledAt >= @from AND Volume IS NOT NULL AND IntervalSeconds IS NOT NULL
GROUP BY InstanceId, Volume, SampledAt
) AS samples
GROUP BY InstanceId, [Hour], Volume
) AS s ON t.InstanceId = s.InstanceId AND t.[Hour] = s.[Hour] AND t.Volume = s.Volume
WHEN MATCHED THEN UPDATE SET Reads = s.Reads, Writes = s.Writes, BytesRead = s.BytesRead, BytesWritten = s.BytesWritten,
ReadStallMs = s.ReadStallMs, WriteStallMs = s.WriteStallMs, Seconds = s.Seconds
WHEN NOT MATCHED THEN INSERT (InstanceId, [Hour], Volume, Reads, Writes, BytesRead, BytesWritten, ReadStallMs, WriteStallMs, Seconds)
VALUES (s.InstanceId, s.[Hour], s.Volume, s.Reads, s.Writes, s.BytesRead, s.BytesWritten, s.ReadStallMs, s.WriteStallMs, s.Seconds);
""", new { from }, commandTimeout: 600).ConfigureAwait(false);
var raw = utc.AddDays(-options.RawDays);
var hourly = utc.AddDays(-options.HourlyDays);
var events = utc.AddDays(-options.EventDays);
var resolved = utc.AddDays(-options.ResolvedFindingDays);
await DeleteInBatchesAsync(connection, "insight.MetricSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.WaitSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.FileIoSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.QueryStatSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.ActiveWaitSamples", "SampledAt < @cutoff", raw).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.LongOperations", "LastSeenAt < @cutoff", raw).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.LongTransactions", "LastSeenAt < @cutoff", raw).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.QueryTexts", "LastSeenAt < @cutoff", raw).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.MetricHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.WaitHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.DiskHourly", "[Hour] < @cutoff", hourly).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.IndexUsageDaily", "[Day] < @cutoff", hourly).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.BlockingEvents", "SampledAt < @cutoff", events).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.Deadlocks", "OccurredAt < @cutoff", events).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.AxBatchRuns", "EndedAt < @cutoff", events).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.AxSequenceSamples", "[Day] < @cutoff", hourly).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.ErrorLogEvents", "LastAt < @cutoff", events).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.Findings", "ResolvedAt < @cutoff", resolved).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.ConfigurationChanges", "DetectedAt < @cutoff", resolved).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.PrivilegeChanges", "DetectedAt < @cutoff", resolved).ConfigureAwait(false);
await DeleteInBatchesAsync(connection, "insight.OutboundSubmissions", "COALESCE(SentAt, GaveUpAt) < @cutoff", utc.AddDays(-7)).ConfigureAwait(false);
}
private static async Task DeleteInBatchesAsync(Microsoft.Data.SqlClient.SqlConnection connection, string table, string predicate, DateTime cutoff)
{
int deleted;
do
{
deleted = await connection.ExecuteAsync($"DELETE TOP ({DeleteBatchSize}) FROM {table} WHERE {predicate};", new { cutoff }, commandTimeout: 300)
.ConfigureAwait(false);
}
while (deleted == DeleteBatchSize);
}
}