sqlinsight-pro/tests/SQLInsight.IntegrationTests/TimeWindowAnalysisTests.cs

using Dapper;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.Data.SqlClient;
using SQLInsight.Collectors.Performance;
using SQLInsight.Core.Data;
using SQLInsight.Core.Instances;
using SQLInsight.Reports;
using SQLInsight.Repository;

namespace SQLInsight.IntegrationTests;

/// <summary>
/// Top SQL against real SQL Server (as the least-privilege agent), and the time window analysis on stored data:
/// repository queries, the web page and the PDF.
/// </summary>
[Collection(SqlServerTestGroup.Name)]
[Trait("Category", "Integration")]
public class TimeWindowAnalysisTests(SqlServerFixture sql) : IAsyncLifetime
{
    private static readonly TimeZoneInfo Copenhagen = TimeZoneInfo.FindSystemTimeZoneById("Europe/Copenhagen");

    private readonly string _database = "SQLInsightWindow_" + Guid.NewGuid().ToString("N")[..8];
    private InsightRepository _repository = null!;
    private int _instanceId;

    public async Task InitializeAsync()
    {
        await sql.ExecuteAsSaAsync($"CREATE DATABASE [{_database}];");
        _repository = new InsightRepository(sql.DatabaseConnectionString(_database));
        await _repository.MigrateAsync();
        var info = new InstanceInfo
        {
            ServerName = "AX01",
            Version = SqlServerVersion.Parse("16.0.4135.4"),
            ProductLevel = "RTM",
            Edition = "Standard Edition",
            EngineEdition = EngineEdition.Standard,
        };
        _instanceId = await _repository.UpsertInstanceAsync(@"AX01\AX30", info, "SqlClient", false, DateTimeOffset.UtcNow);
    }

    public async Task DisposeAsync() =>
        await sql.ExecuteAsSaAsync($"ALTER DATABASE [{_database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{_database}];");

    [Fact]
    public async Task Collector_turns_query_stats_into_deltas_with_text()
    {
        var table = "QsProbe_" + Guid.NewGuid().ToString("N")[..8];
        await sql.ExecuteAsSaAsync($"USE [{SqlServerFixture.TestDatabase}]; CREATE TABLE dbo.{table} (Id int NOT NULL PRIMARY KEY, Amount money NOT NULL); INSERT dbo.{table} SELECT TOP (2000) ROW_NUMBER() OVER (ORDER BY (SELECT 1)), 1 FROM sys.all_columns a CROSS JOIN sys.all_columns b;");
        var probe = $"SELECT SUM(p.Amount) FROM dbo.{table} p JOIN dbo.{table} q ON q.Id = p.Id WHERE p.Id > 0";

        await using var session = await AgentSession.OpenAsync(sql);
        var collector = new QueryStatsCollector();
        var first = await session.CollectAsync<CumulativeSample<QueryStatsSnapshot, QueryStatsDelta>>(collector);
        await Task.Delay(1100);
        for (var i = 0; i < 5; i++)
        {
            await RunInTestDatabaseAsync(probe);
        }

        var second = await session.CollectAsync(collector, first);
        await Task.Delay(1100);
        for (var i = 0; i < 3; i++)
        {
            await RunInTestDatabaseAsync(probe);
        }

        var third = await session.CollectAsync(collector, second);

        Assert.Null(first.Delta);
        // Compiled after the first collection: all five executions are new.
        var compiled = Assert.Single(second.Delta!.Queries, q => q.Text?.Contains(table, StringComparison.Ordinal) == true);
        Assert.Equal(5, compiled.Executions);
        Assert.True(compiled.WorkerTimeUs > 0);
        Assert.True(compiled.LogicalReads > 0);
        Assert.Equal(SqlServerFixture.TestDatabase, compiled.DatabaseName);
        Assert.StartsWith("0x", compiled.Key, StringComparison.Ordinal);
        // Same plan: only the difference.
        Assert.Equal(3, Assert.Single(third.Delta!.Queries, q => q.Key == compiled.Key).Executions);
        Assert.Contains(third.Snapshot.Queries, q => q.Key == compiled.Key);
    }

    [Fact]
    public async Task Repository_sums_top_sql_per_window_and_never_keeps_it_as_a_snapshot()
    {
        var now = DateTimeOffset.UtcNow;
        await SaveQueryDeltaAsync(now.AddMinutes(-20), ("0xAAA", "SELECT * FROM CUSTTABLE", 10, 4_000_000), ("0xBBB", "SELECT * FROM INVENTTRANS", 1, 9_000_000));
        await SaveQueryDeltaAsync(now.AddMinutes(-10), ("0xAAA", "SELECT * FROM CUSTTABLE", 5, 2_000_000));
        await SaveQueryDeltaAsync(now.AddHours(-5), ("0xAAA", "SELECT * FROM CUSTTABLE", 100, 100_000_000)); // outside the window

        var byCpu = await _repository.GetTopQueriesAsync(_instanceId, now.AddHours(-1), now);
        var byExecutions = await _repository.GetTopQueriesAsync(_instanceId, now.AddHours(-1), now, TopQueryOrder.Executions);

        Assert.Equal(["0xBBB", "0xAAA"], byCpu.Select(q => q.QueryKey));
        var custtable = byCpu.Single(q => q.QueryKey == "0xAAA");
        Assert.Equal(15, custtable.Executions);
        Assert.Equal(6_000_000, custtable.WorkerTimeUs);
        Assert.Equal("SELECT * FROM CUSTTABLE", custtable.Text);
        Assert.Equal("0xAAA", byExecutions[0].QueryKey);

        await using var connection = new SqlConnection(sql.DatabaseConnectionString(_database));
        Assert.Equal(0, await connection.ExecuteScalarAsync<int>("SELECT COUNT(*) FROM insight.Snapshots WHERE Kind = 'query-stats' OR Json LIKE '%CUSTTABLE%'"));
    }

    [Fact]
    public async Task Analysis_compares_the_window_with_the_same_hours_in_earlier_weeks()
    {
        var to = FloorHour(DateTimeOffset.UtcNow).AddHours(-1);
        var from = to.AddHours(-1);
        await SeedWindowAsync(from, to);

        var analysis = await TimeWindowAnalysis.LoadAsync(_repository, (await _repository.GetInstanceAsync(@"AX01\AX30"))!, from, to, DateTimeOffset.UtcNow);

        Assert.Equal(WindowResolution.Raw, analysis.Resolution);
        Assert.True(analysis.HasData);
        var locks = analysis.Waits.Single(w => w.WaitType == "LCK_M_X");
        Assert.Equal(1_000, locks.NormalPerMinute); // 60 s per hour in each earlier week
        Assert.True(locks.IsUnusual);
        var cpu = analysis.Metrics.Single(m => m.Metric == Metrics.CpuSqlServer);
        Assert.Equal(90, cpu.Average);
        Assert.Equal(30, cpu.NormalAverage);
        Assert.True(cpu.IsUnusual);
        Assert.Equal(AssessmentLevel.Act, analysis.Assess(cpu).Level);
        Assert.Equal("SELECT * FROM SALESLINE", Assert.Single(analysis.TopQueries).Text);
        Assert.Single(analysis.Blocking);
    }

    [Fact]
    public async Task Page_and_pdf_show_the_window_and_explain_an_empty_one()
    {
        var to = FloorHour(DateTimeOffset.UtcNow).AddHours(-1);
        var from = to.AddHours(-1);
        await SeedWindowAsync(from, to);
        await using var factory = new WebApplicationFactory<Program>().WithWebHostBuilder(b =>
        {
            b.UseEnvironment("Development");
            b.UseSetting("SQLInsight:Repository:ConnectionString", sql.DatabaseConnectionString(_database));
            b.UseSetting("SQLInsight:Web:Authentication", "None");
            b.UseSetting("SQLInsight:Web:TimeZone", "Europe/Copenhagen");
        });
        var client = factory.CreateClient();
        var name = Uri.EscapeDataString(@"AX01\AX30");
        static string Window(DateTimeOffset f, DateTimeOffset t) =>
            $"fra={Uri.EscapeDataString(TimeWindow.ToInput(f, Copenhagen))}&til={Uri.EscapeDataString(TimeWindow.ToInput(t, Copenhagen))}";

        var html = await client.GetStringAsync($"/instanser/{name}/analyse?{Window(from, to)}");
        Assert.Contains("SELECT * FROM SALESLINE", html, StringComparison.Ordinal);
        Assert.Contains("LCK_M_X", html, StringComparison.Ordinal);
        Assert.Contains("Diskhastighed pr. drev", html, StringComparison.Ordinal);
        Assert.Contains("class=\"unusual\"", html, StringComparison.Ordinal);
        Assert.Contains("Hvad betyder det?", html, StringComparison.Ordinal);
        Assert.Contains("class=\"assess act\"", html, StringComparison.Ordinal); // SQL Server CPU 90 %
        Assert.Contains("value=\"" + TimeWindow.ToInput(from, Copenhagen) + "\"", html, StringComparison.Ordinal);

        var empty = await client.GetStringAsync($"/instanser/{name}/analyse?{Window(from.AddDays(-200), to.AddDays(-200))}");
        Assert.Contains("Der er ingen målinger i dette tidsrum", empty, StringComparison.Ordinal);

        var invalid = await client.GetStringAsync($"/instanser/{name}/analyse?{Window(to, from)}");
        Assert.Contains("Til skal ligge efter fra.", invalid, StringComparison.Ordinal);

        var instancePage = await client.GetStringAsync($"/instanser/{name}");
        Assert.Contains($"/instanser/{name}/analyse", instancePage, StringComparison.Ordinal);
        Assert.Contains("Hvad betyder det?", instancePage, StringComparison.Ordinal);
        Assert.Contains("Memory grants pending", instancePage, StringComparison.Ordinal);

        using var pdf = await client.GetAsync($"/analyse.pdf?instans={name}&{Window(from, to)}");
        Assert.Equal("application/pdf", pdf.Content.Headers.ContentType?.MediaType);
        Assert.StartsWith("%PDF", System.Text.Encoding.ASCII.GetString(await pdf.Content.ReadAsByteArrayAsync())[..4], StringComparison.Ordinal);
    }

    [Fact]
    public async Task Disk_speed_is_summed_per_drive_and_kept_hourly()
    {
        var to = FloorHour(DateTimeOffset.UtcNow).AddHours(-1);
        var from = to.AddHours(-1);
        for (var at = from.AddMinutes(5); at <= to; at = at.AddMinutes(5))
        {
            await _repository.SaveCollectedAsync(_instanceId, "file-io", new CumulativeSample<FileIoSnapshot, FileIoDelta>(
                new FileIoSnapshot { CapturedAt = at, SqlServerStartTime = DateTime.UtcNow.AddDays(-1), Files = [] },
                new FileIoDelta
                {
                    From = at.AddMinutes(-5),
                    To = at,
                    Files =
                    [
                        IoFile("AX30", 1, "ROWS", @"D:\AX30.mdf", reads: 3000, readStall: 90_000, bytesRead: 300 * 1048576L),
                        IoFile("AX30", 2, "LOG", @"L:\AX30_log.ldf", writes: 3000, writeStall: 3_000, bytesWritten: 30 * 1048576L),
                    ],
                }));
        }

        await _repository.RunMaintenanceAsync(new RetentionOptions(), DateTimeOffset.UtcNow);
        var raw = await _repository.GetDiskActivityAsync(_instanceId, from, to);
        var hourly = await _repository.GetDiskActivityHourlyAsync(_instanceId, from, to);
        var analysis = await TimeWindowAnalysis.LoadAsync(_repository, (await _repository.GetInstanceAsync(@"AX01\AX30"))!, from, to, DateTimeOffset.UtcNow);

        var data = raw.Single(d => d.Volume == @"D:\");
        Assert.Equal(1, data.DataFiles);
        Assert.Equal(30, data.AverageReadMs);
        Assert.Equal(10, data.ReadIops);       // 3000 reads per 300 s
        Assert.Equal(1, data.ReadMbPerSecond); // 300 MB per 300 s
        Assert.Equal(1, raw.Single(d => d.Volume == @"L:\").AverageWriteMs);
        // The sample on the hour belongs to the next hour (as for waits): 11 samples of 300 s in this one.
        Assert.Equal(11 * 300, hourly.Single(d => d.Volume == @"D:\").Seconds);
        Assert.True(analysis.Disks.Single(d => d.Window.Volume == @"D:\").IsSlow);
        Assert.False(analysis.Disks.Single(d => d.Window.Volume == @"L:\").IsUnusual);
    }

    private static FileIoStat IoFile(string db, int id, string type, string path, long reads = 0, long readStall = 0, long bytesRead = 0,
        long writes = 0, long writeStall = 0, long bytesWritten = 0) => new()
    {
        DatabaseName = db, FileId = id, FileType = type, PhysicalName = path, Reads = reads, ReadStallMs = readStall, BytesRead = bytesRead,
        Writes = writes, WriteStallMs = writeStall, BytesWritten = bytesWritten,
    };

    /// <summary>A busy hour: high CPU, lock waits and one heavy statement, against quiet same hours four weeks back.</summary>
    private async Task SeedWindowAsync(DateTimeOffset from, DateTimeOffset to)
    {
        await using var connection = new SqlConnection(sql.DatabaseConnectionString(_database));
        for (var at = from.AddMinutes(5); at <= to; at = at.AddMinutes(5))
        {
            await connection.ExecuteAsync("""
                INSERT insight.MetricSamples (InstanceId, Metric, SampledAt, Value) VALUES (@id, @cpu, @at, 90), (@id, @batches, @at, 450);
                INSERT insight.WaitSamples (InstanceId, SampledAt, WaitType, IntervalSeconds, WaitingTasks, WaitTimeMs, SignalWaitTimeMs)
                VALUES (@id, @at, 'LCK_M_X', 300, 40, 600000, 0), (@id, @at, 'PAGEIOLATCH_SH', 300, 900, 30000, 100);
                """, new { id = _instanceId, cpu = Metrics.CpuSqlServer, batches = Metrics.BatchRequestsPerSecond, at = at.UtcDateTime });
        }

        for (var week = 1; week <= WindowBaseline.Weeks; week++)
        {
            var hour = from.AddDays(-7 * week).UtcDateTime;
            await connection.ExecuteAsync("""
                INSERT insight.MetricHourly (InstanceId, Metric, [Hour], AvgValue, MaxValue, Samples) VALUES (@id, @cpu, @hour, 30, 40, 12);
                INSERT insight.WaitHourly (InstanceId, [Hour], WaitType, WaitingTasks, WaitTimeMs, SignalWaitTimeMs)
                VALUES (@id, @hour, 'LCK_M_X', 10, 60000, 0), (@id, @hour, 'PAGEIOLATCH_SH', 10000, 360000, 1000);
                """, new { id = _instanceId, cpu = Metrics.CpuSqlServer, hour });
        }

        await connection.ExecuteAsync("""
            INSERT insight.BlockingEvents (InstanceId, SampledAt, HeadSessionId, BlockedCount, MaxWaitMs, SessionsJson)
            VALUES (@id, @at, 55, 7, 42000, '[{"SessionId":55,"Status":"sleeping","ProgramName":"Microsoft Dynamics AX","LoginName":"AXSERVICE"}]');
            """, new { id = _instanceId, at = from.AddMinutes(20).UtcDateTime });
        await SaveQueryDeltaAsync(from.AddMinutes(30), ("0xCCC", "SELECT * FROM SALESLINE", 4000, 120_000_000));
    }

    private Task SaveQueryDeltaAsync(DateTimeOffset at, params (string Key, string Text, long Executions, long Cpu)[] queries) =>
        _repository.SaveCollectedAsync(_instanceId, "query-stats", new CumulativeSample<QueryStatsSnapshot, QueryStatsDelta>(
            new QueryStatsSnapshot { CapturedAt = at, ServerTime = at.UtcDateTime, SqlServerStartTime = DateTime.UtcNow.AddDays(-30), Queries = [] },
            new QueryStatsDelta
            {
                From = at.AddMinutes(-5),
                To = at,
                Queries = [.. queries.Select(q => new QueryStatDelta
                {
                    Key = q.Key, Text = q.Text, DatabaseName = "AX30", Executions = q.Executions, WorkerTimeUs = q.Cpu, ElapsedTimeUs = q.Cpu * 2,
                    LogicalReads = q.Executions * 100, PhysicalReads = 0, LogicalWrites = 0,
                })],
            }));

    /// <summary>Like an application: connected to its own database, so the plan belongs to it.</summary>
    private async Task RunInTestDatabaseAsync(string statement)
    {
        await using var connection = new SqlConnection(sql.DatabaseConnectionString(SqlServerFixture.TestDatabase));
        await connection.ExecuteScalarAsync(statement);
    }

    private static DateTimeOffset FloorHour(DateTimeOffset value) => new(value.Year, value.Month, value.Day, value.Hour, 0, 0, value.Offset);
}