using Dapper; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.Data.SqlClient; using SQLInsight.Collectors.Performance; using SQLInsight.Core.Data; using SQLInsight.Core.Instances; using SQLInsight.Reports; using SQLInsight.Repository; namespace SQLInsight.IntegrationTests; /// /// Top SQL against real SQL Server (as the least-privilege agent), and the time window analysis on stored data: /// repository queries, the web page and the PDF. /// [Collection(SqlServerTestGroup.Name)] [Trait("Category", "Integration")] public class TimeWindowAnalysisTests(SqlServerFixture sql) : IAsyncLifetime { private static readonly TimeZoneInfo Copenhagen = TimeZoneInfo.FindSystemTimeZoneById("Europe/Copenhagen"); private readonly string _database = "SQLInsightWindow_" + Guid.NewGuid().ToString("N")[..8]; private InsightRepository _repository = null!; private int _instanceId; public async Task InitializeAsync() { await sql.ExecuteAsSaAsync($"CREATE DATABASE [{_database}];"); _repository = new InsightRepository(sql.DatabaseConnectionString(_database)); await _repository.MigrateAsync(); var info = new InstanceInfo { ServerName = "AX01", Version = SqlServerVersion.Parse("16.0.4135.4"), ProductLevel = "RTM", Edition = "Standard Edition", EngineEdition = EngineEdition.Standard, }; _instanceId = await _repository.UpsertInstanceAsync(@"AX01\AX30", info, "SqlClient", false, DateTimeOffset.UtcNow); } public async Task DisposeAsync() => await sql.ExecuteAsSaAsync($"ALTER DATABASE [{_database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{_database}];"); [Fact] public async Task Collector_turns_query_stats_into_deltas_with_text() { var table = "QsProbe_" + Guid.NewGuid().ToString("N")[..8]; await sql.ExecuteAsSaAsync($"USE [{SqlServerFixture.TestDatabase}]; CREATE TABLE dbo.{table} (Id int NOT NULL PRIMARY KEY, Amount money NOT NULL); INSERT dbo.{table} SELECT TOP (2000) ROW_NUMBER() OVER (ORDER BY (SELECT 1)), 1 FROM sys.all_columns a CROSS JOIN sys.all_columns b;"); var probe = $"SELECT SUM(p.Amount) FROM dbo.{table} p JOIN dbo.{table} q ON q.Id = p.Id WHERE p.Id > 0"; await using var session = await AgentSession.OpenAsync(sql); var collector = new QueryStatsCollector(); var first = await session.CollectAsync>(collector); await Task.Delay(1100); for (var i = 0; i < 5; i++) { await RunInTestDatabaseAsync(probe); } var second = await session.CollectAsync(collector, first); await Task.Delay(1100); for (var i = 0; i < 3; i++) { await RunInTestDatabaseAsync(probe); } var third = await session.CollectAsync(collector, second); Assert.Null(first.Delta); // Compiled after the first collection: all five executions are new. var compiled = Assert.Single(second.Delta!.Queries, q => q.Text?.Contains(table, StringComparison.Ordinal) == true); Assert.Equal(5, compiled.Executions); Assert.True(compiled.WorkerTimeUs > 0); Assert.True(compiled.LogicalReads > 0); Assert.Equal(SqlServerFixture.TestDatabase, compiled.DatabaseName); Assert.StartsWith("0x", compiled.Key, StringComparison.Ordinal); // Same plan: only the difference. Assert.Equal(3, Assert.Single(third.Delta!.Queries, q => q.Key == compiled.Key).Executions); Assert.Contains(third.Snapshot.Queries, q => q.Key == compiled.Key); } [Fact] public async Task Repository_sums_top_sql_per_window_and_never_keeps_it_as_a_snapshot() { var now = DateTimeOffset.UtcNow; await SaveQueryDeltaAsync(now.AddMinutes(-20), ("0xAAA", "SELECT * FROM CUSTTABLE", 10, 4_000_000), ("0xBBB", "SELECT * FROM INVENTTRANS", 1, 9_000_000)); await SaveQueryDeltaAsync(now.AddMinutes(-10), ("0xAAA", "SELECT * FROM CUSTTABLE", 5, 2_000_000)); await SaveQueryDeltaAsync(now.AddHours(-5), ("0xAAA", "SELECT * FROM CUSTTABLE", 100, 100_000_000)); // outside the window var byCpu = await _repository.GetTopQueriesAsync(_instanceId, now.AddHours(-1), now); var byExecutions = await _repository.GetTopQueriesAsync(_instanceId, now.AddHours(-1), now, TopQueryOrder.Executions); Assert.Equal(["0xBBB", "0xAAA"], byCpu.Select(q => q.QueryKey)); var custtable = byCpu.Single(q => q.QueryKey == "0xAAA"); Assert.Equal(15, custtable.Executions); Assert.Equal(6_000_000, custtable.WorkerTimeUs); Assert.Equal("SELECT * FROM CUSTTABLE", custtable.Text); Assert.Equal("0xAAA", byExecutions[0].QueryKey); await using var connection = new SqlConnection(sql.DatabaseConnectionString(_database)); Assert.Equal(0, await connection.ExecuteScalarAsync("SELECT COUNT(*) FROM insight.Snapshots WHERE Kind = 'query-stats' OR Json LIKE '%CUSTTABLE%'")); } [Fact] public async Task Analysis_compares_the_window_with_the_same_hours_in_earlier_weeks() { var to = FloorHour(DateTimeOffset.UtcNow).AddHours(-1); var from = to.AddHours(-1); await SeedWindowAsync(from, to); var analysis = await TimeWindowAnalysis.LoadAsync(_repository, (await _repository.GetInstanceAsync(@"AX01\AX30"))!, from, to, DateTimeOffset.UtcNow); Assert.Equal(WindowResolution.Raw, analysis.Resolution); Assert.True(analysis.HasData); var locks = analysis.Waits.Single(w => w.WaitType == "LCK_M_X"); Assert.Equal(1_000, locks.NormalPerMinute); // 60 s per hour in each earlier week Assert.True(locks.IsUnusual); var cpu = analysis.Metrics.Single(m => m.Metric == Metrics.CpuSqlServer); Assert.Equal(90, cpu.Average); Assert.Equal(30, cpu.NormalAverage); Assert.True(cpu.IsUnusual); Assert.Equal(AssessmentLevel.Act, analysis.Assess(cpu).Level); Assert.Equal("SELECT * FROM SALESLINE", Assert.Single(analysis.TopQueries).Text); Assert.Single(analysis.Blocking); } [Fact] public async Task Page_and_pdf_show_the_window_and_explain_an_empty_one() { var to = FloorHour(DateTimeOffset.UtcNow).AddHours(-1); var from = to.AddHours(-1); await SeedWindowAsync(from, to); await using var factory = new WebApplicationFactory().WithWebHostBuilder(b => { b.UseEnvironment("Development"); b.UseSetting("SQLInsight:Repository:ConnectionString", sql.DatabaseConnectionString(_database)); b.UseSetting("SQLInsight:Web:Authentication", "None"); b.UseSetting("SQLInsight:Web:TimeZone", "Europe/Copenhagen"); }); var client = factory.CreateClient(); var name = Uri.EscapeDataString(@"AX01\AX30"); static string Window(DateTimeOffset f, DateTimeOffset t) => $"fra={Uri.EscapeDataString(TimeWindow.ToInput(f, Copenhagen))}&til={Uri.EscapeDataString(TimeWindow.ToInput(t, Copenhagen))}"; var html = await client.GetStringAsync($"/instanser/{name}/analyse?{Window(from, to)}"); Assert.Contains("SELECT * FROM SALESLINE", html, StringComparison.Ordinal); Assert.Contains("LCK_M_X", html, StringComparison.Ordinal); Assert.Contains("Diskhastighed pr. drev", html, StringComparison.Ordinal); Assert.Contains("class=\"unusual\"", html, StringComparison.Ordinal); Assert.Contains("Hvad betyder det?", html, StringComparison.Ordinal); Assert.Contains("class=\"assess act\"", html, StringComparison.Ordinal); // SQL Server CPU 90 % Assert.Contains("value=\"" + TimeWindow.ToInput(from, Copenhagen) + "\"", html, StringComparison.Ordinal); var empty = await client.GetStringAsync($"/instanser/{name}/analyse?{Window(from.AddDays(-200), to.AddDays(-200))}"); Assert.Contains("Der er ingen målinger i dette tidsrum", empty, StringComparison.Ordinal); var invalid = await client.GetStringAsync($"/instanser/{name}/analyse?{Window(to, from)}"); Assert.Contains("Til skal ligge efter fra.", invalid, StringComparison.Ordinal); var instancePage = await client.GetStringAsync($"/instanser/{name}"); Assert.Contains($"/instanser/{name}/analyse", instancePage, StringComparison.Ordinal); Assert.Contains("Hvad betyder det?", instancePage, StringComparison.Ordinal); Assert.Contains("Memory grants pending", instancePage, StringComparison.Ordinal); using var pdf = await client.GetAsync($"/analyse.pdf?instans={name}&{Window(from, to)}"); Assert.Equal("application/pdf", pdf.Content.Headers.ContentType?.MediaType); Assert.StartsWith("%PDF", System.Text.Encoding.ASCII.GetString(await pdf.Content.ReadAsByteArrayAsync())[..4], StringComparison.Ordinal); } [Fact] public async Task Disk_speed_is_summed_per_drive_and_kept_hourly() { var to = FloorHour(DateTimeOffset.UtcNow).AddHours(-1); var from = to.AddHours(-1); for (var at = from.AddMinutes(5); at <= to; at = at.AddMinutes(5)) { await _repository.SaveCollectedAsync(_instanceId, "file-io", new CumulativeSample( new FileIoSnapshot { CapturedAt = at, SqlServerStartTime = DateTime.UtcNow.AddDays(-1), Files = [] }, new FileIoDelta { From = at.AddMinutes(-5), To = at, Files = [ IoFile("AX30", 1, "ROWS", @"D:\AX30.mdf", reads: 3000, readStall: 90_000, bytesRead: 300 * 1048576L), IoFile("AX30", 2, "LOG", @"L:\AX30_log.ldf", writes: 3000, writeStall: 3_000, bytesWritten: 30 * 1048576L), ], })); } await _repository.RunMaintenanceAsync(new RetentionOptions(), DateTimeOffset.UtcNow); var raw = await _repository.GetDiskActivityAsync(_instanceId, from, to); var hourly = await _repository.GetDiskActivityHourlyAsync(_instanceId, from, to); var analysis = await TimeWindowAnalysis.LoadAsync(_repository, (await _repository.GetInstanceAsync(@"AX01\AX30"))!, from, to, DateTimeOffset.UtcNow); var data = raw.Single(d => d.Volume == @"D:\"); Assert.Equal(1, data.DataFiles); Assert.Equal(30, data.AverageReadMs); Assert.Equal(10, data.ReadIops); // 3000 reads per 300 s Assert.Equal(1, data.ReadMbPerSecond); // 300 MB per 300 s Assert.Equal(1, raw.Single(d => d.Volume == @"L:\").AverageWriteMs); // The sample on the hour belongs to the next hour (as for waits): 11 samples of 300 s in this one. Assert.Equal(11 * 300, hourly.Single(d => d.Volume == @"D:\").Seconds); Assert.True(analysis.Disks.Single(d => d.Window.Volume == @"D:\").IsSlow); Assert.False(analysis.Disks.Single(d => d.Window.Volume == @"L:\").IsUnusual); } private static FileIoStat IoFile(string db, int id, string type, string path, long reads = 0, long readStall = 0, long bytesRead = 0, long writes = 0, long writeStall = 0, long bytesWritten = 0) => new() { DatabaseName = db, FileId = id, FileType = type, PhysicalName = path, Reads = reads, ReadStallMs = readStall, BytesRead = bytesRead, Writes = writes, WriteStallMs = writeStall, BytesWritten = bytesWritten, }; /// A busy hour: high CPU, lock waits and one heavy statement, against quiet same hours four weeks back. private async Task SeedWindowAsync(DateTimeOffset from, DateTimeOffset to) { await using var connection = new SqlConnection(sql.DatabaseConnectionString(_database)); for (var at = from.AddMinutes(5); at <= to; at = at.AddMinutes(5)) { await connection.ExecuteAsync(""" INSERT insight.MetricSamples (InstanceId, Metric, SampledAt, Value) VALUES (@id, @cpu, @at, 90), (@id, @batches, @at, 450); INSERT insight.WaitSamples (InstanceId, SampledAt, WaitType, IntervalSeconds, WaitingTasks, WaitTimeMs, SignalWaitTimeMs) VALUES (@id, @at, 'LCK_M_X', 300, 40, 600000, 0), (@id, @at, 'PAGEIOLATCH_SH', 300, 900, 30000, 100); """, new { id = _instanceId, cpu = Metrics.CpuSqlServer, batches = Metrics.BatchRequestsPerSecond, at = at.UtcDateTime }); } for (var week = 1; week <= WindowBaseline.Weeks; week++) { var hour = from.AddDays(-7 * week).UtcDateTime; await connection.ExecuteAsync(""" INSERT insight.MetricHourly (InstanceId, Metric, [Hour], AvgValue, MaxValue, Samples) VALUES (@id, @cpu, @hour, 30, 40, 12); INSERT insight.WaitHourly (InstanceId, [Hour], WaitType, WaitingTasks, WaitTimeMs, SignalWaitTimeMs) VALUES (@id, @hour, 'LCK_M_X', 10, 60000, 0), (@id, @hour, 'PAGEIOLATCH_SH', 10000, 360000, 1000); """, new { id = _instanceId, cpu = Metrics.CpuSqlServer, hour }); } await connection.ExecuteAsync(""" INSERT insight.BlockingEvents (InstanceId, SampledAt, HeadSessionId, BlockedCount, MaxWaitMs, SessionsJson) VALUES (@id, @at, 55, 7, 42000, '[{"SessionId":55,"Status":"sleeping","ProgramName":"Microsoft Dynamics AX","LoginName":"AXSERVICE"}]'); """, new { id = _instanceId, at = from.AddMinutes(20).UtcDateTime }); await SaveQueryDeltaAsync(from.AddMinutes(30), ("0xCCC", "SELECT * FROM SALESLINE", 4000, 120_000_000)); } private Task SaveQueryDeltaAsync(DateTimeOffset at, params (string Key, string Text, long Executions, long Cpu)[] queries) => _repository.SaveCollectedAsync(_instanceId, "query-stats", new CumulativeSample( new QueryStatsSnapshot { CapturedAt = at, ServerTime = at.UtcDateTime, SqlServerStartTime = DateTime.UtcNow.AddDays(-30), Queries = [] }, new QueryStatsDelta { From = at.AddMinutes(-5), To = at, Queries = [.. queries.Select(q => new QueryStatDelta { Key = q.Key, Text = q.Text, DatabaseName = "AX30", Executions = q.Executions, WorkerTimeUs = q.Cpu, ElapsedTimeUs = q.Cpu * 2, LogicalReads = q.Executions * 100, PhysicalReads = 0, LogicalWrites = 0, })], })); /// Like an application: connected to its own database, so the plan belongs to it. private async Task RunInTestDatabaseAsync(string statement) { await using var connection = new SqlConnection(sql.DatabaseConnectionString(SqlServerFixture.TestDatabase)); await connection.ExecuteScalarAsync(statement); } private static DateTimeOffset FloorHour(DateTimeOffset value) => new(value.Year, value.Month, value.Day, value.Hour, 0, 0, value.Offset); }