A five-minute collector reads important error log entries from the best
source the agent's rights allow: the error log itself (xp_readerrorlog,
only with securityadmin), the default trace's ErrorLog and Audit Login
Failed events (optional ALTER TRACE), or the exception and connectivity
ring buffers (VIEW SERVER STATE: numbers and severity, failed logins).
Repetitions are summed per error; the watermark is the newest entry
read, because the default trace is buffered. Stored in
insight.ErrorLogEvents (migration 0013, EventDays), resumed after restart.
Rules SQLI-LOG-001..006: corruption 823/824/825, I/O over 15 s (833),
severe errors (17+), login failure bursts (configurable), stack dumps and
AG errors; rules needing the log text are "not measurable" with the
right to grant. Page /instanser/{navn}/fejllog (menu Overvågning) and
GET /api/v1/instances/{name}/error-log with messages redacted.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
129 lines
6.4 KiB
C#
129 lines
6.4 KiB
C#
using Microsoft.AspNetCore.Hosting;
|
|
using Microsoft.AspNetCore.Mvc.Testing;
|
|
using Microsoft.Data.SqlClient;
|
|
using SQLInsight.Collectors.Instance;
|
|
using SQLInsight.Core.Data;
|
|
using SQLInsight.Core.Instances;
|
|
using SQLInsight.Repository;
|
|
|
|
namespace SQLInsight.IntegrationTests;
|
|
|
|
/// <summary>The error log as the least-privilege agent sees it: ring buffer without ALTER TRACE, default trace with it.</summary>
|
|
[Collection(SqlServerTestGroup.Name)]
|
|
[Trait("Category", "Integration")]
|
|
public class ErrorLogTests(SqlServerFixture sql)
|
|
{
|
|
private static async Task<ErrorLogSnapshot> UntilAsync(AgentSession session, ErrorLogCollector collector, ErrorLogSnapshot previous, Func<ErrorLogSnapshot, bool> done,
|
|
int seconds = 30)
|
|
{
|
|
var latest = previous;
|
|
// SQL Server writes the default trace in buffers: events can take a while to become readable.
|
|
for (var i = 0; i < seconds; i++)
|
|
{
|
|
await Task.Delay(1000);
|
|
latest = await session.CollectAsync(collector, latest with { Recent = latest.Recent });
|
|
if (done(latest))
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
return latest;
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Without_alter_trace_failed_logins_come_from_the_ring_buffer_with_the_reason()
|
|
{
|
|
await using var session = await AgentSession.OpenAsync(sql);
|
|
var collector = new ErrorLogCollector();
|
|
var first = await session.CollectAsync<ErrorLogSnapshot>(collector);
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
var wrong = new SqlConnectionStringBuilder(sql.AgentConnectionString) { Password = "forkert-" + i, Pooling = false };
|
|
await using var connection = new SqlConnection(wrong.ConnectionString);
|
|
await Assert.ThrowsAsync<SqlException>(() => connection.OpenAsync());
|
|
}
|
|
|
|
var latest = await UntilAsync(session, collector, first, s => s.Recent.Any(g => g.Category == ErrorLogCategory.LoginFailed && g.Count >= 3));
|
|
|
|
Assert.Equal(ErrorLogSource.RingBuffer, first.Source);
|
|
Assert.Contains("ALTER TRACE", first.Coverage, StringComparison.Ordinal);
|
|
var logins = Assert.Single(latest.Recent, g => g.Category == ErrorLogCategory.LoginFailed);
|
|
Assert.True(logins.Count >= 3);
|
|
Assert.Contains("state 8", logins.Sample, StringComparison.Ordinal);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task With_alter_trace_the_error_log_and_failed_logins_are_read_from_the_default_trace()
|
|
{
|
|
await sql.ExecuteAsSaAsync($"GRANT ALTER TRACE TO [{SqlServerFixture.AgentLogin}];");
|
|
try
|
|
{
|
|
await using var session = await AgentSession.OpenAsync(sql);
|
|
var collector = new ErrorLogCollector(loginFailureBurst: 3);
|
|
var first = await session.CollectAsync<ErrorLogSnapshot>(collector);
|
|
await Assert.ThrowsAsync<SqlException>(() => sql.ExecuteAsSaAsync("RAISERROR (N'SQLInsight test written to the error log', 20, 1) WITH LOG;"));
|
|
for (var i = 0; i < 4; i++)
|
|
{
|
|
var wrong = new SqlConnectionStringBuilder(sql.AgentConnectionString) { Password = "forkert-" + i, Pooling = false };
|
|
await using var connection = new SqlConnection(wrong.ConnectionString);
|
|
await Assert.ThrowsAsync<SqlException>(() => connection.OpenAsync());
|
|
}
|
|
|
|
var latest = await UntilAsync(session, collector, first,
|
|
s => s.Recent.Any(g => g.Error == 50000) && s.Recent.Any(g => g.Category == ErrorLogCategory.LoginFailed && g.Count >= 4), seconds: 120);
|
|
|
|
Assert.Equal(ErrorLogSource.DefaultTrace, latest.Source);
|
|
Assert.Contains("securityadmin", latest.Coverage, StringComparison.Ordinal);
|
|
var logged = Assert.Single(latest.Recent, g => g.Error == 50000);
|
|
Assert.Contains("SQLInsight test written to the error log", logged.Sample, StringComparison.Ordinal);
|
|
Assert.True(latest.Recent.Single(g => g.Category == ErrorLogCategory.LoginFailed).Count >= 4);
|
|
}
|
|
finally
|
|
{
|
|
await sql.ExecuteAsSaAsync($"REVOKE ALTER TRACE FROM [{SqlServerFixture.AgentLogin}];");
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Page_and_api_show_events_with_the_sample_left_out_of_the_api()
|
|
{
|
|
var database = "SQLInsightLog_" + Guid.NewGuid().ToString("N")[..8];
|
|
await sql.ExecuteAsSaAsync($"CREATE DATABASE [{database}];");
|
|
try
|
|
{
|
|
var repository = new InsightRepository(sql.DatabaseConnectionString(database));
|
|
await repository.MigrateAsync();
|
|
var now = DateTimeOffset.UtcNow;
|
|
var id = await repository.UpsertInstanceAsync("prod-sql01", new InstanceInfo
|
|
{
|
|
ServerName = "SQL01", Version = SqlServerVersion.Parse("16.0.4135.4"), ProductLevel = "RTM", Edition = "Standard Edition", EngineEdition = EngineEdition.Standard,
|
|
}, "SqlClient", false, now);
|
|
var groups = new[] { new ErrorLogGroup(ErrorLogCategory.Corruption, 824, 24, 2, now.AddMinutes(-4), now.AddMinutes(-1), "incorrect checksum in AX.mdf") };
|
|
await repository.SaveCollectedAsync(id, "error-log", new ErrorLogSnapshot
|
|
{
|
|
CapturedAt = now, Source = ErrorLogSource.DefaultTrace, ReadUntil = now, Groups = groups, Recent = groups,
|
|
});
|
|
|
|
await using var factory = new WebApplicationFactory<Program>().WithWebHostBuilder(b =>
|
|
{
|
|
b.UseEnvironment("Development");
|
|
b.UseSetting("SQLInsight:Repository:ConnectionString", sql.DatabaseConnectionString(database));
|
|
b.UseSetting("SQLInsight:Web:Authentication", "None");
|
|
});
|
|
var client = factory.CreateClient();
|
|
var page = await client.GetStringAsync("/instanser/prod-sql01/fejllog");
|
|
var api = await client.GetStringAsync("/api/v1/instances/prod-sql01/error-log");
|
|
|
|
Assert.Contains("Beskadigede sider", page, StringComparison.Ordinal);
|
|
Assert.Contains("incorrect checksum", page, StringComparison.Ordinal);
|
|
Assert.Contains("824", api, StringComparison.Ordinal);
|
|
Assert.Contains("[udeladt]", api, StringComparison.Ordinal);
|
|
Assert.DoesNotContain("incorrect checksum", api, StringComparison.Ordinal);
|
|
}
|
|
finally
|
|
{
|
|
await sql.ExecuteAsSaAsync($"ALTER DATABASE [{database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{database}];");
|
|
}
|
|
}
|
|
}
|