Søren Eggert Lundsteen Olsen
25ecdbc3e7
Migration 0018_windows_servers_synthetic_checks: - insight.WindowsServers (SQL Server hosts kept in step by the agent, servers added in SQLInsight Web), ServerPerformance (per minute, one row per sample), ServerDiskSamples (forecast history), ServerEvents (grouped like ErrorLogEvents), ServerSnapshots (latest per area). - insight.SyntheticChecks (SQL login password sealed for the agent, as in 1.3) and SyntheticCheckResults. - insight.ServerFindings and ServerRuleStates, scoped to a server or a check. - AgentRequests.TargetId and the kinds ServerTest and SyntheticCheckRun. - SQLInsight Web gets INSERT and column UPDATE only; host, name and kind are fixed, nothing the agent records can be written, no DELETE. Findings are opened, updated and resolved like instance findings; a round where a rule is not measurable keeps what was found open. Retention for samples, disk history, events, check results and resolved findings (RunServerMaintenanceAsync). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
252 lines
17 KiB
C#
252 lines
17 KiB
C#
using Microsoft.Data.SqlClient;
|
|
using SQLInsight.Core.Findings;
|
|
using SQLInsight.Core.Servers;
|
|
using SQLInsight.Core.SyntheticChecks;
|
|
using SQLInsight.Repository;
|
|
using SQLInsight.Rules.Engine;
|
|
|
|
namespace SQLInsight.IntegrationTests;
|
|
|
|
/// <summary>
|
|
/// Windows servers and synthetic checks in the repository (0018): what SQLInsight Web may change with its read-only
|
|
/// login, the SQL Server hosts kept in step by the agent, samples, events, check results, findings and retention.
|
|
/// </summary>
|
|
[Collection(SqlServerTestGroup.Name)]
|
|
[Trait("Category", "Integration")]
|
|
public class ServerRepositoryTests(SqlServerFixture sql) : IAsyncLifetime
|
|
{
|
|
private readonly string _database = "SQLInsightSrv_" + Guid.NewGuid().ToString("N")[..8];
|
|
private InsightRepository _repository = null!;
|
|
private string _webConnectionString = string.Empty;
|
|
private static readonly DateTimeOffset Now = DateTimeOffset.UtcNow;
|
|
|
|
public async Task InitializeAsync()
|
|
{
|
|
await sql.ExecuteAsSaAsync($"CREATE DATABASE [{_database}];");
|
|
_repository = new InsightRepository(sql.DatabaseConnectionString(_database));
|
|
await _repository.MigrateAsync();
|
|
}
|
|
|
|
public Task DisposeAsync() => sql.ExecuteAsSaAsync($"ALTER DATABASE [{_database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{_database}];");
|
|
|
|
private async Task<InsightRepository> ReadOnlyRepositoryAsync()
|
|
{
|
|
var login = "web_" + Guid.NewGuid().ToString("N")[..8];
|
|
const string password = "Read-Only-Login-1!";
|
|
await sql.ExecuteAsSaAsync($"""
|
|
CREATE LOGIN [{login}] WITH PASSWORD = N'{password}', CHECK_POLICY = OFF;
|
|
USE [{_database}]; CREATE USER [{login}] FOR LOGIN [{login}]; ALTER ROLE db_datareader ADD MEMBER [{login}];
|
|
""");
|
|
_webConnectionString = new SqlConnectionStringBuilder(sql.DatabaseConnectionString(_database)) { UserID = login, Password = password, IntegratedSecurity = false }
|
|
.ConnectionString;
|
|
return new InsightRepository(_webConnectionString);
|
|
}
|
|
|
|
private async Task ExecuteAsWebAsync(string sqlText)
|
|
{
|
|
await using var connection = new SqlConnection(_webConnectionString);
|
|
await connection.OpenAsync();
|
|
await using var command = connection.CreateCommand();
|
|
command.CommandText = sqlText;
|
|
await command.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
private static SyntheticCheckInput Check(string name, int? serverId = null, byte[]? secret = null) =>
|
|
new(name, SyntheticCheckKind.Tcp, serverId, new SyntheticCheckSettings { Host = "aos01", Port = 2712, ServiceName = serverId is null ? null : "AOS60$01" }, secret,
|
|
secret is null ? null : "key-1", 300, 15);
|
|
|
|
[Fact]
|
|
public async Task Web_login_can_add_and_change_servers_and_checks_but_nothing_the_agent_records()
|
|
{
|
|
var web = await ReadOnlyRepositoryAsync();
|
|
|
|
var server = await web.AddWindowsServerAsync(new NewWindowsServer("AOS01", "AOS01.firma.local", ServerRoles.Application, ["AOS60$01"], []), @"FIRMA\anna", Now);
|
|
Assert.True(await web.UpdateWindowsServerAsync(server, ServerRoles.Application, ["AOS60$01", "W3SVC"], ["MyBatch"], @"FIRMA\anna", Now));
|
|
Assert.True(await web.SetWindowsServerPausedAsync(server, true, @"FIRMA\anna", Now));
|
|
var check = await web.AddSyntheticCheckAsync(Check("AOS01 port", server, [1, 2, 3]), @"FIRMA\anna", Now);
|
|
Assert.True(await web.UpdateSyntheticCheckAsync(check, Check("AOS01 AOS-port", server) with { IntervalSeconds = 120 }, @"FIRMA\anna", Now));
|
|
Assert.True(await web.SetSyntheticCheckPausedAsync(check, true, @"FIRMA\anna", Now));
|
|
var request = await web.AddServerRequestAsync(ServerRequestKind.ServerTest, server, @"FIRMA\anna", Now);
|
|
|
|
var row = (await _repository.GetWindowsServerAsync(server))!;
|
|
Assert.Equal(("aos01.firma.local", ServerRoles.Application, true, ServerSources.Web), (row.Host, row.Role, row.Paused, row.Source));
|
|
Assert.Equal(["AOS60$01", "W3SVC"], row.Watched);
|
|
Assert.Equal(["MyBatch"], row.Ignored);
|
|
var checkRow = (await _repository.GetSyntheticCheckAsync(check))!;
|
|
Assert.Equal(("AOS01 AOS-port", 120, true, "AOS01"), (checkRow.Name, checkRow.IntervalSeconds, checkRow.HasSecret, checkRow.ServerName));
|
|
Assert.Equal("AOS60$01", checkRow.Settings.ServiceName);
|
|
Assert.Equal(server, await web.GetRequestTargetAsync(request));
|
|
Assert.Equal(request, (await web.GetLatestServerRequestAsync(ServerRequestKind.ServerTest, server))!.RequestId);
|
|
|
|
// What only the agent does is refused for the web login.
|
|
await Assert.ThrowsAsync<SqlException>(() => web.RecordServerContactAsync(server, new ServerOsInfo { Caption = "x" }, "DCOM", null, Now));
|
|
await Assert.ThrowsAsync<SqlException>(() => web.SaveServerPerformanceAsync(server, new ServerPerformanceSample(Now, 1, 1, 1, 1, 1)));
|
|
await Assert.ThrowsAsync<SqlException>(() => web.RecordSyntheticCheckResultAsync(check, new SyntheticCheckResult { At = Now, Success = true, Message = "ok" }));
|
|
await Assert.ThrowsAsync<SqlException>(() => web.RecordServerAreasAsync(server, []));
|
|
await Assert.ThrowsAsync<SqlException>(() => web.SaveServerSnapshotAsync(server, ServerArea.Services, new ServerServiceSnapshot { CapturedAt = Now }, Now));
|
|
await Assert.ThrowsAsync<SqlException>(() => ExecuteAsWebAsync($"UPDATE insight.WindowsServers SET Host = N'evil' WHERE ServerId = {server}"));
|
|
await Assert.ThrowsAsync<SqlException>(() => ExecuteAsWebAsync($"UPDATE insight.SyntheticChecks SET Kind = 'Sql' WHERE CheckId = {check}"));
|
|
await Assert.ThrowsAsync<SqlException>(() => ExecuteAsWebAsync($"DELETE FROM insight.SyntheticChecks WHERE CheckId = {check}"));
|
|
await Assert.ThrowsAsync<SqlException>(() => ExecuteAsWebAsync("DELETE FROM insight.ServerFindings"));
|
|
|
|
Assert.True(await web.RemoveSyntheticCheckAsync(check, @"FIRMA\anna", Now));
|
|
Assert.Equal(ConfigurationChangeOutcome.Changed, await web.RemoveWindowsServerAsync(server, @"FIRMA\anna", Now));
|
|
Assert.Empty(await _repository.GetWindowsServersAsync());
|
|
Assert.Empty(await _repository.GetSyntheticChecksAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Names_and_hosts_are_unique_among_servers_in_use()
|
|
{
|
|
await _repository.AddWindowsServerAsync(new NewWindowsServer("FIL01", "fil01", ServerRoles.File, [], []), null, Now);
|
|
|
|
var host = await Assert.ThrowsAsync<SqlException>(() => _repository.AddWindowsServerAsync(new NewWindowsServer("Fil", "FIL01", ServerRoles.File, [], []), null, Now));
|
|
var name = await Assert.ThrowsAsync<SqlException>(() => _repository.AddWindowsServerAsync(new NewWindowsServer("FIL01", "fil02", ServerRoles.File, [], []), null, Now));
|
|
Assert.True(host.Number is 2601 or 2627);
|
|
Assert.True(name.Number is 2601 or 2627);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Sql_server_hosts_follow_the_instances_and_keep_their_settings()
|
|
{
|
|
await _repository.AddWindowsServerAsync(new NewWindowsServer("SQL02 (manuelt)", "sql02.firma.local", ServerRoles.Sql, [], []), null, Now);
|
|
await _repository.SyncSqlHostServersAsync([("sql01.firma.local", "SQL01"), ("SQL02.firma.local", "SQL02"), ("sql01.firma.local", "SQL01")], Now);
|
|
|
|
var servers = await _repository.GetWindowsServersAsync();
|
|
Assert.Equal(["SQL01", "SQL02 (manuelt)"], servers.Select(s => s.Name).ToList());
|
|
var sql01 = servers.Single(s => s.Name == "SQL01");
|
|
Assert.True(sql01.IsSqlHost);
|
|
Assert.Equal(ServerRoles.Sql, sql01.Role);
|
|
await _repository.UpdateWindowsServerAsync(sql01.ServerId, ServerRoles.Sql, ["MSSQLSERVER"], [], "anna", Now);
|
|
|
|
Assert.Equal(ConfigurationChangeOutcome.FromConfigFile, await _repository.RemoveWindowsServerAsync(sql01.ServerId, "anna", Now));
|
|
await _repository.SyncSqlHostServersAsync([("sql01.firma.local", "SQL01")], Now);
|
|
Assert.Equal(["MSSQLSERVER"], (await _repository.GetWindowsServerAsync(sql01.ServerId))!.Watched);
|
|
|
|
await _repository.SyncSqlHostServersAsync([], Now);
|
|
Assert.Equal(["SQL02 (manuelt)"], (await _repository.GetWindowsServersAsync()).Select(s => s.Name).ToList());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Samples_disk_history_events_and_snapshots_round_trip()
|
|
{
|
|
var id = await _repository.AddWindowsServerAsync(new NewWindowsServer("AOS01", "aos01", ServerRoles.Application, [], []), null, Now);
|
|
await _repository.RecordServerContactAsync(id, new ServerOsInfo { Caption = "Windows Server 2019", Version = "10.0.17763", TotalMemoryMb = 16384, LogicalProcessors = 4,
|
|
LastBootUpTime = Now.AddDays(-3) }, "WS-Man", [new ServerAreaStatus(ServerArea.Events, false, "Adgang nægtet", ServerArea.Right(ServerArea.Events))], Now);
|
|
await _repository.RecordServerErrorAsync(id, "RPC-serveren er ikke tilgængelig.", Now.AddMinutes(-5));
|
|
for (var m = 0; m < 5; m++)
|
|
{
|
|
await _repository.SaveServerPerformanceAsync(id, new ServerPerformanceSample(Now.AddMinutes(-m), 10 + m, 4000, 16384, 3, 50));
|
|
}
|
|
|
|
await _repository.SaveServerPerformanceAsync(id, new ServerPerformanceSample(Now, 99, 1, 1, 1, 1)); // same minute: kept once
|
|
for (var h = 0; h < 60; h++)
|
|
{
|
|
await _repository.SaveServerDisksAsync(id, Now.AddHours(-h), [new ServerDisk(@"D:\", "Data", "NTFS", 100, 20 + (h * 0.5)), new ServerDisk(@"C:\", null, "NTFS", 80, 40)]);
|
|
}
|
|
|
|
var (groups, _) = ServerEventAnalysis.Group([new ServerEventEntry("System", "disk", 153, 2, Now.AddMinutes(-3), "Disk 1 svarede ikke")], []);
|
|
await _repository.SaveServerEventsAsync(id, new ServerEventSnapshot { CapturedAt = Now, ReadUntil = Now, Groups = groups, Recent = groups });
|
|
await _repository.SaveServerSnapshotAsync(id, ServerArea.Services, new ServerServiceSnapshot { CapturedAt = Now, Services = [new ServerService("W3SVC", "IIS", "Running", "Auto")] }, Now);
|
|
|
|
var row = (await _repository.GetWindowsServerAsync(id))!;
|
|
Assert.True(row.Reachable);
|
|
Assert.Equal(("WS-Man", "Windows Server 2019", 4), (row.Protocol, row.OperatingSystem, row.LogicalProcessors));
|
|
Assert.Equal("Adgang nægtet", Assert.Single(row.Areas).Detail);
|
|
var performance = await _repository.GetServerPerformanceAsync(id, Now.AddHours(-1));
|
|
Assert.Equal(5, performance.Count);
|
|
Assert.Equal(10, performance[^1].CpuPercent!.Value, 3);
|
|
var history = await _repository.GetServerDiskHistoryAsync(id, Now.AddDays(-14));
|
|
Assert.Equal(2, history.Count);
|
|
Assert.InRange(history[@"D:\"].Count, 60, 61);
|
|
var forecast = DiskForecaster.Forecast(@"D:\", history[@"D:\"], Now);
|
|
Assert.Equal(20 / 12d, forecast.DaysUntilFull!.Value, 1); // 12 GB a day, 20 GB free
|
|
var events = await _repository.GetServerEventsAsync(id, Now.AddDays(-1), Now);
|
|
Assert.Equal(("disk", 153, "Disk", "Disk 1 svarede ikke"), (events[0].Source, events[0].EventId, events[0].Category, events[0].Sample));
|
|
var services = await _repository.GetServerSnapshotAsync<ServerServiceSnapshot>(id, ServerArea.Services);
|
|
Assert.Equal("W3SVC", Assert.Single(services!.Services).Name);
|
|
}
|
|
|
|
private static RuleResult Found(string ruleId, params Finding[] findings) =>
|
|
new() { RuleId = ruleId, RuleTitle = ruleId, Outcome = findings.Length == 0 ? RuleOutcome.Passed : RuleOutcome.Findings, Findings = findings };
|
|
|
|
private static Finding Finding(string ruleId, string subject, Severity severity = Severity.High) => new()
|
|
{
|
|
RuleId = ruleId, Subject = subject, Severity = severity, Title = "t " + subject, Explanation = "e", Recommendation = "r",
|
|
Evidence = [new Evidence("Eksempel", "\\\\fil01\\hemmeligt", Sensitive: true)],
|
|
};
|
|
|
|
[Fact]
|
|
public async Task Findings_open_update_resolve_per_server_and_check_and_survive_a_not_measurable_round()
|
|
{
|
|
var server = await _repository.AddWindowsServerAsync(new NewWindowsServer("AOS01", "aos01", ServerRoles.Application, [], []), null, Now);
|
|
var check = await _repository.AddSyntheticCheckAsync(Check("AOS port"), null, Now);
|
|
|
|
var first = await _repository.SaveScopedRuleResultsAsync(FindingScope.Server, server,
|
|
[Found("SQLI-WIN-005", Finding("SQLI-WIN-005", @"disk: D:\"), Finding("SQLI-WIN-005", @"disk: E:\"))], Now);
|
|
await _repository.SaveScopedRuleResultsAsync(FindingScope.Check, check, [Found("SQLI-SYN-001", Finding("SQLI-SYN-001", "check"))], Now);
|
|
Assert.Equal(2, first.Opened.Count);
|
|
|
|
// Not measurable (e.g. rights removed): the findings stay open.
|
|
await _repository.SaveScopedRuleResultsAsync(FindingScope.Server, server,
|
|
[new RuleResult { RuleId = "SQLI-WIN-005", RuleTitle = "x", Outcome = RuleOutcome.NotMeasurable, Reason = "Adgang nægtet" }], Now.AddMinutes(1));
|
|
Assert.Equal(2, (await _repository.GetServerFindingsAsync(FindingScope.Server, server)).Count);
|
|
Assert.Equal("Adgang nægtet", Assert.Single(await _repository.GetServerRuleStatesAsync(FindingScope.Server, server)).Reason);
|
|
|
|
var later = await _repository.SaveScopedRuleResultsAsync(FindingScope.Server, server, [Found("SQLI-WIN-005", Finding("SQLI-WIN-005", @"disk: D:\", Severity.Critical))], Now.AddMinutes(2));
|
|
Assert.Empty(later.Opened);
|
|
Assert.Equal(@"disk: E:\", Assert.Single(later.Resolved).Subject);
|
|
|
|
var active = await _repository.GetServerFindingsAsync();
|
|
Assert.Equal(2, active.Count);
|
|
Assert.Equal(("AOS01", Severity.Critical), (active.Single(f => f.Scope == FindingScope.Server).ScopeName, active.Single(f => f.Scope == FindingScope.Server).Severity));
|
|
Assert.Equal("AOS port", active.Single(f => f.Scope == FindingScope.Check).ScopeName);
|
|
Assert.True(active[0].Evidence[0].Sensitive);
|
|
var counts = await _repository.GetServerFindingCountsAsync();
|
|
Assert.Equal(1, counts[(FindingScope.Server, server)][Severity.Critical]);
|
|
var resolved = Assert.Single(await _repository.GetServerFindingsAsync(FindingScope.Server, server, FindingStatus.Resolved));
|
|
Assert.Equal(resolved.FindingId, (await _repository.GetServerFindingAsync(resolved.FindingId))!.FindingId);
|
|
|
|
// Back again: a new occurrence, with the old one as history.
|
|
var again = await _repository.SaveScopedRuleResultsAsync(FindingScope.Server, server,
|
|
[Found("SQLI-WIN-005", Finding("SQLI-WIN-005", @"disk: D:\"), Finding("SQLI-WIN-005", @"disk: E:\"))], Now.AddMinutes(3));
|
|
var reopened = Assert.Single(again.Opened);
|
|
Assert.Single(await _repository.GetServerFindingHistoryAsync((await _repository.GetServerFindingAsync(reopened.FindingId))!));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Check_results_count_failures_in_a_row_and_retention_bounds_everything()
|
|
{
|
|
var server = await _repository.AddWindowsServerAsync(new NewWindowsServer("AOS01", "aos01", ServerRoles.Application, [], []), null, Now);
|
|
var check = await _repository.AddSyntheticCheckAsync(Check("AOS port", server), null, Now);
|
|
await _repository.RecordSyntheticCheckResultAsync(check, new SyntheticCheckResult { At = Now.AddMinutes(-10), Success = true, DurationMs = 5, Message = "ok" });
|
|
await _repository.RecordSyntheticCheckResultAsync(check, new SyntheticCheckResult
|
|
{
|
|
At = Now.AddMinutes(-5), Success = false, DurationMs = 15000, Message = "Timeout", ServiceState = "Running",
|
|
Details = [new SyntheticCheckDetail("Fejl", new string('x', 3000))],
|
|
});
|
|
await _repository.RecordSyntheticCheckResultAsync(check, new SyntheticCheckResult { At = Now, Success = false, DurationMs = 15000, Message = "Timeout", Partial = true });
|
|
|
|
var row = (await _repository.GetSyntheticCheckAsync(check))!;
|
|
Assert.Equal((false, 2, "Timeout"), (row.LastSuccess, row.ConsecutiveFailures, row.LastMessage));
|
|
var results = await _repository.GetSyntheticCheckResultsAsync(check);
|
|
Assert.Equal([false, false, true], results.Select(r => r.Success).ToList());
|
|
Assert.Equal("Running", results[1].ServiceState);
|
|
Assert.Equal(400, results[1].Details.Single().Value.Length);
|
|
Assert.True(results[0].ToResult().Partial);
|
|
|
|
// Old data goes; removed and paused servers' findings are resolved.
|
|
await _repository.SaveServerPerformanceAsync(server, new ServerPerformanceSample(Now.AddDays(-20), 1, 1, 1, 1, 1));
|
|
await _repository.SaveServerDisksAsync(server, Now.AddDays(-40), [new ServerDisk(@"C:\", null, null, 1, 1)]);
|
|
await _repository.RecordSyntheticCheckResultAsync(check, new SyntheticCheckResult { At = Now.AddDays(-20), Success = true, Message = "gammel" });
|
|
await _repository.SaveScopedRuleResultsAsync(FindingScope.Server, server, [Found("SQLI-WIN-001", Finding("SQLI-WIN-001", "cpu"))], Now);
|
|
await _repository.SetWindowsServerPausedAsync(server, true, null, Now);
|
|
|
|
await _repository.RunServerMaintenanceAsync(new ServerRetention(), Now);
|
|
|
|
Assert.Empty(await _repository.GetServerPerformanceAsync(server, Now.AddDays(-30)));
|
|
Assert.Empty(await _repository.GetServerDiskHistoryAsync(server, Now.AddDays(-60)));
|
|
Assert.Equal(3, (await _repository.GetSyntheticCheckResultsAsync(check)).Count);
|
|
Assert.Empty(await _repository.GetServerFindingsAsync(FindingScope.Server, server));
|
|
}
|
|
}
|