using System.Net; using System.Net.Sockets; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using SQLInsight.Agent; using SQLInsight.Agent.Configuration; using SQLInsight.Agent.Servers; using SQLInsight.Connectivity; using SQLInsight.Core.Configuration; using SQLInsight.Core.Findings; using SQLInsight.Core.Servers; using SQLInsight.Core.SyntheticChecks; using SQLInsight.Notifications; using SQLInsight.Repository; using SQLInsight.Rules.Engine; using SQLInsight.Servers; using SQLInsight.Servers.Synthetic; namespace SQLInsight.IntegrationTests; /// /// The agent's side of Windows servers and synthetic checks against a real repository, with a fake Windows server (the /// CIM implementation itself is tested against this computer in SQLInsight.Servers.Tests). /// [Collection(SqlServerTestGroup.Name)] [Trait("Category", "Integration")] public sealed class ServerAgentTests(SqlServerFixture sql) : IAsyncLifetime { private readonly string _database = "SQLInsightSrvAgent_" + Guid.NewGuid().ToString("N")[..8]; private readonly string _keys = Path.Combine(Path.GetTempPath(), "sqlinsight-keys-" + Guid.NewGuid().ToString("N")); private InsightRepository _repository = null!; private AgentCredentialKey _key = null!; private readonly Recorder _recorder = new(); private static readonly Uri WebUrl = new("https://sqlinsight.test/"); public async Task InitializeAsync() { await sql.ExecuteAsSaAsync($"CREATE DATABASE [{_database}];"); _repository = new InsightRepository(sql.DatabaseConnectionString(_database)); await _repository.MigrateAsync(); _key = AgentCredentialKey.LoadOrCreate(_keys); await _repository.PublishAgentKeyAsync(_key.Describe(DateTimeOffset.UtcNow)); } public async Task DisposeAsync() { _key.Dispose(); Directory.Delete(_keys, recursive: true); await sql.ExecuteAsSaAsync($"ALTER DATABASE [{_database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{_database}];"); } private sealed class Recorder : INotificationChannel { public List Received { get; } = []; public string Name => "recorder"; public Severity MinSeverity => Severity.Info; public Task SendAsync(IReadOnlyList notifications, CancellationToken cancellationToken) { Received.AddRange(notifications); return Task.CompletedTask; } } private NotificationDispatcher Dispatcher() => new([_recorder], new NotificationOptions { MaxMessagesPerHour = 1000 }, TimeProvider.System, NullLogger.Instance); /// A Windows server as the tests want it: what each area returns, or the error it throws. private sealed class FakeServer : IWindowsServerSource, IWindowsServerSession { public ServerAccessException? ConnectError { get; set; } public ServerAccessException? ServicesError { get; set; } public ulong Idle { get; set; } public ulong Clock { get; set; } = 1_000_000_000; public double FreeGb { get; set; } = 3; public string AosState { get; set; } = "Running"; public int Connects { get; private set; } public string Protocol => "WS-Man"; public string? ProtocolNote => null; public Task ConnectAsync(string host, CancellationToken cancellationToken) { Connects++; return ConnectError is { } error ? throw error : Task.FromResult(this); } public Task GetOperatingSystemAsync(CancellationToken cancellationToken) => Task.FromResult(new ServerOsInfo { Caption = "Microsoft Windows Server 2019 Standard", Version = "10.0.17763", TotalMemoryMb = 16384, LogicalProcessors = 4, LastBootUpTime = DateTimeOffset.UtcNow.AddDays(-10), ComputerName = "AOS01", }); public Task GetPerformanceAsync(DateTimeOffset now, CancellationToken cancellationToken) { // Each call: one minute passed, 95 % of it busy. Clock += 600_000_000; Idle += 30_000_000; return Task.FromResult(new ServerPerformanceRaw { CapturedAt = now, ProcessorIdle100Ns = Idle, ProcessorTimestamp100Ns = Clock, AvailableMb = 6000, CommittedBytes = 4, CommitLimit = 10, }); } public Task> GetDisksAsync(CancellationToken cancellationToken) => Task.FromResult>([new ServerDisk(@"C:\", "System", "NTFS", 100, 40), new ServerDisk(@"D:\", "Data", "NTFS", 100, FreeGb)]); public Task> GetServicesAsync(CancellationToken cancellationToken) => ServicesError is { } error ? throw error : Task.FromResult>([new ServerService("AOS60$01", "Microsoft Dynamics AX Object Server 6.0$01", AosState, "Auto")]); public Task GetServiceAsync(string name, CancellationToken cancellationToken) => Task.FromResult(new ServerService(name, name, AosState, "Auto")); public Task GetEventsAsync(DateTimeOffset since, int max, CancellationToken cancellationToken) => Task.FromResult(new ServerEventRead( [new ServerEventEntry("System", "disk", 153, 2, DateTimeOffset.UtcNow.AddMinutes(-1), @"The IO operation at logical block address 0x1 for Disk 1 was retried.")], "Event Log (RPC)", false)); public Task GetTasksAsync(Func watch, int maxWatched, CancellationToken cancellationToken) => Task.FromResult(new ServerTaskRead( [new ServerTask { Path = @"\", Name = "AX export", State = "Ready", LastTaskResult = 1, LastRunTime = DateTimeOffset.UtcNow.AddHours(-1) }], 140)); public void Dispose() { } } private ServerMonitor Monitor(WindowsServerRow row, FakeServer server) => new(row, server, _repository, new ServerOptions(), NullLogger.Instance, Dispatcher(), WebUrl); private async Task AddServerAsync(string name = "AOS01", IReadOnlyList? watched = null) { var id = await _repository.AddWindowsServerAsync(new NewWindowsServer(name, name.ToLowerInvariant() + ".firma.local", ServerRoles.Application, watched ?? [], []), "anna", DateTimeOffset.UtcNow); return (await _repository.GetWindowsServerAsync(id))!; } [Fact] public async Task A_cycle_reads_every_area_stores_it_and_opens_findings_with_links_to_the_server_pages() { var row = await AddServerAsync(); var server = new FakeServer(); var monitor = Monitor(row, server); var now = DateTimeOffset.UtcNow; await monitor.RunCycleAsync(now, CancellationToken.None); var findings = await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId); Assert.Contains(findings, f => f.RuleId == "SQLI-WIN-005" && f.Subject == @"disk: D:\"); // 3 GB free Assert.Contains(findings, f => f.RuleId == "SQLI-WIN-009" && f.Subject == "event: System/disk/153"); Assert.Contains(findings, f => f.RuleId == "SQLI-WIN-012" && f.Subject == @"task: \AX export"); var stored = (await _repository.GetWindowsServerAsync(row.ServerId))!; Assert.Equal(("WS-Man", "Microsoft Windows Server 2019 Standard", 4), (stored.Protocol, stored.OperatingSystem, stored.LogicalProcessors)); Assert.All(stored.Areas, a => Assert.True(a.Ok, a.Detail)); Assert.Equal(ServerArea.All.Count, stored.Areas.Count); Assert.NotNull(await _repository.GetServerSnapshotAsync(row.ServerId, ServerArea.Disks)); Assert.Single(await _repository.GetServerEventsAsync(row.ServerId, now.AddDays(-1), now)); var opened = _recorder.Received.Where(n => n.Kind == NotificationKind.FindingOpened).ToList(); Assert.Equal(findings.Count, opened.Count); Assert.All(opened, n => Assert.StartsWith("https://sqlinsight.test/servere/fund/", n.Url!.ToString(), StringComparison.Ordinal)); Assert.All(opened, n => Assert.Equal("AOS01", n.Instance)); // The CPU rule needs a quarter of an hour of minutes: the samples accumulate. for (var minute = 1; minute <= 16; minute++) { await monitor.RunCycleAsync(now.AddMinutes(minute), CancellationToken.None); } var cpu = Assert.Single(await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId), f => f.RuleId == "SQLI-WIN-001"); Assert.Contains("95 %", cpu.Title, StringComparison.Ordinal); Assert.Equal(17, (await _repository.GetServerPerformanceAsync(row.ServerId, now.AddHours(-1))).Count); } [Fact] public async Task A_missing_right_is_not_measurable_with_the_reason_and_logged_findings_stay_open() { var row = await AddServerAsync(watched: ["AOS60$01"]); var server = new FakeServer { AosState = "Stopped" }; var monitor = Monitor(row, server); var now = DateTimeOffset.UtcNow; await monitor.RunCycleAsync(now, CancellationToken.None); Assert.Contains(await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId), f => f.RuleId == "SQLI-WIN-008"); server.ServicesError = ServerErrors.Describe(ServerErrors.AccessDenied, "AccessDenied", "Access is denied.", ServerArea.Services, "aos01", @"FIRMA\AGENT01$", "WS-Man"); await monitor.RunCycleAsync(now.AddMinutes(6), CancellationToken.None); var states = await _repository.GetServerRuleStatesAsync(FindingScope.Server, row.ServerId); var watched = states.Single(s => s.RuleId == "SQLI-WIN-008"); Assert.Equal(RuleOutcome.NotMeasurable, watched.Outcome); Assert.Contains("Service Control Manager", watched.Reason, StringComparison.Ordinal); Assert.Contains(await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId), f => f.RuleId == "SQLI-WIN-008"); // still open var services = (await _repository.GetWindowsServerAsync(row.ServerId))!.Areas.Single(a => a.Area == ServerArea.Services); Assert.False(services.Ok); Assert.Contains("grant-windows-server.ps1", services.Detail, StringComparison.Ordinal); } [Fact] public async Task An_unreachable_server_is_notified_once_and_again_when_it_answers() { var row = await AddServerAsync(); var server = new FakeServer { ConnectError = new ServerAccessException(ServerAccessProblem.Unreachable, "aos01.firma.local svarer ikke på WinRM (WS-Man, TCP 5985)."), }; var monitor = Monitor(row, server); var now = DateTimeOffset.UtcNow; await monitor.RunCycleAsync(now, CancellationToken.None); await monitor.RunCycleAsync(now.AddMinutes(16), CancellationToken.None); server.ConnectError = null; await monitor.RunCycleAsync(now.AddMinutes(32), CancellationToken.None); Assert.Single(_recorder.Received, n => n.Kind == NotificationKind.InstanceUnreachable); Assert.Single(_recorder.Received, n => n.Kind == NotificationKind.InstanceRecovered); Assert.Equal(new Uri(WebUrl, $"servere/{row.ServerId}"), _recorder.Received.First(n => n.Kind == NotificationKind.InstanceUnreachable).Url); var stored = (await _repository.GetWindowsServerAsync(row.ServerId))!; Assert.True(stored.Reachable); } private SyntheticCheckService Checks(FakeServer server, IBusinessConnectorLocator? locator = null) => new( Options.Create(new SyntheticCheckOptions()), new SyntheticCheckRunner(new SqlConnectionFactory(), locator ?? new NoBusinessConnector(), new ProcessRunner(), TimeProvider.System), server, _repository, _key, TimeProvider.System, NullLogger.Instance, Dispatcher(), Options.Create(new NotificationOptions { WebUrl = WebUrl }), Options.Create(new AgentOptions()), new RepositoryReadiness()); private sealed class NoBusinessConnector : IBusinessConnectorLocator { public BusinessConnectorInstallation? Find() => null; } private static int ClosedPort() { using var probe = new TcpListener(IPAddress.Loopback, 0); probe.Start(); return ((IPEndPoint)probe.LocalEndpoint).Port; } [Fact] public async Task A_check_that_fails_while_the_service_runs_says_so_after_two_failures() { var row = await AddServerAsync(); var id = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("AOS01 AX", SyntheticCheckKind.AxLogon, row.ServerId, new SyntheticCheckSettings { Host = "127.0.0.1", Port = ClosedPort(), ServiceName = "AOS60$01" }, null, null, 300, 5), "anna", DateTimeOffset.UtcNow); var service = Checks(new FakeServer()); var check = (await _repository.GetSyntheticCheckAsync(id))!; var first = await service.RunCheckAsync(check, CancellationToken.None); Assert.Empty(await _repository.GetServerFindingsAsync(FindingScope.Check, id)); await service.RunCheckAsync(check, CancellationToken.None); Assert.False(first.Success); Assert.Equal("Running", first.ServiceState); var finding = Assert.Single(await _repository.GetServerFindingsAsync(FindingScope.Check, id)); Assert.Equal("AOS01 AX svarer ikke, selv om tjenesten AOS60$01 kører", finding.Title); Assert.Equal(2, (await _repository.GetSyntheticCheckAsync(id))!.ConsecutiveFailures); Assert.StartsWith("https://sqlinsight.test/servere/fund/", Assert.Single(_recorder.Received).Url!.ToString(), StringComparison.Ordinal); } [Fact] public async Task A_sql_check_uses_the_password_sealed_by_the_web_ui_and_only_the_agent_can_open_it() { var settings = new SyntheticCheckSettings { Server = sql.Server, Authentication = "SqlLogin", UserName = SqlServerFixture.AgentLogin, TrustServerCertificate = true, WarnMilliseconds = 60000, }; var key = (await _repository.GetCurrentAgentKeyAsync())!; var cipher = CredentialEnvelope.Seal(SqlServerFixture.AgentPassword, key.KeyId, key.PublicKey, settings.SecretContext()); var id = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("SQL01 login", SyntheticCheckKind.Sql, null, settings, cipher, key.KeyId, 300, 15), "anna", DateTimeOffset.UtcNow); var result = await Checks(new FakeServer()).RunCheckAsync((await _repository.GetSyntheticCheckAsync(id))!, CancellationToken.None); Assert.True(result.Success, result.Message); Assert.Equal(RuleOutcome.Passed, (await _repository.GetServerRuleStatesAsync(FindingScope.Check, id)).Single(s => s.RuleId == "SQLI-SYN-001").Outcome); // Pointed at another server without typing the password again: it is not sent. await _repository.UpdateSyntheticCheckAsync(id, new SyntheticCheckInput("SQL01 login", SyntheticCheckKind.Sql, null, settings with { Server = "evil.example" }, null, null, 300, 15), "mallory", DateTimeOffset.UtcNow); var refused = await Checks(new FakeServer()).RunCheckAsync((await _repository.GetSyntheticCheckAsync(id))!, CancellationToken.None); Assert.False(refused.Success); Assert.Contains("kan ikke bruges", refused.Message, StringComparison.Ordinal); Assert.NotNull((await _repository.GetSyntheticCheckAsync(id))!.SecretError); } [Fact] public async Task Web_requests_test_a_server_and_run_a_check_as_the_agent() { var row = await AddServerAsync(); var server = new FakeServer { ServicesError = new ServerAccessException(ServerAccessProblem.AccessDenied, "Adgang nægtet til tjenester.") }; var id = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("Port", SyntheticCheckKind.Tcp, null, new SyntheticCheckSettings { Host = "127.0.0.1", Port = ClosedPort() }, null, null, 300, 5), "anna", DateTimeOffset.UtcNow); var handler = new ServerRequestHandler(_repository, server, Checks(server), TimeProvider.System); var requests = new AgentRequestService(_repository, new InstanceConfigurationStore(Options.Create(new AgentOptions()), _repository, _key, NullLogger.Instance), _key, new SqlConnectionFactory(), new RepositoryReadiness(), TimeProvider.System, NullLogger.Instance, handler); var test = await RunAsync(requests, ServerRequestKind.ServerTest, row.ServerId); var run = await RunAsync(requests, ServerRequestKind.SyntheticCheckRun, id); var missing = await RunAsync(requests, ServerRequestKind.ServerTest, 4711); Assert.Equal(AgentRequestStatus.Done, test.Status); var result = AgentRequestJson.Deserialize(test.Result)!; Assert.True(result.Reachable); Assert.Equal("Adgang nægtet til tjenester.", Assert.Single(result.Missing).Detail); Assert.Equal(AgentRequestStatus.Done, run.Status); Assert.False(AgentRequestJson.Deserialize(run.Result)!.Success); Assert.Equal(1, (await _repository.GetSyntheticCheckAsync(id))!.ConsecutiveFailures); Assert.Equal(AgentRequestStatus.Failed, missing.Status); Assert.Contains("findes ikke", missing.Error, StringComparison.Ordinal); } private async Task RunAsync(AgentRequestService service, string kind, int target) { var id = await _repository.AddServerRequestAsync(kind, target, "anna", DateTimeOffset.UtcNow); var claimed = await _repository.ClaimAgentRequestAsync(DateTimeOffset.UtcNow); Assert.Equal(id, claimed!.RequestId); await service.HandleAsync(claimed, CancellationToken.None); return (await _repository.GetAgentRequestAsync(id))!; } }