sqlinsight-pro/tests/SQLInsight.IntegrationTests/ServerAgentTests.cs

using System.Net;
using System.Net.Sockets;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using SQLInsight.Agent;
using SQLInsight.Agent.Configuration;
using SQLInsight.Agent.Servers;
using SQLInsight.Connectivity;
using SQLInsight.Core.Configuration;
using SQLInsight.Core.Findings;
using SQLInsight.Core.Servers;
using SQLInsight.Core.SyntheticChecks;
using SQLInsight.Notifications;
using SQLInsight.Repository;
using SQLInsight.Rules.Engine;
using SQLInsight.Servers;
using SQLInsight.Servers.Synthetic;

namespace SQLInsight.IntegrationTests;

/// <summary>
/// The agent's side of Windows servers and synthetic checks against a real repository, with a fake Windows server (the
/// CIM implementation itself is tested against this computer in SQLInsight.Servers.Tests).
/// </summary>
[Collection(SqlServerTestGroup.Name)]
[Trait("Category", "Integration")]
public sealed class ServerAgentTests(SqlServerFixture sql) : IAsyncLifetime
{
    private readonly string _database = "SQLInsightSrvAgent_" + Guid.NewGuid().ToString("N")[..8];
    private readonly string _keys = Path.Combine(Path.GetTempPath(), "sqlinsight-keys-" + Guid.NewGuid().ToString("N"));
    private InsightRepository _repository = null!;
    private AgentCredentialKey _key = null!;
    private readonly Recorder _recorder = new();
    private static readonly Uri WebUrl = new("https://sqlinsight.test/");

    public async Task InitializeAsync()
    {
        await sql.ExecuteAsSaAsync($"CREATE DATABASE [{_database}];");
        _repository = new InsightRepository(sql.DatabaseConnectionString(_database));
        await _repository.MigrateAsync();
        _key = AgentCredentialKey.LoadOrCreate(_keys);
        await _repository.PublishAgentKeyAsync(_key.Describe(DateTimeOffset.UtcNow));
    }

    public async Task DisposeAsync()
    {
        _key.Dispose();
        Directory.Delete(_keys, recursive: true);
        await sql.ExecuteAsSaAsync($"ALTER DATABASE [{_database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{_database}];");
    }

    private sealed class Recorder : INotificationChannel
    {
        public List<Notification> Received { get; } = [];
        public string Name => "recorder";
        public Severity MinSeverity => Severity.Info;

        public Task SendAsync(IReadOnlyList<Notification> notifications, CancellationToken cancellationToken)
        {
            Received.AddRange(notifications);
            return Task.CompletedTask;
        }
    }

    private NotificationDispatcher Dispatcher() => new([_recorder], new NotificationOptions { MaxMessagesPerHour = 1000 }, TimeProvider.System, NullLogger<NotificationDispatcher>.Instance);

    /// <summary>A Windows server as the tests want it: what each area returns, or the error it throws.</summary>
    private sealed class FakeServer : IWindowsServerSource, IWindowsServerSession
    {
        public ServerAccessException? ConnectError { get; set; }
        public ServerAccessException? ServicesError { get; set; }
        public ulong Idle { get; set; }
        public ulong Clock { get; set; } = 1_000_000_000;
        public double FreeGb { get; set; } = 3;
        public string AosState { get; set; } = "Running";
        public int Connects { get; private set; }

        public string Protocol => "WS-Man";
        public string? ProtocolNote => null;

        public Task<IWindowsServerSession> ConnectAsync(string host, CancellationToken cancellationToken)
        {
            Connects++;
            return ConnectError is { } error ? throw error : Task.FromResult<IWindowsServerSession>(this);
        }

        public Task<ServerOsInfo> GetOperatingSystemAsync(CancellationToken cancellationToken) => Task.FromResult(new ServerOsInfo
        {
            Caption = "Microsoft Windows Server 2019 Standard", Version = "10.0.17763", TotalMemoryMb = 16384, LogicalProcessors = 4,
            LastBootUpTime = DateTimeOffset.UtcNow.AddDays(-10), ComputerName = "AOS01",
        });

        public Task<ServerPerformanceRaw> GetPerformanceAsync(DateTimeOffset now, CancellationToken cancellationToken)
        {
            // Each call: one minute passed, 95 % of it busy.
            Clock += 600_000_000;
            Idle += 30_000_000;
            return Task.FromResult(new ServerPerformanceRaw
            {
                CapturedAt = now, ProcessorIdle100Ns = Idle, ProcessorTimestamp100Ns = Clock, AvailableMb = 6000, CommittedBytes = 4, CommitLimit = 10,
            });
        }

        /// <summary>The disks never answer: only the cycle's deadline ends the wait.</summary>
        public bool DisksHang { get; set; }

        public async Task<IReadOnlyList<ServerDisk>> GetDisksAsync(CancellationToken cancellationToken)
        {
            if (DisksHang)
            {
                await Task.Delay(Timeout.Infinite, cancellationToken);
            }

            return [new ServerDisk(@"C:\", "System", "NTFS", 100, 40), new ServerDisk(@"D:\", "Data", "NTFS", 100, FreeGb)];
        }

        public Task<IReadOnlyList<ServerService>> GetServicesAsync(CancellationToken cancellationToken) => ServicesError is { } error
            ? throw error
            : Task.FromResult<IReadOnlyList<ServerService>>([new ServerService("AOS60$01", "Microsoft Dynamics AX Object Server 6.0$01", AosState, "Auto")]);

        public Task<ServerService?> GetServiceAsync(string name, CancellationToken cancellationToken) =>
            Task.FromResult<ServerService?>(new ServerService(name, name, AosState, "Auto"));

        public Task<ServerEventRead> GetEventsAsync(IReadOnlyDictionary<string, DateTimeOffset> since, int max, CancellationToken cancellationToken) => Task.FromResult(new ServerEventRead(
            [new ServerEventEntry("System", "disk", 153, 2, DateTimeOffset.UtcNow.AddMinutes(-1), @"The IO operation at logical block address 0x1 for Disk 1 was retried.")],
            "Event Log (RPC)", false));

        public Task<ServerTaskRead> GetTasksAsync(Func<string, bool> watch, int maxWatched, TimeSpan budget, CancellationToken cancellationToken) => Task.FromResult(new ServerTaskRead(
            [new ServerTask { Path = @"\", Name = "AX export", State = "Ready", LastTaskResult = 1, LastRunTime = DateTimeOffset.UtcNow.AddHours(-1) }], 140));

        public void Dispose()
        {
        }
    }

    private ServerMonitor Monitor(WindowsServerRow row, FakeServer server) =>
        new(row, server, _repository, new ServerOptions(), NullLogger.Instance, Dispatcher(), WebUrl);

    private async Task<WindowsServerRow> AddServerAsync(string name = "AOS01", IReadOnlyList<string>? watched = null)
    {
        var id = await _repository.AddWindowsServerAsync(new NewWindowsServer(name, name.ToLowerInvariant() + ".firma.local", ServerRoles.Application, watched ?? [], []), "anna",
            DateTimeOffset.UtcNow);
        return (await _repository.GetWindowsServerAsync(id))!;
    }

    [Fact]
    public async Task A_cycle_reads_every_area_stores_it_and_opens_findings_with_links_to_the_server_pages()
    {
        var row = await AddServerAsync();
        var server = new FakeServer();
        var monitor = Monitor(row, server);
        var now = DateTimeOffset.UtcNow;

        await monitor.RunCycleAsync(now, CancellationToken.None);

        var findings = await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId);
        Assert.Contains(findings, f => f.RuleId == "SQLI-WIN-005" && f.Subject == @"disk: D:\");      // 3 GB free
        Assert.Contains(findings, f => f.RuleId == "SQLI-WIN-009" && f.Subject == "event: System/disk/153");
        Assert.Contains(findings, f => f.RuleId == "SQLI-WIN-012" && f.Subject == @"task: \AX export");
        var stored = (await _repository.GetWindowsServerAsync(row.ServerId))!;
        Assert.Equal(("WS-Man", "Microsoft Windows Server 2019 Standard", 4), (stored.Protocol, stored.OperatingSystem, stored.LogicalProcessors));
        Assert.All(stored.Areas, a => Assert.True(a.Ok, a.Detail));
        Assert.Equal(ServerArea.All.Count, stored.Areas.Count);
        Assert.NotNull(await _repository.GetServerSnapshotAsync<ServerDiskSnapshot>(row.ServerId, ServerArea.Disks));
        Assert.Single(await _repository.GetServerEventsAsync(row.ServerId, now.AddDays(-1), now));
        var opened = _recorder.Received.Where(n => n.Kind == NotificationKind.FindingOpened).ToList();
        Assert.Equal(findings.Count, opened.Count);
        Assert.All(opened, n => Assert.StartsWith("https://sqlinsight.test/servere/fund/", n.Url!.ToString(), StringComparison.Ordinal));
        Assert.All(opened, n => Assert.Equal("AOS01", n.Instance));

        // The CPU rule needs a quarter of an hour of minutes: the samples accumulate.
        for (var minute = 1; minute <= 16; minute++)
        {
            await monitor.RunCycleAsync(now.AddMinutes(minute), CancellationToken.None);
        }

        var cpu = Assert.Single(await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId), f => f.RuleId == "SQLI-WIN-001");
        Assert.Contains("95 %", cpu.Title, StringComparison.Ordinal);
        Assert.Equal(17, (await _repository.GetServerPerformanceAsync(row.ServerId, now.AddHours(-1))).Count);
    }

    [Fact]
    public async Task A_missing_right_is_not_measurable_with_the_reason_and_logged_findings_stay_open()
    {
        var row = await AddServerAsync(watched: ["AOS60$01"]);
        var server = new FakeServer { AosState = "Stopped" };
        var monitor = Monitor(row, server);
        var now = DateTimeOffset.UtcNow;
        await monitor.RunCycleAsync(now, CancellationToken.None);
        Assert.Contains(await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId), f => f.RuleId == "SQLI-WIN-008");

        server.ServicesError = ServerErrors.Describe(ServerErrors.AccessDenied, "AccessDenied", "Access is denied.", ServerArea.Services, "aos01", @"FIRMA\AGENT01$", "WS-Man");
        await monitor.RunCycleAsync(now.AddMinutes(6), CancellationToken.None);

        var states = await _repository.GetServerRuleStatesAsync(FindingScope.Server, row.ServerId);
        var watched = states.Single(s => s.RuleId == "SQLI-WIN-008");
        Assert.Equal(RuleOutcome.NotMeasurable, watched.Outcome);
        Assert.Contains("Service Control Manager", watched.Reason, StringComparison.Ordinal);
        Assert.Contains(await _repository.GetServerFindingsAsync(FindingScope.Server, row.ServerId), f => f.RuleId == "SQLI-WIN-008"); // still open
        var services = (await _repository.GetWindowsServerAsync(row.ServerId))!.Areas.Single(a => a.Area == ServerArea.Services);
        Assert.False(services.Ok);
        Assert.Contains("grant-windows-server.ps1", services.Detail, StringComparison.Ordinal);
    }

    [Fact]
    public async Task An_unreachable_server_is_notified_once_and_again_when_it_answers()
    {
        var row = await AddServerAsync();
        var server = new FakeServer
        {
            ConnectError = new ServerAccessException(ServerAccessProblem.Unreachable, "aos01.firma.local svarer ikke på WinRM (WS-Man, TCP 5985)."),
        };
        var monitor = Monitor(row, server);
        var now = DateTimeOffset.UtcNow;

        await monitor.RunCycleAsync(now, CancellationToken.None);
        await monitor.RunCycleAsync(now.AddMinutes(16), CancellationToken.None);
        server.ConnectError = null;
        await monitor.RunCycleAsync(now.AddMinutes(32), CancellationToken.None);

        Assert.Single(_recorder.Received, n => n.Kind == NotificationKind.InstanceUnreachable);
        Assert.Single(_recorder.Received, n => n.Kind == NotificationKind.InstanceRecovered);
        Assert.Equal(new Uri(WebUrl, $"servere/{row.ServerId}"), _recorder.Received.First(n => n.Kind == NotificationKind.InstanceUnreachable).Url);
        var stored = (await _repository.GetWindowsServerAsync(row.ServerId))!;
        Assert.True(stored.Reachable);
    }

    private SyntheticCheckService Checks(FakeServer server, IBusinessConnectorLocator? locator = null, ISqlConnectionFactory? sql = null) => new(
        Options.Create(new SyntheticCheckOptions()),
        new SyntheticCheckRunner(sql ?? new SqlConnectionFactory(), locator ?? new NoBusinessConnector(), new ProcessRunner(), TimeProvider.System),
        server, _repository, _key, TimeProvider.System, NullLogger<SyntheticCheckService>.Instance, Dispatcher(),
        Options.Create(new NotificationOptions { WebUrl = WebUrl }), Options.Create(new AgentOptions()), new RepositoryReadiness());

    private sealed class NoBusinessConnector : IBusinessConnectorLocator
    {
        public BusinessConnectorInstallation? Find() => null;
    }

    [Fact]
    public async Task A_server_that_hangs_is_cut_off_at_the_cycle_deadline_and_backed_off()
    {
        var row = await AddServerAsync();
        var server = new FakeServer { DisksHang = true };
        var monitor = new ServerMonitor(row, server, _repository, new ServerOptions { CycleTimeout = TimeSpan.FromSeconds(5) }, NullLogger.Instance, Dispatcher(), WebUrl);
        var now = DateTimeOffset.UtcNow;

        var cycle = monitor.RunCycleAsync(now, CancellationToken.None);
        Assert.Same(cycle, await Task.WhenAny(cycle, Task.Delay(TimeSpan.FromMinutes(1))));
        await cycle;

        var areas = (await _repository.GetWindowsServerAsync(row.ServerId))!.Areas;
        Assert.True(areas.Single(a => a.Area == ServerArea.Performance).Ok); // read before the deadline
        Assert.All(areas.Where(a => a.Area != ServerArea.Performance), a => Assert.False(a.Ok));
        Assert.All(areas.Where(a => a.Area != ServerArea.Performance), a => Assert.Contains("svarede ikke inden for", a.Detail, StringComparison.Ordinal));
        Assert.Equal(now + ServerMonitor.Backoff(1), monitor.RetryAt);
        Assert.False(monitor.IsDue(now.AddSeconds(30)));

        // Once it answers again, everything is read and the backoff is gone.
        server.DisksHang = false;
        await monitor.RunCycleAsync(now.AddMinutes(16), CancellationToken.None);
        Assert.All((await _repository.GetWindowsServerAsync(row.ServerId))!.Areas, a => Assert.True(a.Ok, a.Detail));
        Assert.Equal(DateTimeOffset.MinValue, monitor.RetryAt);
    }

    private static int ClosedPort()
    {
        using var probe = new TcpListener(IPAddress.Loopback, 0);
        probe.Start();
        return ((IPEndPoint)probe.LocalEndpoint).Port;
    }

    [Fact]
    public async Task A_check_that_fails_while_the_service_runs_says_so_after_two_failures()
    {
        var row = await AddServerAsync();
        var id = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("AOS01 AX", SyntheticCheckKind.AxLogon, row.ServerId,
            new SyntheticCheckSettings { Host = "127.0.0.1", Port = ClosedPort(), ServiceName = "AOS60$01" }, null, null, 300, 5), "anna", DateTimeOffset.UtcNow);
        var service = Checks(new FakeServer());
        var check = (await _repository.GetSyntheticCheckAsync(id))!;

        var first = await service.RunCheckAsync(check, CancellationToken.None);
        Assert.Empty(await _repository.GetServerFindingsAsync(FindingScope.Check, id));
        await service.RunCheckAsync(check, CancellationToken.None);

        Assert.False(first.Success);
        Assert.Equal("Running", first.ServiceState);
        var finding = Assert.Single(await _repository.GetServerFindingsAsync(FindingScope.Check, id));
        Assert.Equal("AOS01 AX svarer ikke, selv om tjenesten AOS60$01 kører", finding.Title);
        Assert.Equal(2, (await _repository.GetSyntheticCheckAsync(id))!.ConsecutiveFailures);
        Assert.StartsWith("https://sqlinsight.test/servere/fund/", Assert.Single(_recorder.Received).Url!.ToString(), StringComparison.Ordinal);
    }

    [Fact]
    public async Task A_sql_check_uses_the_password_sealed_by_the_web_ui_and_only_the_agent_can_open_it()
    {
        var settings = new SyntheticCheckSettings
        {
            Server = sql.Server, Authentication = "SqlLogin", UserName = SqlServerFixture.AgentLogin, TrustServerCertificate = true, WarnMilliseconds = 60000,
        };
        var key = (await _repository.GetCurrentAgentKeyAsync())!;
        var cipher = CredentialEnvelope.Seal(SqlServerFixture.AgentPassword, key.KeyId, key.PublicKey, settings.SecretContext());
        var id = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("SQL01 login", SyntheticCheckKind.Sql, null, settings, cipher, key.KeyId, 300, 15), "anna",
            DateTimeOffset.UtcNow);

        var result = await Checks(new FakeServer()).RunCheckAsync((await _repository.GetSyntheticCheckAsync(id))!, CancellationToken.None);

        Assert.True(result.Success, result.Message);
        Assert.Equal(RuleOutcome.Passed, (await _repository.GetServerRuleStatesAsync(FindingScope.Check, id)).Single(s => s.RuleId == "SQLI-SYN-001").Outcome);

        // Pointed at another server without typing the password again: it is not sent.
        await _repository.UpdateSyntheticCheckAsync(id, new SyntheticCheckInput("SQL01 login", SyntheticCheckKind.Sql, null, settings with { Server = "evil.example" }, null, null, 300, 15),
            "mallory", DateTimeOffset.UtcNow);
        var refused = await Checks(new FakeServer()).RunCheckAsync((await _repository.GetSyntheticCheckAsync(id))!, CancellationToken.None);
        Assert.False(refused.Success);
        Assert.Contains("kan ikke bruges", refused.Message, StringComparison.Ordinal);
        Assert.NotNull((await _repository.GetSyntheticCheckAsync(id))!.SecretError);
    }

    [Fact]
    public async Task Web_requests_test_a_server_and_run_a_check_as_the_agent()
    {
        var row = await AddServerAsync();
        var server = new FakeServer { ServicesError = new ServerAccessException(ServerAccessProblem.AccessDenied, "Adgang nægtet til tjenester.") };
        var id = await _repository.AddSyntheticCheckAsync(new SyntheticCheckInput("Port", SyntheticCheckKind.Tcp, null,
            new SyntheticCheckSettings { Host = "127.0.0.1", Port = ClosedPort() }, null, null, 300, 5), "anna", DateTimeOffset.UtcNow);
        var handler = new ServerRequestHandler(_repository, server, Checks(server), TimeProvider.System);
        var requests = new AgentRequestService(_repository, new InstanceConfigurationStore(Options.Create(new AgentOptions()), _repository, _key,
            NullLogger<InstanceConfigurationStore>.Instance), _key, new SqlConnectionFactory(), new RepositoryReadiness(), TimeProvider.System,
            NullLogger<AgentRequestService>.Instance, handler);

        var test = await RunAsync(requests, ServerRequestKind.ServerTest, row.ServerId);
        var run = await RunAsync(requests, ServerRequestKind.SyntheticCheckRun, id);
        var missing = await RunAsync(requests, ServerRequestKind.ServerTest, 4711);

        Assert.Equal(AgentRequestStatus.Done, test.Status);
        var result = AgentRequestJson.Deserialize<WindowsServerTestResult>(test.Result)!;
        Assert.True(result.Reachable);
        Assert.Equal("Adgang nægtet til tjenester.", Assert.Single(result.Missing).Detail);
        Assert.Equal(AgentRequestStatus.Done, run.Status);
        Assert.False(AgentRequestJson.Deserialize<SyntheticCheckResult>(run.Result)!.Success);
        Assert.Equal(1, (await _repository.GetSyntheticCheckAsync(id))!.ConsecutiveFailures);
        Assert.Equal(AgentRequestStatus.Failed, missing.Status);
        Assert.Contains("findes ikke", missing.Error, StringComparison.Ordinal);
    }

    private async Task<AgentRequestRow> RunAsync(AgentRequestService service, string kind, int target)
    {
        var id = await _repository.AddServerRequestAsync(kind, target, "anna", DateTimeOffset.UtcNow);
        var claimed = await _repository.ClaimAgentRequestAsync(DateTimeOffset.UtcNow);
        Assert.Equal(id, claimed!.RequestId);
        await service.HandleAsync(claimed, CancellationToken.None);
        return (await _repository.GetAgentRequestAsync(id))!;
    }
}