sqlinsight-pro/src/SQLInsight.Agent/Servers/ServerMonitor.cs

using SQLInsight.Core.Collection;
using SQLInsight.Core.Servers;
using SQLInsight.Notifications;
using SQLInsight.Repository;
using SQLInsight.Rules.Engine;
using SQLInsight.Rules.Servers;
using SQLInsight.Servers;

namespace SQLInsight.Agent.Servers;

/// <summary>
/// Looks at one Windows server: reads the areas that are due over one CIM session, keeps the latest data for the rules,
/// evaluates them and stores findings. Only reads; a missing right makes that area "ikke målbar" with the exact reason
/// (logged once per change, never every round). Not thread-safe; one cycle at a time.
/// </summary>
public sealed partial class ServerMonitor(
    WindowsServerRow server,
    IWindowsServerSource source,
    InsightRepository repository,
    ServerOptions options,
    ILogger logger,
    NotificationDispatcher? notifications = null,
    Uri? webUrl = null)
{
    /// <summary>Rules are evaluated at least this often even when only performance was collected.</summary>
    public static readonly TimeSpan EvaluationInterval = TimeSpan.FromMinutes(5);

    private readonly CollectedData _data = new();
    private readonly Dictionary<string, DateTimeOffset> _nextDue = new(StringComparer.Ordinal);
    private readonly Dictionary<string, ServerAreaStatus> _areas = new(StringComparer.Ordinal);
    private readonly Dictionary<string, string> _reported = new(StringComparer.Ordinal);
    private ServerPerformanceRaw? _previousRaw;
    private IReadOnlyList<ServerPerformanceSample> _samples = [];
    private DateTimeOffset? _lastBoot;
    private DateTimeOffset? _restartSeenAt;
    private ServerEventSnapshot? _events;
    private DateTimeOffset _nextEvaluation = DateTimeOffset.MinValue;
    private bool _seeded;
    private bool? _wasReachable;

    public WindowsServerRow Server { get; private set; } = server;

    public int ServerId => Server.ServerId;

    /// <summary>The configuration changed in SQLInsight Web (role, watch lists): used from the next cycle.</summary>
    public void Update(WindowsServerRow row) => Server = row;

    public bool IsDue(DateTimeOffset now) => ServerArea.All.Any(a => !_nextDue.TryGetValue(a, out var due) || due <= now);

    public async Task RunCycleAsync(DateTimeOffset now, CancellationToken cancellationToken)
    {
        var due = ServerArea.All.Where(a => !_nextDue.TryGetValue(a, out var at) || at <= now).ToList();
        if (due.Count == 0)
        {
            return;
        }

        foreach (var area in due)
        {
            _nextDue[area] = now + options.IntervalOf(area);
        }

        if (!_seeded)
        {
            await SeedAsync(now, cancellationToken).ConfigureAwait(false);
            _seeded = true;
        }

        IWindowsServerSession session;
        try
        {
            session = await source.ConnectAsync(Server.Host, cancellationToken).ConfigureAwait(false);
        }
        catch (ServerAccessException ex)
        {
            await NotReachableAsync(ex, due, now, cancellationToken).ConfigureAwait(false);
            return;
        }

        using (session)
        {
            if (_wasReachable == false)
            {
                await NotifyAsync([NotificationFactory.Recovered(Server.Name, webUrl, now) with { Url = ServerNotifications.ServerPage(webUrl, ServerId) }], cancellationToken)
                    .ConfigureAwait(false);
            }

            _wasReachable = true;
            _reported.Remove("connect");
            foreach (var area in due)
            {
                await CollectAsync(session, area, now, cancellationToken).ConfigureAwait(false);
            }

            if (due.Any(a => a != ServerArea.Performance) || now >= _nextEvaluation)
            {
                _nextEvaluation = now + EvaluationInterval;
                await EvaluateAsync(now, cancellationToken).ConfigureAwait(false);
            }
        }
    }

    /// <summary>Unreachable: recorded and notified once. Refused by rights at the door: every area is "ikke målbar" with the reason.</summary>
    private async Task NotReachableAsync(ServerAccessException ex, IReadOnlyList<string> due, DateTimeOffset now, CancellationToken cancellationToken)
    {
        if (Report("connect", ex.Message))
        {
            LogNotReachable(logger, Server.Name, ex.Message);
        }

        await repository.RecordServerErrorAsync(ServerId, ex.Message, now, cancellationToken).ConfigureAwait(false);
        if (ex.Problem is ServerAccessProblem.AccessDenied or ServerAccessProblem.NotSupported)
        {
            foreach (var area in ServerArea.All)
            {
                Unavailable(area, ex.Message);
            }

            await repository.RecordServerAreasAsync(ServerId, _areas.Values.ToList(), cancellationToken).ConfigureAwait(false);
            if (due.Any(a => a != ServerArea.Performance) || now >= _nextEvaluation)
            {
                _nextEvaluation = now + EvaluationInterval;
                await EvaluateAsync(now, cancellationToken).ConfigureAwait(false);
            }

            return;
        }

        if (_wasReachable != false)
        {
            await NotifyAsync([NotificationFactory.Unreachable(Server.Name, ex.Message, webUrl, now) with { Url = ServerNotifications.ServerPage(webUrl, ServerId) }],
                cancellationToken).ConfigureAwait(false);
        }

        _wasReachable = false;
    }

    private async Task CollectAsync(IWindowsServerSession session, string area, DateTimeOffset now, CancellationToken cancellationToken)
    {
        try
        {
            var detail = area switch
            {
                ServerArea.Performance => await CollectPerformanceAsync(session, now, cancellationToken).ConfigureAwait(false),
                ServerArea.Disks => await CollectDisksAsync(session, now, cancellationToken).ConfigureAwait(false),
                ServerArea.Services => await CollectServicesAsync(session, now, cancellationToken).ConfigureAwait(false),
                ServerArea.Events => await CollectEventsAsync(session, now, cancellationToken).ConfigureAwait(false),
                ServerArea.Tasks => await CollectTasksAsync(session, now, cancellationToken).ConfigureAwait(false),
                _ => null,
            };
            _areas[area] = new ServerAreaStatus(area, true, detail, ServerArea.Right(area));
            if (_reported.Remove(area))
            {
                LogAreaRecovered(logger, Server.Name, area);
            }
        }
        catch (ServerAccessException ex)
        {
            Unavailable(area, ex.Message);
            if (Report(area, ex.Message))
            {
                LogAreaUnavailable(logger, Server.Name, area, ex.Message);
            }
        }
    }

    private void Unavailable(string area, string reason)
    {
        _areas[area] = new ServerAreaStatus(area, false, reason, ServerArea.Right(area));
        switch (area)
        {
            case ServerArea.Performance:
                _data.MarkUnavailable<ServerPerformanceSnapshot>(reason);
                break;
            case ServerArea.Disks:
                _data.MarkUnavailable<ServerDiskSnapshot>(reason);
                break;
            case ServerArea.Services:
                _data.MarkUnavailable<ServerServiceSnapshot>(reason);
                break;
            case ServerArea.Events:
                _data.MarkUnavailable<ServerEventSnapshot>(reason);
                break;
            case ServerArea.Tasks:
                _data.MarkUnavailable<ServerTaskSnapshot>(reason);
                break;
        }
    }

    /// <summary>Whether a problem is new (or changed) and should be logged: the same problem every minute is not news.</summary>
    private bool Report(string key, string message)
    {
        if (_reported.TryGetValue(key, out var previous) && previous == message)
        {
            return false;
        }

        _reported[key] = message;
        return true;
    }

    public async Task<string?> CollectPerformanceAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(session);
        var os = await session.GetOperatingSystemAsync(cancellationToken).ConfigureAwait(false);
        var raw = await session.GetPerformanceAsync(now, cancellationToken).ConfigureAwait(false);
        DateTimeOffset? previousBoot = null;
        if (os.LastBootUpTime is { } boot && _lastBoot is { } before && Math.Abs((boot - before).TotalMinutes) > 2)
        {
            // Restarted since the last look: the raw counters started over too.
            previousBoot = before;
            _restartSeenAt = now;
            _previousRaw = null;
        }

        _lastBoot = os.LastBootUpTime ?? _lastBoot;
        var sample = ServerPerformanceMath.Sample(_previousRaw, raw, os, options.IntervalOf(ServerArea.Performance));
        _previousRaw = raw;
        _samples = ServerPerformanceMath.Append(_samples, sample);
        var snapshot = new ServerPerformanceSnapshot
        {
            CapturedAt = now,
            Os = os,
            Raw = raw,
            Samples = _samples,
            PreviousBootUpTime = previousBoot,
            RestartSeenAt = _restartSeenAt,
            Protocol = session.Protocol,
        };
        _data.Set(snapshot);
        await repository.SaveServerPerformanceAsync(ServerId, sample, cancellationToken).ConfigureAwait(false);
        await repository.RecordServerContactAsync(ServerId, os, session.Protocol, null, now, cancellationToken).ConfigureAwait(false);
        await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Performance, snapshot with { Samples = [] }, now, cancellationToken).ConfigureAwait(false);
        return sample.CpuPercent is { } cpu ? $"CPU {cpu:0} %" : null;
    }

    public async Task<string?> CollectDisksAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(session);
        var disks = await session.GetDisksAsync(cancellationToken).ConfigureAwait(false);
        await repository.SaveServerDisksAsync(ServerId, now, disks, cancellationToken).ConfigureAwait(false);
        var historyDays = Math.Max(1, options.Disks.ForecastHistoryDays);
        var history = await repository.GetServerDiskHistoryAsync(ServerId, now.AddDays(-historyDays), cancellationToken).ConfigureAwait(false);
        var settings = new DiskForecastSettings { HistoryDays = historyDays };
        var forecasts = disks.Select(d => DiskForecaster.Forecast(d.Volume, history.TryGetValue(d.Volume, out var points) ? points : [], now, settings)).ToList();
        var snapshot = new ServerDiskSnapshot { CapturedAt = now, Disks = disks, Forecasts = forecasts };
        _data.Set(snapshot);
        await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Disks, snapshot, now, cancellationToken).ConfigureAwait(false);
        return $"{disks.Count} drev";
    }

    public async Task<string?> CollectServicesAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(session);
        var services = await session.GetServicesAsync(cancellationToken).ConfigureAwait(false);
        var snapshot = new ServerServiceSnapshot { CapturedAt = now, Services = services };
        _data.Set(snapshot);
        await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Services, snapshot, now, cancellationToken).ConfigureAwait(false);
        return $"{services.Count} tjenester";
    }

    /// <summary>The first time, the last day is read, so recent problems show at once; then from where the last read stopped.</summary>
    public async Task<string?> CollectEventsAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(session);
        var since = _events?.ReadUntil ?? now - ServerEventAnalysis.RecentWindow;
        var read = await session.GetEventsAsync(since, Math.Max(10, options.EventLog.MaxEventsPerCollection), cancellationToken).ConfigureAwait(false);
        var (groups, ignored) = ServerEventAnalysis.Group(read.Entries, ServerEventAnalysis.DefaultIgnore.Concat(options.EventLog.Ignore));

        // Strictly after the newest event read; without events, keep ten minutes' margin for events written late.
        var readUntil = read.Entries.Count > 0 ? read.Entries.Max(e => e.At) : Max(since, now.AddMinutes(-10));
        var snapshot = new ServerEventSnapshot
        {
            CapturedAt = now,
            ReadUntil = readUntil,
            Method = read.Method,
            Groups = groups,
            Recent = ServerEventAnalysis.Recent(_events?.Recent ?? [], groups, now),
            Ignored = ignored,
            Coverage = read.Truncated
                ? $"Der var flere end {options.EventLog.MaxEventsPerCollection} hændelser i en log siden sidst; resten læses ved næste indsamling."
                : read.Note,
        };
        _events = snapshot;
        _data.Set(snapshot);
        await repository.SaveServerEventsAsync(ServerId, snapshot, cancellationToken).ConfigureAwait(false);
        await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Events, snapshot with { Groups = [] }, now, cancellationToken).ConfigureAwait(false);
        return $"{read.Entries.Count} hændelser ({read.Method})";

        static DateTimeOffset Max(DateTimeOffset a, DateTimeOffset b) => a > b ? a : b;
    }

    public async Task<string?> CollectTasksAsync(IWindowsServerSession session, DateTimeOffset now, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(session);
        var tasks = options.Tasks;
        var read = await session.GetTasksAsync(
            tasks.Watches,
            Math.Max(1, tasks.MaxWatched), cancellationToken).ConfigureAwait(false);
        var snapshot = new ServerTaskSnapshot { CapturedAt = now, Tasks = read.Watched, Visible = read.Visible, Coverage = read.Note };
        _data.Set(snapshot);
        await repository.SaveServerSnapshotAsync(ServerId, ServerArea.Tasks, snapshot, now, cancellationToken).ConfigureAwait(false);
        return $"{read.Watched.Count} af {read.Visible} opgaver overvåges";
    }

    private async Task EvaluateAsync(DateTimeOffset now, CancellationToken cancellationToken)
    {
        await repository.RecordServerAreasAsync(ServerId, ServerArea.All.Where(_areas.ContainsKey).Select(a => _areas[a]).ToList(), cancellationToken).ConfigureAwait(false);
        var context = new ServerRuleContext(Server.ToIdentity(), now, _data, options) { PerformanceInterval = options.IntervalOf(ServerArea.Performance) };
        var results = ScopedRuleEngine.Evaluate(WindowsServerRules.Create(), context);
        var changes = await repository.SaveScopedRuleResultsAsync(FindingScope.Server, ServerId, results, now, cancellationToken).ConfigureAwait(false);
        await NotifyAsync(ServerNotifications.For(Server.Name, changes, webUrl, now).ToList(), cancellationToken).ConfigureAwait(false);
        if (logger.IsEnabled(LogLevel.Debug))
        {
            var findings = results.Sum(r => r.Findings.Count);
            var notMeasurable = results.Count(r => r.Outcome is RuleOutcome.NotMeasurable or RuleOutcome.Error);
            LogCycle(logger, Server.Name, findings, notMeasurable);
        }
    }

    /// <summary>What a restart of the agent would lose: the last hour's samples, the boot time and where the event log was read to.</summary>
    private async Task SeedAsync(DateTimeOffset now, CancellationToken cancellationToken)
    {
        var stored = await repository.GetServerPerformanceAsync(ServerId, now - ServerPerformanceMath.Window, cancellationToken).ConfigureAwait(false);
        _samples = stored.Select(s => new ServerPerformanceSample(new DateTimeOffset(DateTime.SpecifyKind(s.SampledAt, DateTimeKind.Utc)), s.CpuPercent, s.AvailableMb,
            s.AvailableMb is { } mb && s.AvailablePct is > 0 ? mb * 100 / s.AvailablePct.Value : null, s.PagesPerSec, s.CommitPct)).ToList();
        var performance = await repository.GetServerSnapshotAsync<ServerPerformanceSnapshot>(ServerId, ServerArea.Performance, cancellationToken).ConfigureAwait(false);
        _lastBoot = performance?.Os.LastBootUpTime;
        _restartSeenAt = performance?.RestartSeenAt;
        _events = await repository.GetServerSnapshotAsync<ServerEventSnapshot>(ServerId, ServerArea.Events, cancellationToken).ConfigureAwait(false);
    }

    private async Task NotifyAsync(List<Notification> events, CancellationToken cancellationToken)
    {
        if (notifications is not null && events.Count > 0)
        {
            await notifications.DispatchAsync(events, cancellationToken).ConfigureAwait(false);
        }
    }

    [LoggerMessage(Level = LogLevel.Warning, Message = "Windows server {Server} cannot be read: {Reason}")]
    private static partial void LogNotReachable(ILogger logger, string server, string reason);

    [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: {Area} is not measurable: {Reason}")]
    private static partial void LogAreaUnavailable(ILogger logger, string server, string area, string reason);

    [LoggerMessage(Level = LogLevel.Information, Message = "Windows server {Server}: {Area} can be read again")]
    private static partial void LogAreaRecovered(ILogger logger, string server, string area);

    [LoggerMessage(Level = LogLevel.Debug, Message = "Windows server {Server}: {Findings} findings, {NotMeasurable} rules not measurable")]
    private static partial void LogCycle(ILogger logger, string server, int findings, int notMeasurable);
}