diff --git a/Directory.Packages.props b/Directory.Packages.props index 30d90f3..827ed48 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -7,6 +7,9 @@ + + + diff --git a/SQLInsight.slnx b/SQLInsight.slnx index 7af94bf..0ff6098 100644 --- a/SQLInsight.slnx +++ b/SQLInsight.slnx @@ -6,6 +6,7 @@ + @@ -15,6 +16,7 @@ + diff --git a/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs b/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs index 2742f43..c6e1a13 100644 --- a/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs +++ b/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs @@ -2,6 +2,7 @@ using SQLInsight.Collectors; using SQLInsight.Connectivity; using SQLInsight.Core.Collection; using SQLInsight.Core.Instances; +using SQLInsight.Notifications; using SQLInsight.Repository; using SQLInsight.Rules.Engine; @@ -18,8 +19,12 @@ public sealed partial class InstanceMonitor( ISqlConnectionFactory connectionFactory, RuleEngine ruleEngine, InsightRepository repository, - ILogger logger) + ILogger logger, + NotificationDispatcher? notifications = null, + Uri? webUrl = null) { + private bool? _wasReachable; + private readonly CollectedData _data = new(); private readonly Dictionary _previous = []; private readonly Dictionary _nextDue = []; @@ -63,6 +68,12 @@ public sealed partial class InstanceMonitor( { LogConnectFailed(logger, ex, instance.Name); await repository.RecordInstanceErrorAsync(instance.Name, ex.Message, now, cancellationToken).ConfigureAwait(false); + if (_wasReachable != false) + { + await NotifyAsync([NotificationFactory.Unreachable(instance.Name, ex.Message, webUrl, now)], cancellationToken).ConfigureAwait(false); + } + + _wasReachable = false; return null; } @@ -81,7 +92,17 @@ public sealed partial class InstanceMonitor( await RefreshBaselineAsync(instanceId, now, cancellationToken).ConfigureAwait(false); var results = ruleEngine.Evaluate(new RuleContext(info, capabilities, now, _data)); - await repository.SaveRuleResultsAsync(instanceId, results, now, cancellationToken).ConfigureAwait(false); + var changes = await repository.SaveRuleResultsAsync(instanceId, results, now, cancellationToken).ConfigureAwait(false); + var events = new List(); + if (_wasReachable == false) + { + events.Add(NotificationFactory.Recovered(instance.Name, webUrl, now)); + } + + _wasReachable = true; + events.AddRange(changes.Opened.Select(o => NotificationFactory.Opened(instance.Name, o.FindingId, o.Finding, webUrl, now))); + events.AddRange(changes.Resolved.Select(r => NotificationFactory.Resolved(instance.Name, r.FindingId, r.RuleId, r.Title, (Core.Findings.Severity)r.Severity, webUrl, now))); + await NotifyAsync(events, cancellationToken).ConfigureAwait(false); var findings = 0; var notMeasurable = 0; foreach (var result in results) @@ -125,6 +146,14 @@ public sealed partial class InstanceMonitor( } } + private async Task NotifyAsync(List events, CancellationToken cancellationToken) + { + if (notifications is not null && events.Count > 0) + { + await notifications.DispatchAsync(events, cancellationToken).ConfigureAwait(false); + } + } + /// /// The baseline for the current hour of the week, from the repository's hourly history. Refreshed at each new hour. /// diff --git a/src/SQLInsight.Agent/Monitoring/MonitoringService.cs b/src/SQLInsight.Agent/Monitoring/MonitoringService.cs index 83b2ce3..75bfb0b 100644 --- a/src/SQLInsight.Agent/Monitoring/MonitoringService.cs +++ b/src/SQLInsight.Agent/Monitoring/MonitoringService.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.Options; using SQLInsight.Collectors; using SQLInsight.Connectivity; +using SQLInsight.Notifications; using SQLInsight.Repository; using SQLInsight.Rules.Engine; @@ -15,7 +16,9 @@ public sealed partial class MonitoringService( RuleEngine ruleEngine, InsightRepository repository, TimeProvider timeProvider, - ILoggerFactory loggerFactory) : BackgroundService + ILoggerFactory loggerFactory, + NotificationDispatcher notifications, + IOptions notificationOptions) : BackgroundService { private readonly ILogger _logger = loggerFactory.CreateLogger(); @@ -26,7 +29,8 @@ public sealed partial class MonitoringService( var intervals = new Dictionary(settings.CollectorIntervals, StringComparer.OrdinalIgnoreCase); var monitors = settings.Instances - .Select(i => new InstanceMonitor(i, BuiltInCollectors.Create(), intervals, connectionFactory, ruleEngine, repository, loggerFactory.CreateLogger($"SQLInsight.Instance.{i.Name}"))) + .Select(i => new InstanceMonitor(i, BuiltInCollectors.Create(), intervals, connectionFactory, ruleEngine, repository, + loggerFactory.CreateLogger($"SQLInsight.Instance.{i.Name}"), notifications, notificationOptions.Value.WebUrl)) .ToList(); if (monitors.Count == 0) { diff --git a/src/SQLInsight.Agent/Program.cs b/src/SQLInsight.Agent/Program.cs index 55b754c..255cbda 100644 --- a/src/SQLInsight.Agent/Program.cs +++ b/src/SQLInsight.Agent/Program.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Options; using SQLInsight.Agent; using SQLInsight.Agent.Monitoring; using SQLInsight.Connectivity; +using SQLInsight.Notifications; using SQLInsight.Repository; using SQLInsight.Rules; using SQLInsight.Rules.Engine; @@ -26,6 +27,30 @@ builder.Services.AddSingleton(sp => return new InsightRepository(connectionString); }); +builder.Services.Configure(builder.Configuration.GetSection(NotificationOptions.SectionName)); +builder.Services.AddHttpClient("notifications", c => c.Timeout = TimeSpan.FromSeconds(15)); +builder.Services.AddSingleton(sp => +{ + var options = sp.GetRequiredService>().Value; + var http = sp.GetRequiredService(); + var channels = new List(); + if (options.Email.Enabled) + { + channels.Add(new EmailChannel(options.Email)); + } + + if (options.Teams.Enabled) + { + channels.Add(new TeamsChannel(options.Teams, http.CreateClient("notifications"))); + } + + if (options.Webhook.Enabled) + { + channels.Add(new WebhookChannel(options.Webhook, http.CreateClient("notifications"))); + } + + return new NotificationDispatcher(channels, options, sp.GetRequiredService(), sp.GetRequiredService>()); +}); builder.Services.AddHostedService(); var host = builder.Build(); diff --git a/src/SQLInsight.Agent/SQLInsight.Agent.csproj b/src/SQLInsight.Agent/SQLInsight.Agent.csproj index 2fc5325..fbe7931 100644 --- a/src/SQLInsight.Agent/SQLInsight.Agent.csproj +++ b/src/SQLInsight.Agent/SQLInsight.Agent.csproj @@ -9,9 +9,11 @@ + + diff --git a/src/SQLInsight.Agent/appsettings.json b/src/SQLInsight.Agent/appsettings.json index 30ff676..037953b 100644 --- a/src/SQLInsight.Agent/appsettings.json +++ b/src/SQLInsight.Agent/appsettings.json @@ -13,6 +13,31 @@ "MaxParallelInstances": 4, "CollectorIntervals": { }, + "Notifications": { + "WebUrl": "https://sqlinsight.eksempel.local/", + "NotifyResolved": true, + "MaxMessagesPerHour": 20, + "Email": { + "Enabled": false, + "Host": "smtp.eksempel.local", + "Port": 587, + "Security": "StartTls", + "From": "sqlinsight@eksempel.local", + "To": [ "dba@eksempel.local" ], + "MinSeverity": "High" + }, + "Teams": { + "Enabled": false, + "WebhookUrl": null, + "MinSeverity": "High" + }, + "Webhook": { + "Enabled": false, + "Url": null, + "Secret": null, + "MinSeverity": "Medium" + } + }, "Retention": { "RawDays": 14, "HourlyDays": 400, diff --git a/src/SQLInsight.Notifications/EmailChannel.cs b/src/SQLInsight.Notifications/EmailChannel.cs new file mode 100644 index 0000000..e1b2855 --- /dev/null +++ b/src/SQLInsight.Notifications/EmailChannel.cs @@ -0,0 +1,77 @@ +using System.Net; +using System.Text; +using MailKit.Net.Smtp; +using MailKit.Security; +using MimeKit; +using SQLInsight.Core.Findings; + +namespace SQLInsight.Notifications; + +/// +/// E-mail via SMTP (MailKit). Plain text and HTML; all values are HTML-encoded. +/// +public sealed class EmailChannel(EmailOptions options) : INotificationChannel +{ + public string Name => "email"; + public Severity MinSeverity => options.MinSeverity; + + public async Task SendAsync(IReadOnlyList notifications, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(notifications); + var message = Build(options, notifications); + + using var client = new SmtpClient(); + var security = options.Security.ToUpperInvariant() switch + { + "NONE" => SecureSocketOptions.None, + "SSLONCONNECT" => SecureSocketOptions.SslOnConnect, + _ => SecureSocketOptions.StartTls, + }; + await client.ConnectAsync(options.Host, options.Port, security, cancellationToken).ConfigureAwait(false); + if (!string.IsNullOrEmpty(options.UserName)) + { + await client.AuthenticateAsync(options.UserName, options.Password ?? string.Empty, cancellationToken).ConfigureAwait(false); + } + + await client.SendAsync(message, cancellationToken).ConfigureAwait(false); + await client.DisconnectAsync(true, cancellationToken).ConfigureAwait(false); + } + + internal static MimeMessage Build(EmailOptions options, IReadOnlyList notifications) + { + var message = new MimeMessage(); + message.From.Add(MailboxAddress.Parse(options.From)); + foreach (var to in options.To) + { + message.To.Add(MailboxAddress.Parse(to)); + } + + message.Subject = Texts.Subject(notifications); + + var text = new StringBuilder(); + var html = new StringBuilder("
"); + foreach (var n in notifications) + { + text.AppendLine(Texts.Headline(n)); + html.Append("

").Append(WebUtility.HtmlEncode(Texts.Headline(n))).Append(""); + if (n.Summary is { } summary) + { + text.AppendLine(summary); + html.Append("
").Append(WebUtility.HtmlEncode(summary)); + } + + if (n.Url is { } url) + { + text.AppendLine(url.ToString()); + html.Append("
Åbn i SQLInsight"); + } + + text.AppendLine(); + html.Append("

"); + } + + html.Append("

Sendt af SQLInsight Pro

"); + message.Body = new BodyBuilder { TextBody = text.ToString(), HtmlBody = html.ToString() }.ToMessageBody(); + return message; + } +} diff --git a/src/SQLInsight.Notifications/HttpChannels.cs b/src/SQLInsight.Notifications/HttpChannels.cs new file mode 100644 index 0000000..e930ebd --- /dev/null +++ b/src/SQLInsight.Notifications/HttpChannels.cs @@ -0,0 +1,104 @@ +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using SQLInsight.Core.Findings; + +namespace SQLInsight.Notifications; + +/// +/// Microsoft Teams via a Workflows webhook, posting an Adaptive Card. (Office 365 connectors are retired.) +/// +public sealed class TeamsChannel(TeamsOptions options, HttpClient http) : INotificationChannel +{ + public string Name => "teams"; + public Severity MinSeverity => options.MinSeverity; + + public async Task SendAsync(IReadOnlyList notifications, CancellationToken cancellationToken) + { + var url = options.WebhookUrl ?? throw new InvalidOperationException("SQLInsight:Notifications:Teams:WebhookUrl mangler."); + using var response = await http.PostAsJsonAsync(url, BuildCard(notifications), cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + } + + internal static object BuildCard(IReadOnlyList notifications) + { + var body = new List + { + new { type = "TextBlock", text = Texts.Subject(notifications), weight = "Bolder", size = "Medium", wrap = true }, + }; + foreach (var n in notifications) + { + body.Add(new + { + type = "Container", + separator = true, + items = new object[] + { + new { type = "TextBlock", text = Texts.Headline(n), weight = "Bolder", color = Color(n), wrap = true }, + new { type = "TextBlock", text = n.Summary ?? string.Empty, wrap = true, isSubtle = true }, + }, + selectAction = n.Url is null ? null : new { type = "Action.OpenUrl", url = n.Url.ToString() }, + }); + } + + return new + { + type = "message", + attachments = new[] + { + new + { + contentType = "application/vnd.microsoft.card.adaptive", + content = new Dictionary + { + ["$schema"] = "http://adaptivecards.io/schemas/adaptive-card.json", + ["type"] = "AdaptiveCard", + ["version"] = "1.4", + ["body"] = body, + }, + }, + }, + }; + } + + private static string Color(Notification n) => n.Kind switch + { + NotificationKind.FindingResolved or NotificationKind.InstanceRecovered => "Good", + _ when n.Severity >= Severity.High => "Attention", + _ => "Warning", + }; +} + +/// +/// A generic JSON webhook for ticket systems, chat bots or automation. Optionally signed with HMAC-SHA256. +/// +public sealed class WebhookChannel(WebhookOptions options, HttpClient http) : INotificationChannel +{ + public const string SignatureHeader = "X-SQLInsight-Signature"; + + internal static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web) + { + Converters = { new System.Text.Json.Serialization.JsonStringEnumConverter() }, + }; + + public string Name => "webhook"; + public Severity MinSeverity => options.MinSeverity; + + public async Task SendAsync(IReadOnlyList notifications, CancellationToken cancellationToken) + { + var url = options.Url ?? throw new InvalidOperationException("SQLInsight:Notifications:Webhook:Url mangler."); + var body = JsonSerializer.Serialize(new { source = "SQLInsight Pro", notifications }, Json); + using var content = new StringContent(body, Encoding.UTF8, "application/json"); + if (!string.IsNullOrEmpty(options.Secret)) + { + content.Headers.Add(SignatureHeader, Sign(options.Secret, body)); + } + + using var response = await http.PostAsync(url, content, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + } + + public static string Sign(string secret, string body) => + "sha256=" + Convert.ToHexStringLower(HMACSHA256.HashData(Encoding.UTF8.GetBytes(secret), Encoding.UTF8.GetBytes(body))); +} diff --git a/src/SQLInsight.Notifications/Notification.cs b/src/SQLInsight.Notifications/Notification.cs new file mode 100644 index 0000000..8d7f979 --- /dev/null +++ b/src/SQLInsight.Notifications/Notification.cs @@ -0,0 +1,43 @@ +using SQLInsight.Core.Findings; + +namespace SQLInsight.Notifications; + +public enum NotificationKind +{ + FindingOpened, + FindingResolved, + InstanceUnreachable, + InstanceRecovered, +} + +/// +/// Something worth telling a person about. Carries no query text or evidence values, only what is needed to decide +/// whether to open the web UI. +/// +public sealed record Notification +{ + public required NotificationKind Kind { get; init; } + public required string Instance { get; init; } + public required string Title { get; init; } + public Severity Severity { get; init; } + public string? RuleId { get; init; } + public string? Subject { get; init; } + + /// The first sentence of the explanation. + public string? Summary { get; init; } + + /// Link to the finding or instance in the web UI, when a base URL is configured. + public Uri? Url { get; init; } + + public required DateTimeOffset At { get; init; } +} + +public interface INotificationChannel +{ + string Name { get; } + + /// Lowest severity this channel wants for new findings. Instance outages are always sent. + Severity MinSeverity { get; } + + Task SendAsync(IReadOnlyList notifications, CancellationToken cancellationToken); +} diff --git a/src/SQLInsight.Notifications/NotificationDispatcher.cs b/src/SQLInsight.Notifications/NotificationDispatcher.cs new file mode 100644 index 0000000..238836d --- /dev/null +++ b/src/SQLInsight.Notifications/NotificationDispatcher.cs @@ -0,0 +1,144 @@ +using Microsoft.Extensions.Logging; +using SQLInsight.Core.Findings; + +namespace SQLInsight.Notifications; + +/// +/// Sends notifications to every enabled channel: filtered by severity, one message per batch, and capped per hour so +/// a bad night does not produce hundreds of mails. A failing channel is logged and never stops monitoring. +/// +public sealed partial class NotificationDispatcher(IEnumerable channels, NotificationOptions options, TimeProvider time, ILogger logger) +{ + private readonly Dictionary> _sent = []; + private readonly Dictionary _suppressed = []; + private readonly Lock _lock = new(); + + public async Task DispatchAsync(IReadOnlyList notifications, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(notifications); + foreach (var channel in channels) + { + var relevant = notifications.Where(n => IsRelevant(channel, n)).ToList(); + if (relevant.Count == 0 || !TryReserve(channel.Name, relevant.Count)) + { + continue; + } + + try + { + await channel.SendAsync(relevant, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogSendFailed(logger, ex, channel.Name, relevant.Count); + } + } + } + + private bool IsRelevant(INotificationChannel channel, Notification n) => n.Kind switch + { + NotificationKind.FindingOpened => n.Severity >= channel.MinSeverity, + NotificationKind.FindingResolved => options.NotifyResolved && n.Severity >= channel.MinSeverity, + _ => true, + }; + + private bool TryReserve(string channel, int count) + { + lock (_lock) + { + var now = time.GetUtcNow(); + if (!_sent.TryGetValue(channel, out var sent)) + { + _sent[channel] = sent = new Queue(); + } + + while (sent.Count > 0 && sent.Peek() < now.AddHours(-1)) + { + sent.Dequeue(); + } + + if (sent.Count >= options.MaxMessagesPerHour) + { + _suppressed[channel] = _suppressed.GetValueOrDefault(channel) + count; + LogSuppressed(logger, channel, count, options.MaxMessagesPerHour); + return false; + } + + sent.Enqueue(now); + return true; + } + } + + /// Notifications suppressed by the hourly cap, per channel (for the UI and tests). + public IReadOnlyDictionary Suppressed + { + get + { + lock (_lock) + { + return new Dictionary(_suppressed); + } + } + } + + [LoggerMessage(Level = LogLevel.Error, Message = "Notification channel {Channel} failed to send {Count} notifications")] + private static partial void LogSendFailed(ILogger logger, Exception exception, string channel, int count); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Notification channel {Channel}: {Count} notifications suppressed, over {Max} messages per hour")] + private static partial void LogSuppressed(ILogger logger, string channel, int count, int max); +} + +/// +/// Builds notifications from what the agent saw. +/// +public static class NotificationFactory +{ + public static Notification Opened(string instance, long findingId, Finding finding, Uri? webUrl, DateTimeOffset at) + { + ArgumentNullException.ThrowIfNull(finding); + return new Notification + { + Kind = NotificationKind.FindingOpened, + Instance = instance, + Title = finding.Title, + Severity = finding.Severity, + RuleId = finding.RuleId, + Subject = finding.Subject, + Summary = Texts.FirstSentence(finding.Explanation), + Url = webUrl is null ? null : new Uri(webUrl, $"fund/{findingId}"), + At = at, + }; + } + + public static Notification Resolved(string instance, long findingId, string ruleId, string title, Severity severity, Uri? webUrl, DateTimeOffset at) => new() + { + Kind = NotificationKind.FindingResolved, + Instance = instance, + Title = title, + Severity = severity, + RuleId = ruleId, + Url = webUrl is null ? null : new Uri(webUrl, $"fund/{findingId}"), + At = at, + }; + + public static Notification Unreachable(string instance, string error, Uri? webUrl, DateTimeOffset at) => new() + { + Kind = NotificationKind.InstanceUnreachable, + Instance = instance, + Title = $"{instance} kan ikke nås", + Severity = Severity.Critical, + Summary = error, + Url = webUrl is null ? null : new Uri(webUrl, $"instanser/{Uri.EscapeDataString(instance)}"), + At = at, + }; + + public static Notification Recovered(string instance, Uri? webUrl, DateTimeOffset at) => new() + { + Kind = NotificationKind.InstanceRecovered, + Instance = instance, + Title = $"{instance} svarer igen", + Severity = Severity.Info, + Url = webUrl is null ? null : new Uri(webUrl, $"instanser/{Uri.EscapeDataString(instance)}"), + At = at, + }; +} diff --git a/src/SQLInsight.Notifications/NotificationOptions.cs b/src/SQLInsight.Notifications/NotificationOptions.cs new file mode 100644 index 0000000..c5b4552 --- /dev/null +++ b/src/SQLInsight.Notifications/NotificationOptions.cs @@ -0,0 +1,58 @@ +using SQLInsight.Core.Findings; + +namespace SQLInsight.Notifications; + +public sealed class NotificationOptions +{ + public const string SectionName = "SQLInsight:Notifications"; + + /// Base URL of SQLInsight Web, used for links, e.g. https://sqlinsight.domæne.local. + public Uri? WebUrl { get; set; } + + /// Also notify when a finding is resolved. + public bool NotifyResolved { get; set; } = true; + + /// At most this many messages per channel per hour; the rest are summarized in the next message. + public int MaxMessagesPerHour { get; set; } = 20; + + public EmailOptions Email { get; set; } = new(); + public TeamsOptions Teams { get; set; } = new(); + public WebhookOptions Webhook { get; set; } = new(); +} + +public sealed class EmailOptions +{ + public bool Enabled { get; set; } + public string Host { get; set; } = string.Empty; + public int Port { get; set; } = 587; + + /// None, StartTls (default) or SslOnConnect. + public string Security { get; set; } = "StartTls"; + + public string? UserName { get; set; } + public string? Password { get; set; } + public string From { get; set; } = string.Empty; + public IList To { get; } = []; + public Severity MinSeverity { get; set; } = Severity.High; +} + +public sealed class TeamsOptions +{ + public bool Enabled { get; set; } + + /// A Teams Workflows ("Post to a channel when a webhook request is received") URL. + public Uri? WebhookUrl { get; set; } + + public Severity MinSeverity { get; set; } = Severity.High; +} + +public sealed class WebhookOptions +{ + public bool Enabled { get; set; } + public Uri? Url { get; set; } + + /// When set, each request carries X-SQLInsight-Signature: sha256=HMAC(secret, body). + public string? Secret { get; set; } + + public Severity MinSeverity { get; set; } = Severity.Medium; +} diff --git a/src/SQLInsight.Notifications/SQLInsight.Notifications.csproj b/src/SQLInsight.Notifications/SQLInsight.Notifications.csproj new file mode 100644 index 0000000..ce45395 --- /dev/null +++ b/src/SQLInsight.Notifications/SQLInsight.Notifications.csproj @@ -0,0 +1,15 @@ + + + SQLInsight.Notifications + + + + + + + + + + + + diff --git a/src/SQLInsight.Notifications/Texts.cs b/src/SQLInsight.Notifications/Texts.cs new file mode 100644 index 0000000..c24d0cc --- /dev/null +++ b/src/SQLInsight.Notifications/Texts.cs @@ -0,0 +1,34 @@ +using SQLInsight.Core.Findings; + +namespace SQLInsight.Notifications; + +internal static class Texts +{ + public static string Severity(Severity severity) => severity switch + { + Core.Findings.Severity.Critical => "Kritisk", + Core.Findings.Severity.High => "Høj", + Core.Findings.Severity.Medium => "Middel", + Core.Findings.Severity.Low => "Lav", + _ => "Info", + }; + + public static string Headline(Notification n) => n.Kind switch + { + NotificationKind.FindingOpened => $"[{Severity(n.Severity)}] {n.Instance}: {n.Title}", + NotificationKind.FindingResolved => $"[Løst] {n.Instance}: {n.Title}", + NotificationKind.InstanceUnreachable => $"[Kan ikke nås] {n.Instance}", + NotificationKind.InstanceRecovered => $"[Svarer igen] {n.Instance}", + _ => n.Title, + }; + + public static string Subject(IReadOnlyList batch) => batch.Count == 1 + ? "SQLInsight: " + Headline(batch[0]) + : $"SQLInsight: {batch.Count} hændelser på {string.Join(", ", batch.Select(n => n.Instance).Distinct())}"; + + public static string FirstSentence(string text) + { + var end = text.IndexOf(". ", StringComparison.Ordinal); + return end < 0 ? text : text[..(end + 1)]; + } +} diff --git a/src/SQLInsight.Repository/InsightRepository.cs b/src/SQLInsight.Repository/InsightRepository.cs index 2dac9f4..82689d0 100644 --- a/src/SQLInsight.Repository/InsightRepository.cs +++ b/src/SQLInsight.Repository/InsightRepository.cs @@ -97,10 +97,13 @@ public sealed partial class InsightRepository(string connectionString) /// Stores the outcome of a rule evaluation and tracks findings: new ones start, existing ones are updated, and /// ones no longer reported by a rule that ran are resolved. A rule that could not run leaves its findings as they were. /// - public async Task SaveRuleResultsAsync(int instanceId, IReadOnlyList results, DateTimeOffset evaluatedAt, CancellationToken cancellationToken = default) + /// The findings that started and the ones that were resolved, for notifications. + public async Task SaveRuleResultsAsync(int instanceId, IReadOnlyList results, DateTimeOffset evaluatedAt, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(results); var now = Utc(evaluatedAt); + var opened = new List<(long, Finding)>(); + var resolved = new List(); await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false); await using var transaction = (SqlTransaction)await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); @@ -119,15 +122,16 @@ public sealed partial class InsightRepository(string connectionString) } var current = result.Findings.Select(f => f.Subject ?? string.Empty).ToList(); - await connection.ExecuteAsync(""" + resolved.AddRange(await connection.QueryAsync(""" UPDATE insight.Findings SET ResolvedAt = @now + OUTPUT inserted.FindingId, inserted.RuleId, inserted.Subject, inserted.Title, inserted.Severity WHERE InstanceId = @instanceId AND RuleId = @ruleId AND ResolvedAt IS NULL AND Subject NOT IN (SELECT value FROM OPENJSON(@current)); - """, new { instanceId, ruleId = result.RuleId, now, current = JsonSerializer.Serialize(current, Json) }, transaction).ConfigureAwait(false); + """, new { instanceId, ruleId = result.RuleId, now, current = JsonSerializer.Serialize(current, Json) }, transaction).ConfigureAwait(false)); foreach (var finding in result.Findings) { - await connection.ExecuteAsync(""" + var newId = await connection.QuerySingleOrDefaultAsync(""" UPDATE insight.Findings SET Severity = @severity, Title = @title, Explanation = @explanation, Recommendation = @recommendation, EvidenceJson = @evidence, ReferencesJson = @references, FixScript = @fixScript, LastSeenAt = @now @@ -135,6 +139,7 @@ public sealed partial class InsightRepository(string connectionString) IF @@ROWCOUNT = 0 INSERT insight.Findings (InstanceId, RuleId, Subject, Severity, Title, Explanation, Recommendation, EvidenceJson, ReferencesJson, FixScript, FirstSeenAt, LastSeenAt) + OUTPUT inserted.FindingId VALUES (@instanceId, @ruleId, @subject, @severity, @title, @explanation, @recommendation, @evidence, @references, @fixScript, @now, @now); """, new { @@ -150,10 +155,15 @@ public sealed partial class InsightRepository(string connectionString) fixScript = finding.FixScript, now, }, transaction).ConfigureAwait(false); + if (newId is { } id) + { + opened.Add((id, finding)); + } } } await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return new FindingChanges(opened, resolved); } /// @@ -342,6 +352,20 @@ public sealed partial class InsightRepository } } +public sealed record FindingChanges(IReadOnlyList<(long FindingId, Finding Finding)> Opened, IReadOnlyList Resolved) +{ + public static readonly FindingChanges None = new([], []); +} + +public sealed record ResolvedFinding +{ + public long FindingId { get; init; } + public string RuleId { get; init; } = string.Empty; + public string Subject { get; init; } = string.Empty; + public string Title { get; init; } = string.Empty; + public byte Severity { get; init; } +} + public static class Metrics { public const string CpuSqlServer = "cpu.sqlserver"; diff --git a/tests/SQLInsight.IntegrationTests/NotificationIntegrationTests.cs b/tests/SQLInsight.IntegrationTests/NotificationIntegrationTests.cs new file mode 100644 index 0000000..8ea100c --- /dev/null +++ b/tests/SQLInsight.IntegrationTests/NotificationIntegrationTests.cs @@ -0,0 +1,112 @@ +using System.Net.Http.Json; +using System.Text.Json; +using DotNet.Testcontainers.Builders; +using Microsoft.Extensions.Logging.Abstractions; +using SQLInsight.Agent; +using SQLInsight.Agent.Monitoring; +using SQLInsight.Collectors; +using SQLInsight.Connectivity; +using SQLInsight.Core.Findings; +using SQLInsight.Notifications; +using SQLInsight.Repository; +using SQLInsight.Rules; +using SQLInsight.Rules.Engine; + +namespace SQLInsight.IntegrationTests; + +[Collection(SqlServerTestGroup.Name)] +[Trait("Category", "Integration")] +public class NotificationIntegrationTests(SqlServerFixture sql) : IAsyncLifetime +{ + private readonly string _database = "SQLInsightNotify_" + Guid.NewGuid().ToString("N")[..8]; + private InsightRepository _repository = null!; + + public async Task InitializeAsync() + { + await sql.ExecuteAsSaAsync($"CREATE DATABASE [{_database}];"); + _repository = new InsightRepository(sql.DatabaseConnectionString(_database)); + await _repository.MigrateAsync(); + } + + public Task DisposeAsync() => sql.ExecuteAsSaAsync($"ALTER DATABASE [{_database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{_database}];"); + + private sealed class Recorder : INotificationChannel + { + public List Received { get; } = []; + public string Name => "recorder"; + public Severity MinSeverity => Severity.Info; + + public Task SendAsync(IReadOnlyList notifications, CancellationToken cancellationToken) + { + Received.AddRange(notifications); + return Task.CompletedTask; + } + } + + private InstanceMonitor Monitor(string name, Recorder recorder, string? password = null) => new( + new MonitoredInstance + { + Name = name, Server = sql.Server, Authentication = AuthenticationMode.SqlLogin, UserName = SqlServerFixture.AgentLogin, + Password = password ?? SqlServerFixture.AgentPassword, Provider = ConnectionProvider.SqlClient, TrustServerCertificate = true, + }, + BuiltInCollectors.Create(), new Dictionary(), new SqlConnectionFactory(), new RuleEngine(BuiltInRules.Create()), + _repository, NullLogger.Instance, + new NotificationDispatcher([recorder], new NotificationOptions(), TimeProvider.System, NullLogger.Instance), + new Uri("https://sqlinsight.test/")); + + [Fact] + public async Task New_findings_are_notified_once_and_outages_on_transition() + { + var recorder = new Recorder(); + var monitor = Monitor("notify", recorder); + var now = DateTimeOffset.UtcNow; + + await monitor.RunCycleAsync(now, CancellationToken.None); + var first = recorder.Received.Count(n => n.Kind == NotificationKind.FindingOpened); + await monitor.RunCycleAsync(now.AddMinutes(16), CancellationToken.None); + + Assert.True(first > 0); + Assert.Contains(recorder.Received, n => n.RuleId == "SQLI-CFG-001" && n.Url!.ToString().StartsWith("https://sqlinsight.test/fund/", StringComparison.Ordinal)); + // Findings still present in the second cycle are not notified again. + Assert.Single(recorder.Received, n => n.Kind == NotificationKind.FindingOpened && n.RuleId == "SQLI-CFG-001"); + + var down = new Recorder(); + var broken = Monitor("notify-down", down, password: "wrong"); + await broken.RunCycleAsync(now, CancellationToken.None); + await broken.RunCycleAsync(now.AddMinutes(1), CancellationToken.None); + Assert.Single(down.Received, n => n.Kind == NotificationKind.InstanceUnreachable); + } + + [Fact] + public async Task Email_is_delivered_through_a_real_smtp_server() + { + var mailpit = new ContainerBuilder("axllent/mailpit:latest") + .WithPortBinding(1025, true) + .WithPortBinding(8025, true) + .WithWaitStrategy(Wait.ForUnixContainer().UntilHttpRequestIsSucceeded(r => r.ForPort(8025).ForPath("/livez"))) + .Build(); + await mailpit.StartAsync(); + try + { + var options = new EmailOptions { Host = mailpit.Hostname, Port = mailpit.GetMappedPublicPort(1025), Security = "None", From = "sqlinsight@test.local" }; + options.To.Add("dba@test.local"); + var notification = NotificationFactory.Opened("prod-sql01", 7, new Finding + { + RuleId = "SQLI-DB-007", Severity = Severity.Critical, Title = "master har aldrig haft en fuld backup", + Explanation = "Uden backup kan databasen ikke gendannes.", Recommendation = "Tag backup.", + }, new Uri("https://sqlinsight.test/"), DateTimeOffset.UtcNow); + + await new EmailChannel(options).SendAsync([notification], CancellationToken.None); + + using var http = new HttpClient { BaseAddress = new Uri($"http://{mailpit.Hostname}:{mailpit.GetMappedPublicPort(8025)}/") }; + var messages = await http.GetFromJsonAsync("api/v1/messages"); + var message = messages.GetProperty("messages")[0]; + Assert.Equal("SQLInsight: [Kritisk] prod-sql01: master har aldrig haft en fuld backup", message.GetProperty("Subject").GetString()); + Assert.Equal("dba@test.local", message.GetProperty("To")[0].GetProperty("Address").GetString()); + } + finally + { + await mailpit.DisposeAsync(); + } + } +} diff --git a/tests/SQLInsight.IntegrationTests/SQLInsight.IntegrationTests.csproj b/tests/SQLInsight.IntegrationTests/SQLInsight.IntegrationTests.csproj index 78581d7..a3e8523 100644 --- a/tests/SQLInsight.IntegrationTests/SQLInsight.IntegrationTests.csproj +++ b/tests/SQLInsight.IntegrationTests/SQLInsight.IntegrationTests.csproj @@ -13,6 +13,7 @@ + diff --git a/tests/SQLInsight.Notifications.Tests/NotificationTests.cs b/tests/SQLInsight.Notifications.Tests/NotificationTests.cs new file mode 100644 index 0000000..f2e5d06 --- /dev/null +++ b/tests/SQLInsight.Notifications.Tests/NotificationTests.cs @@ -0,0 +1,160 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using SQLInsight.Core.Findings; + +namespace SQLInsight.Notifications.Tests; + +public class NotificationTests +{ + private static readonly DateTimeOffset Now = new(2026, 9, 29, 12, 0, 0, TimeSpan.Zero); + + private sealed class ManualTime(DateTimeOffset now) : TimeProvider + { + public DateTimeOffset Now { get; set; } = now; + public override DateTimeOffset GetUtcNow() => Now; + } + + private sealed class RecordingChannel(Severity min, bool fail = false) : INotificationChannel + { + public List> Batches { get; } = []; + public string Name => "recording"; + public Severity MinSeverity => min; + + public Task SendAsync(IReadOnlyList notifications, CancellationToken cancellationToken) + { + if (fail) + { + throw new HttpRequestException("down"); + } + + Batches.Add(notifications); + return Task.CompletedTask; + } + } + + private static Notification Opened(Severity severity, string title = "max server memory er ikke sat") => + NotificationFactory.Opened("prod-sql01", 42, new Finding + { + RuleId = "SQLI-CFG-001", Severity = severity, Title = title, + Explanation = "SQL Server må bruge al hukommelse. Mere tekst.", Recommendation = "Sæt den.", + }, new Uri("https://sqlinsight.local/"), Now); + + private static NotificationDispatcher Dispatcher(INotificationChannel channel, TimeProvider? time = null, int max = 20, bool resolved = true) => + new([channel], new NotificationOptions { MaxMessagesPerHour = max, NotifyResolved = resolved }, time ?? new ManualTime(Now), NullLogger.Instance); + + [Fact] + public async Task Findings_below_the_channels_severity_are_not_sent_but_outages_always_are() + { + var channel = new RecordingChannel(Severity.High); + + await Dispatcher(channel).DispatchAsync([Opened(Severity.Medium), Opened(Severity.Critical), NotificationFactory.Unreachable("legacy", "timeout", null, Now)], default); + + var batch = Assert.Single(channel.Batches); + Assert.Equal([NotificationKind.FindingOpened, NotificationKind.InstanceUnreachable], batch.Select(n => n.Kind)); + } + + [Fact] + public async Task Resolved_can_be_switched_off() + { + var channel = new RecordingChannel(Severity.Low); + + await Dispatcher(channel, resolved: false).DispatchAsync([NotificationFactory.Resolved("x", 1, "R", "t", Severity.High, null, Now)], default); + + Assert.Empty(channel.Batches); + } + + [Fact] + public async Task Messages_are_capped_per_hour() + { + var time = new ManualTime(Now); + var channel = new RecordingChannel(Severity.Low); + var dispatcher = Dispatcher(channel, time, max: 2); + + for (var i = 0; i < 5; i++) + { + await dispatcher.DispatchAsync([Opened(Severity.High)], default); + } + + Assert.Equal(2, channel.Batches.Count); + Assert.Equal(3, dispatcher.Suppressed["recording"]); + + time.Now = Now.AddMinutes(61); + await dispatcher.DispatchAsync([Opened(Severity.High)], default); + Assert.Equal(3, channel.Batches.Count); + } + + [Fact] + public async Task A_failing_channel_does_not_throw() + { + await Dispatcher(new RecordingChannel(Severity.Low, fail: true)).DispatchAsync([Opened(Severity.High)], default); + } + + [Fact] + public void Opened_notification_links_to_the_finding_and_keeps_only_the_first_sentence() + { + var n = Opened(Severity.High); + + Assert.Equal("https://sqlinsight.local/fund/42", n.Url!.ToString()); + Assert.Equal("SQL Server må bruge al hukommelse.", n.Summary); + } + + [Fact] + public void Email_is_html_encoded_and_has_a_useful_subject() + { + var options = new EmailOptions { From = "sqlinsight@x.local" }; + options.To.Add("dba@x.local"); + + var message = EmailChannel.Build(options, [Opened(Severity.Critical, "")]); + + Assert.Equal("SQLInsight: [Kritisk] prod-sql01: ", message.Subject); + Assert.Contains("<script>", message.HtmlBody, StringComparison.Ordinal); + Assert.DoesNotContain("