diff --git a/src/SQLInsight.Agent/AgentOptions.cs b/src/SQLInsight.Agent/AgentOptions.cs index d2dd35a..91fa309 100644 --- a/src/SQLInsight.Agent/AgentOptions.cs +++ b/src/SQLInsight.Agent/AgentOptions.cs @@ -1,4 +1,5 @@ using SQLInsight.Connectivity; +using SQLInsight.Repository; namespace SQLInsight.Agent; @@ -6,14 +7,34 @@ public sealed class AgentOptions { public const string SectionName = "SQLInsight"; - public TimeSpan Interval { get; set; } = TimeSpan.FromMinutes(15); + /// How often the scheduler checks whether collectors are due. + public TimeSpan Tick { get; set; } = TimeSpan.FromSeconds(15); + + /// How many instances are checked at the same time. + public int MaxParallelInstances { get; set; } = 4; + + /// Overrides of collector intervals by collector name, e.g. "blocking": "00:00:30". + public IDictionary CollectorIntervals { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public RepositoryOptions Repository { get; set; } = new(); + + public RetentionOptions Retention { get; set; } = new(); public IList Instances { get; } = []; } +public sealed class RepositoryOptions +{ + /// + /// Connection string to the SQLInsight repository database. Use a SQL Server 2016+ instance other than the + /// ones being monitored. + /// + public string? ConnectionString { get; set; } +} + public sealed class MonitoredInstance { - /// Stable name shown in the UI and used as the key in SQLInsight Hub. + /// Stable name shown in the UI and used as the key in the repository and SQLInsight Hub. public string Name { get; set; } = string.Empty; public string Server { get; set; } = string.Empty; @@ -23,6 +44,7 @@ public sealed class MonitoredInstance public ConnectionProvider Provider { get; set; } = ConnectionProvider.Auto; public EncryptionMode Encryption { get; set; } = EncryptionMode.Mandatory; public bool TrustServerCertificate { get; set; } + public string OdbcDriver { get; set; } = "ODBC Driver 18 for SQL Server"; public ConnectionTarget ToTarget() => new() { @@ -33,5 +55,6 @@ public sealed class MonitoredInstance Provider = Provider, Encryption = Encryption, TrustServerCertificate = TrustServerCertificate, + OdbcDriver = OdbcDriver, }; } diff --git a/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs b/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs new file mode 100644 index 0000000..dbf97cb --- /dev/null +++ b/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs @@ -0,0 +1,130 @@ +using SQLInsight.Collectors; +using SQLInsight.Connectivity; +using SQLInsight.Core.Collection; +using SQLInsight.Core.Instances; +using SQLInsight.Repository; +using SQLInsight.Rules.Engine; + +namespace SQLInsight.Agent.Monitoring; + +/// +/// Monitors one instance: runs collectors when they are due, keeps the latest data and each collector's previous +/// result (for deltas), evaluates the rules and stores everything. Not thread-safe; one cycle at a time. +/// +public sealed partial class InstanceMonitor( + MonitoredInstance instance, + IReadOnlyList collectors, + IReadOnlyDictionary intervalOverrides, + ISqlConnectionFactory connectionFactory, + RuleEngine ruleEngine, + InsightRepository repository, + ILogger logger) +{ + private readonly CollectedData _data = new(); + private readonly Dictionary _previous = []; + private readonly Dictionary _nextDue = []; + + public string Name => instance.Name; + + public TimeSpan IntervalOf(ICollector collector) => + intervalOverrides.TryGetValue(collector.Name, out var interval) ? interval : collector.DefaultInterval; + + public bool IsDue(DateTimeOffset now) => collectors.Any(c => !_nextDue.TryGetValue(c.Name, out var due) || due <= now); + + /// Runs the collectors that are due, then evaluates all rules. Returns the rule results, or null if nothing was due. + public async Task?> RunCycleAsync(DateTimeOffset now, CancellationToken cancellationToken) + { + var due = collectors.Where(c => !_nextDue.TryGetValue(c.Name, out var at) || at <= now).ToList(); + if (due.Count == 0) + { + return null; + } + + foreach (var collector in due) + { + _nextDue[collector.Name] = now + IntervalOf(collector); + } + + OpenedConnection opened; + InstanceInfo info; + int instanceId; + try + { + opened = await connectionFactory.OpenAsync(instance.ToTarget(), cancellationToken).ConfigureAwait(false); + info = await InstanceProbe.DetectAsync(opened.Connection, cancellationToken).ConfigureAwait(false); + instanceId = await repository.UpsertInstanceAsync(instance.Name, info, opened.Provider.ToString(), opened.UsedLegacyFallback, now, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogConnectFailed(logger, ex, instance.Name); + await repository.RecordInstanceErrorAsync(instance.Name, ex.Message, now, cancellationToken).ConfigureAwait(false); + return null; + } + + await using (opened.ConfigureAwait(false)) + { + if (opened.UsedLegacyFallback) + { + LogLegacyFallback(logger, instance.Name); + } + + var capabilities = InstanceCapabilities.From(info); + foreach (var collector in due) + { + await CollectAsync(collector, opened, info, capabilities, instanceId, now, cancellationToken).ConfigureAwait(false); + } + + var results = ruleEngine.Evaluate(new RuleContext(info, capabilities, now, _data)); + await repository.SaveRuleResultsAsync(instanceId, results, now, cancellationToken).ConfigureAwait(false); + var findings = 0; + var notMeasurable = 0; + foreach (var result in results) + { + findings += result.Findings.Count; + notMeasurable += result.Outcome is RuleOutcome.NotMeasurable or RuleOutcome.Error ? 1 : 0; + } + + LogCycle(logger, instance.Name, due.Count, findings, notMeasurable); + return results; + } + } + + private async Task CollectAsync( + ICollector collector, OpenedConnection opened, InstanceInfo info, InstanceCapabilities capabilities, int instanceId, DateTimeOffset now, CancellationToken cancellationToken) + { + var missing = capabilities.Missing(collector.RequiredCapabilities); + if (missing.Count > 0) + { + _data.MarkUnavailable(collector.DataType, $"Ikke målbar på {info.Release.DisplayName()}: kræver {string.Join(", ", missing)}."); + return; + } + + try + { + _previous.TryGetValue(collector.Name, out var previous); + var result = await collector.CollectAsync(new CollectorContext(opened.Connection, info, capabilities, now, previous, cancellationToken)).ConfigureAwait(false); + _previous[collector.Name] = result; + _data.Set(collector.DataType, result); + await repository.SaveCollectedAsync(instanceId, collector.Name, result, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + // Typically a permission the agent lacks on this instance. Rules depending on it say why. + LogCollectorFailed(logger, ex, instance.Name, collector.Name); + _data.MarkUnavailable(collector.DataType, $"Collector '{collector.Name}' fejlede: {ex.Message}"); + } + } + + [LoggerMessage(Level = LogLevel.Error, Message = "{Instance}: could not connect")] + private static partial void LogConnectFailed(ILogger logger, Exception exception, string instance); + + [LoggerMessage(Level = LogLevel.Warning, Message = "{Instance}: SqlClient could not connect, using the legacy ODBC connection. Encryption may be weaker.")] + private static partial void LogLegacyFallback(ILogger logger, string instance); + + [LoggerMessage(Level = LogLevel.Warning, Message = "{Instance}: collector {Collector} failed")] + private static partial void LogCollectorFailed(ILogger logger, Exception exception, string instance, string collector); + + [LoggerMessage(Level = LogLevel.Information, Message = "{Instance}: {Collectors} collectors, {Findings} findings, {NotMeasurable} rules not measurable")] + private static partial void LogCycle(ILogger logger, string instance, int collectors, int findings, int notMeasurable); +} diff --git a/src/SQLInsight.Agent/Monitoring/MonitoringService.cs b/src/SQLInsight.Agent/Monitoring/MonitoringService.cs new file mode 100644 index 0000000..83b2ce3 --- /dev/null +++ b/src/SQLInsight.Agent/Monitoring/MonitoringService.cs @@ -0,0 +1,128 @@ +using Microsoft.Extensions.Options; +using SQLInsight.Collectors; +using SQLInsight.Connectivity; +using SQLInsight.Repository; +using SQLInsight.Rules.Engine; + +namespace SQLInsight.Agent.Monitoring; + +/// +/// The agent's main loop: migrates the repository, then checks every instance on each tick and runs maintenance hourly. +/// +public sealed partial class MonitoringService( + IOptions options, + ISqlConnectionFactory connectionFactory, + RuleEngine ruleEngine, + InsightRepository repository, + TimeProvider timeProvider, + ILoggerFactory loggerFactory) : BackgroundService +{ + private readonly ILogger _logger = loggerFactory.CreateLogger(); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var settings = options.Value; + await MigrateWithRetryAsync(stoppingToken).ConfigureAwait(false); + + var intervals = new Dictionary(settings.CollectorIntervals, StringComparer.OrdinalIgnoreCase); + var monitors = settings.Instances + .Select(i => new InstanceMonitor(i, BuiltInCollectors.Create(), intervals, connectionFactory, ruleEngine, repository, loggerFactory.CreateLogger($"SQLInsight.Instance.{i.Name}"))) + .ToList(); + if (monitors.Count == 0) + { + LogNoInstances(_logger); + } + + using var throttle = new SemaphoreSlim(Math.Max(1, settings.MaxParallelInstances)); + var running = new Dictionary(); + var nextMaintenance = timeProvider.GetUtcNow(); + using var timer = new PeriodicTimer(settings.Tick, timeProvider); + + do + { + var now = timeProvider.GetUtcNow(); + foreach (var monitor in monitors) + { + // A slow instance keeps its own cycle; it never delays the others. + if (running.TryGetValue(monitor, out var task) && !task.IsCompleted || !monitor.IsDue(now)) + { + continue; + } + + running[monitor] = RunThrottledAsync(monitor, now, throttle, stoppingToken); + } + + if (now >= nextMaintenance) + { + nextMaintenance = now.AddHours(1); + await RunMaintenanceAsync(settings, now, stoppingToken).ConfigureAwait(false); + } + } + while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); + + await Task.WhenAll(running.Values).ConfigureAwait(false); + } + + private async Task RunThrottledAsync(InstanceMonitor monitor, DateTimeOffset now, SemaphoreSlim throttle, CancellationToken cancellationToken) + { + await throttle.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await monitor.RunCycleAsync(now, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + // Typically the repository is unreachable. Try again next time. + LogCycleFailed(_logger, ex, monitor.Name); + } + finally + { + throttle.Release(); + } + } + + private async Task MigrateWithRetryAsync(CancellationToken cancellationToken) + { + for (var attempt = 1; ; attempt++) + { + try + { + var applied = await repository.MigrateAsync(cancellationToken).ConfigureAwait(false); + LogMigrated(_logger, applied); + return; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogMigrationFailed(_logger, ex, attempt); + await Task.Delay(TimeSpan.FromSeconds(Math.Min(60, 5 * attempt)), timeProvider, cancellationToken).ConfigureAwait(false); + } + } + } + + private async Task RunMaintenanceAsync(AgentOptions settings, DateTimeOffset now, CancellationToken cancellationToken) + { + try + { + await repository.RunMaintenanceAsync(settings.Retention, now, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogMaintenanceFailed(_logger, ex); + } + } + + [LoggerMessage(Level = LogLevel.Warning, Message = "No instances configured under 'SQLInsight:Instances'.")] + private static partial void LogNoInstances(ILogger logger); + + [LoggerMessage(Level = LogLevel.Information, Message = "Repository ready ({Applied} migrations applied).")] + private static partial void LogMigrated(ILogger logger, int applied); + + [LoggerMessage(Level = LogLevel.Error, Message = "Repository migration failed (attempt {Attempt}); retrying.")] + private static partial void LogMigrationFailed(ILogger logger, Exception exception, int attempt); + + [LoggerMessage(Level = LogLevel.Error, Message = "{Instance}: cycle failed")] + private static partial void LogCycleFailed(ILogger logger, Exception exception, string instance); + + [LoggerMessage(Level = LogLevel.Error, Message = "Repository maintenance failed.")] + private static partial void LogMaintenanceFailed(ILogger logger, Exception exception); +} diff --git a/src/SQLInsight.Agent/Program.cs b/src/SQLInsight.Agent/Program.cs index 4d6d23e..55b754c 100644 --- a/src/SQLInsight.Agent/Program.cs +++ b/src/SQLInsight.Agent/Program.cs @@ -1,5 +1,8 @@ +using Microsoft.Extensions.Options; using SQLInsight.Agent; +using SQLInsight.Agent.Monitoring; using SQLInsight.Connectivity; +using SQLInsight.Repository; using SQLInsight.Rules; using SQLInsight.Rules.Engine; @@ -12,7 +15,18 @@ builder.Services.Configure(builder.Configuration.GetSection(AgentO builder.Services.AddSingleton(TimeProvider.System); builder.Services.AddSingleton(); builder.Services.AddSingleton(new RuleEngine(BuiltInRules.Create())); -builder.Services.AddHostedService(); +builder.Services.AddSingleton(sp => +{ + var connectionString = sp.GetRequiredService>().Value.Repository.ConnectionString; + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new InvalidOperationException( + "SQLInsight:Repository:ConnectionString mangler. Angiv forbindelsen til repository-databasen (SQL Server 2016+, ikke en af de overvågede instanser)."); + } + + return new InsightRepository(connectionString); +}); +builder.Services.AddHostedService(); var host = builder.Build(); host.Run(); diff --git a/src/SQLInsight.Agent/SQLInsight.Agent.csproj b/src/SQLInsight.Agent/SQLInsight.Agent.csproj index 49f26d5..2fc5325 100644 --- a/src/SQLInsight.Agent/SQLInsight.Agent.csproj +++ b/src/SQLInsight.Agent/SQLInsight.Agent.csproj @@ -7,6 +7,8 @@ + + diff --git a/src/SQLInsight.Agent/Worker.cs b/src/SQLInsight.Agent/Worker.cs deleted file mode 100644 index 55e9b57..0000000 --- a/src/SQLInsight.Agent/Worker.cs +++ /dev/null @@ -1,91 +0,0 @@ -using Microsoft.Extensions.Options; -using SQLInsight.Connectivity; -using SQLInsight.Core.Findings; -using SQLInsight.Core.Instances; -using SQLInsight.Rules.Engine; - -namespace SQLInsight.Agent; - -public sealed partial class Worker( - ISqlConnectionFactory connectionFactory, - RuleEngine ruleEngine, - IOptions options, - TimeProvider timeProvider, - ILogger logger) : BackgroundService -{ - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - var settings = options.Value; - if (settings.Instances.Count == 0) - { - LogNoInstances(logger); - } - - using var timer = new PeriodicTimer(settings.Interval, timeProvider); - do - { - foreach (var instance in settings.Instances) - { - await CheckInstanceAsync(instance, stoppingToken).ConfigureAwait(false); - } - } - while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); - } - - private async Task CheckInstanceAsync(MonitoredInstance instance, CancellationToken cancellationToken) - { - try - { - await using var opened = await connectionFactory.OpenAsync(instance.ToTarget(), cancellationToken).ConfigureAwait(false); - if (opened.UsedLegacyFallback) - { - LogLegacyFallback(logger, instance.Name); - } - - var info = await InstanceProbe.DetectAsync(opened.Connection, cancellationToken).ConfigureAwait(false); - var context = new RuleContext(info, InstanceCapabilities.From(info), timeProvider.GetUtcNow()); - LogDetected(logger, instance.Name, info.Release, info.Version, info.Edition); - - foreach (var result in ruleEngine.Evaluate(context)) - { - switch (result.Outcome) - { - case RuleOutcome.Findings: - foreach (var finding in result.Findings) - { - LogFinding(logger, instance.Name, finding.Severity, result.RuleId, finding.Title); - } - break; - case RuleOutcome.NotMeasurable: - case RuleOutcome.Error: - LogNotEvaluated(logger, instance.Name, result.RuleId, result.Outcome, result.Reason); - break; - case RuleOutcome.Passed: - break; - } - } - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - LogInstanceFailed(logger, ex, instance.Name); - } - } - - [LoggerMessage(Level = LogLevel.Warning, Message = "No instances configured under 'SQLInsight:Instances'.")] - private static partial void LogNoInstances(ILogger logger); - - [LoggerMessage(Level = LogLevel.Warning, Message = "{Instance}: SqlClient could not connect, using the legacy ODBC connection. Encryption may be weaker.")] - private static partial void LogLegacyFallback(ILogger logger, string instance); - - [LoggerMessage(Level = LogLevel.Information, Message = "{Instance}: {Release} ({Version}, {Edition})")] - private static partial void LogDetected(ILogger logger, string instance, SqlServerRelease release, SqlServerVersion version, string edition); - - [LoggerMessage(Level = LogLevel.Warning, Message = "{Instance}: [{Severity}] {RuleId} {Title}")] - private static partial void LogFinding(ILogger logger, string instance, Severity severity, string ruleId, string title); - - [LoggerMessage(Level = LogLevel.Information, Message = "{Instance}: {RuleId} {Outcome}: {Reason}")] - private static partial void LogNotEvaluated(ILogger logger, string instance, string ruleId, RuleOutcome outcome, string? reason); - - [LoggerMessage(Level = LogLevel.Error, Message = "{Instance}: check failed")] - private static partial void LogInstanceFailed(ILogger logger, Exception exception, string instance); -} diff --git a/src/SQLInsight.Agent/appsettings.json b/src/SQLInsight.Agent/appsettings.json index c0de34b..30ff676 100644 --- a/src/SQLInsight.Agent/appsettings.json +++ b/src/SQLInsight.Agent/appsettings.json @@ -6,7 +6,19 @@ } }, "SQLInsight": { - "Interval": "00:15:00", + "Repository": { + "ConnectionString": "Server=sqlmon01.eksempel.local;Database=SQLInsight;Integrated Security=true;Encrypt=true" + }, + "Tick": "00:00:15", + "MaxParallelInstances": 4, + "CollectorIntervals": { + }, + "Retention": { + "RawDays": 14, + "HourlyDays": 400, + "EventDays": 90, + "ResolvedFindingDays": 365 + }, "Instances": [ { "Name": "eksempel-sql01", diff --git a/tests/SQLInsight.IntegrationTests/AgentPipelineTests.cs b/tests/SQLInsight.IntegrationTests/AgentPipelineTests.cs new file mode 100644 index 0000000..369c1e6 --- /dev/null +++ b/tests/SQLInsight.IntegrationTests/AgentPipelineTests.cs @@ -0,0 +1,96 @@ +using Microsoft.Extensions.Logging.Abstractions; +using SQLInsight.Agent; +using SQLInsight.Agent.Monitoring; +using SQLInsight.Collectors; +using SQLInsight.Connectivity; +using SQLInsight.Core.Findings; +using SQLInsight.Repository; +using SQLInsight.Rules; +using SQLInsight.Rules.Engine; + +namespace SQLInsight.IntegrationTests; + +/// +/// The agent's real pipeline end to end: least-privilege connection, all collectors, all rules, repository. +/// +[Collection(SqlServerTestGroup.Name)] +[Trait("Category", "Integration")] +public class AgentPipelineTests(SqlServerFixture sql) : IAsyncLifetime +{ + private readonly string _database = "SQLInsightAgent_" + Guid.NewGuid().ToString("N")[..8]; + private InsightRepository _repository = null!; + + public async Task InitializeAsync() + { + await sql.ExecuteAsSaAsync($"CREATE DATABASE [{_database}];"); + _repository = new InsightRepository(sql.DatabaseConnectionString(_database)); + await _repository.MigrateAsync(); + } + + public Task DisposeAsync() => sql.ExecuteAsSaAsync($"ALTER DATABASE [{_database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{_database}];"); + + private InstanceMonitor Monitor(string name, string? password = null) => new( + new MonitoredInstance + { + Name = name, + Server = sql.Server, + Authentication = AuthenticationMode.SqlLogin, + UserName = SqlServerFixture.AgentLogin, + Password = password ?? SqlServerFixture.AgentPassword, + Provider = ConnectionProvider.SqlClient, + TrustServerCertificate = true, + }, + BuiltInCollectors.Create(), + new Dictionary(), + new SqlConnectionFactory(), + new RuleEngine(BuiltInRules.Create()), + _repository, + NullLogger.Instance); + + [Fact] + public async Task Two_cycles_store_findings_rule_states_and_time_series() + { + var monitor = Monitor("container"); + var start = DateTimeOffset.UtcNow; + + var first = await monitor.RunCycleAsync(start, CancellationToken.None); + await sql.ExecuteAsSaAsync("SELECT COUNT(*) FROM sys.objects a CROSS JOIN sys.objects b;"); + var nothingDue = await monitor.RunCycleAsync(start.AddSeconds(10), CancellationToken.None); + var second = await monitor.RunCycleAsync(start.AddMinutes(16), CancellationToken.None); + + Assert.NotNull(first); + Assert.Null(nothingDue); + Assert.NotNull(second); + Assert.Equal(BuiltInRules.Create().Count, second.Count); + Assert.DoesNotContain(second, r => r.Outcome == RuleOutcome.Error); + + var instance = await _repository.GetInstanceAsync("container"); + Assert.NotNull(instance); + Assert.False(instance.IsFailing); + + var findings = await _repository.GetActiveFindingsAsync(instance.InstanceId); + // A fresh container has well-known problems: default max server memory and master never backed up. + Assert.Contains(findings, f => f.RuleId == "SQLI-CFG-001" && f.Severity == Severity.High); + Assert.Contains(findings, f => f.RuleId == "SQLI-DB-007" && f.Subject == "database: master"); + + var states = await _repository.GetRuleStatesAsync(instance.InstanceId); + Assert.Equal(BuiltInRules.Create().Count, states.Count); + + // Wait stats need two samples; after the second cycle they are measurable. + Assert.NotEqual(RuleOutcome.NotMeasurable, states.Single(s => s.RuleId == "SQLI-WAIT-003").Outcome); + Assert.NotEmpty(await _repository.GetTopWaitsAsync(instance.InstanceId, start.AddHours(-1))); + } + + [Fact] + public async Task Unreachable_instance_is_recorded_as_failing() + { + var monitor = Monitor("wrong-password", password: "not-the-password"); + + var result = await monitor.RunCycleAsync(DateTimeOffset.UtcNow, CancellationToken.None); + + Assert.Null(result); + var instance = await _repository.GetInstanceAsync("wrong-password"); + Assert.True(instance!.IsFailing); + Assert.Contains("Login failed", instance.LastError, StringComparison.Ordinal); + } +} diff --git a/tests/SQLInsight.IntegrationTests/SQLInsight.IntegrationTests.csproj b/tests/SQLInsight.IntegrationTests/SQLInsight.IntegrationTests.csproj index 93983c2..f08f1f4 100644 --- a/tests/SQLInsight.IntegrationTests/SQLInsight.IntegrationTests.csproj +++ b/tests/SQLInsight.IntegrationTests/SQLInsight.IntegrationTests.csproj @@ -10,6 +10,7 @@ +