using System.Data.Common; using SQLInsight.Collectors.Discovery; using SQLInsight.Connectivity; using SQLInsight.Core.Configuration; using SQLInsight.Core.Instances; using SQLInsight.Repository; namespace SQLInsight.Agent.Configuration; /// Set by the monitoring service when the repository is migrated and the configuration file mirrored. public sealed class RepositoryReadiness { private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); public Task Ready => _ready.Task; public void MarkReady() => _ready.TrySetResult(); } /// /// Runs what SQLInsight Web asks for (insight.AgentRequests): connection tests, database discovery and instance /// discovery. They run here, as the agent's own identity, because that is what connects to the instances in production; /// a test from the web server would prove nothing. Also refreshes every instance's database list hourly, so new /// databases are noticed (and monitored if the instance says so), and publishes the agent's public key. /// Nothing is written to the monitored instances; results go to the repository only. /// public sealed partial class AgentRequestService( InsightRepository repository, InstanceConfigurationStore store, AgentCredentialKey key, ISqlConnectionFactory connectionFactory, RepositoryReadiness readiness, TimeProvider timeProvider, ILogger logger, Servers.ServerRequestHandler? serverRequests = null) : BackgroundService { /// How often the queue is looked at. Short, so the web UI gets an answer within seconds. public static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(2); /// How often every instance's databases are listed again without being asked. public static readonly TimeSpan DiscoveryInterval = TimeSpan.FromHours(1); public static readonly TimeSpan RequestTimeout = TimeSpan.FromMinutes(2); /// The least time between two routine discoveries of one instance, also when the last one could not be stored. public static readonly TimeSpan RoutineRetry = TimeSpan.FromMinutes(15); private readonly Dictionary _routineAttempts = []; protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await readiness.Ready.WaitAsync(stoppingToken).ConfigureAwait(false); var nextHousekeeping = DateTimeOffset.MinValue; using var timer = new PeriodicTimer(PollInterval, timeProvider); do { try { var now = timeProvider.GetUtcNow(); if (now >= nextHousekeeping) { nextHousekeeping = now.AddMinutes(10); await repository.PublishAgentKeyAsync(key.Describe(now), stoppingToken).ConfigureAwait(false); await repository.CleanUpAgentRequestsAsync(now, StaleAfter, TimeSpan.FromMinutes(30), TimeSpan.FromDays(30), stoppingToken) .ConfigureAwait(false); } // Everything waiting first, then at most one routine discovery per round. while (await repository.ClaimAgentRequestAsync(timeProvider.GetUtcNow(), stoppingToken).ConfigureAwait(false) is { } request) { await HandleAsync(request, stoppingToken).ConfigureAwait(false); } await RefreshOneAsync(timeProvider.GetUtcNow(), stoppingToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { // Typically the repository is unreachable. Try again next round. LogRoundFailed(logger, ex); } } while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); } /// Runs one request and stores its result. A failure is the request's result, never the service's. public async Task HandleAsync(AgentRequestRow request, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(request); var limit = TimeoutOf(request.Kind); using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); timeout.CancelAfter(limit); string? result = null; string? error = null; try { result = request.Kind switch { AgentRequestKind.ConnectionTest => AgentRequestJson.Serialize(await TestAsync(Require(request), timeout.Token).ConfigureAwait(false)), AgentRequestKind.DatabaseDiscovery => AgentRequestJson.Serialize(await DiscoverAsync(Require(request), timeout.Token).ConfigureAwait(false)), AgentRequestKind.InstanceDiscovery => AgentRequestJson.Serialize(await InstanceFinder.FindAsync( AgentRequestJson.Deserialize(request.Parameters)?.Host, TimeSpan.FromSeconds(3), timeout.Token).ConfigureAwait(false)), _ when serverRequests is not null && Servers.ServerRequestHandler.Handles(request.Kind) => await serverRequests.HandleAsync(request, timeout.Token).ConfigureAwait(false), _ => throw new InvalidOperationException($"Ukendt anmodning: {request.Kind}."), }; } catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) { error = $"Agenten fik ikke svar inden for {limit.TotalMinutes:0} minutter."; } // Anything else (a lock timeout, a repository error while storing a discovery …) is the request's result too, so it // is never left as Running for the web UI to wait on. catch (Exception ex) when (ex is not OperationCanceledException) { error = ex.Message; } LogHandled(logger, request.RequestId, request.Kind, error ?? "ok"); await repository.CompleteAgentRequestAsync(request.RequestId, error is null, result, error, timeProvider.GetUtcNow(), CancellationToken.None).ConfigureAwait(false); } /// How long a request may take: a synthetic check run now may take its own timeout plus the lookup of its service. public TimeSpan TimeoutOf(string kind) => serverRequests is not null && Servers.ServerRequestHandler.Handles(kind) ? Longest(RequestTimeout, Servers.ServerRequestHandler.RequestTimeout) : RequestTimeout; /// A request still Running after this was lost (the agent stopped): the longest request plus a minute. private static TimeSpan StaleAfter => Longest(RequestTimeout, Servers.ServerRequestHandler.RequestTimeout) + TimeSpan.FromMinutes(1); private static TimeSpan Longest(TimeSpan a, TimeSpan b) => a > b ? a : b; private static int Require(AgentRequestRow request) => request.ConfiguredInstanceId ?? throw new InvalidOperationException("Anmodningen mangler en instans."); /// Connects as the agent and checks its rights. Not reaching the instance is a result, with a hint, not an error. public async Task TestAsync(int instanceId, CancellationToken cancellationToken) { var (_, instance) = await store.GetAsync(instanceId, cancellationToken).ConfigureAwait(false); OpenedConnection opened; try { opened = await connectionFactory.OpenAsync(instance.ToTarget(), cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { return new ConnectionTestResult { Connected = false, Error = ex.Message, Hint = Hint(ex, instance) }; } await using (opened.ConfigureAwait(false)) { var info = await InstanceProbe.DetectAsync(opened.Connection, cancellationToken).ConfigureAwait(false); var notes = new List(); if (opened.UsedLegacyFallback) { notes.Add("Forbindelsen bruger den ældre ODBC-driver, fordi instansen er for gammel til SqlClient. Krypteringen kan være svagere."); } var result = new ConnectionTestResult { Connected = true, Provider = opened.Provider.ToString(), UsedLegacyFallback = opened.UsedLegacyFallback, ProductVersion = info.Version.ToString(), Release = info.Release.DisplayName(), Edition = info.Edition, ServerName = info.ServerName, Notes = notes, }; return await PermissionCheck.CheckAsync(opened.Connection, info, result, cancellationToken).ConfigureAwait(false); } } /// Lists the instance's databases and stores them. New ones are monitored when the instance monitors new databases. public async Task DiscoverAsync(int instanceId, CancellationToken cancellationToken) { var (_, instance) = await store.GetAsync(instanceId, cancellationToken).ConfigureAwait(false); List databases; try { await using var opened = await connectionFactory.OpenAsync(instance.ToTarget(), cancellationToken).ConfigureAwait(false); var info = await InstanceProbe.DetectAsync(opened.Connection, cancellationToken).ConfigureAwait(false); databases = await DatabaseDiscovery.DiscoverAsync(opened.Connection, info, cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { await repository.RecordDiscoveryErrorAsync(instanceId, ex.Message, timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); throw new DiscoveryFailedException(ex.Message + " " + Hint(ex, instance), ex); } return await repository.SaveDiscoveredDatabasesAsync(instanceId, databases, timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); } /// The routine discovery of the active instance that waited longest, if one is due. private async Task RefreshOneAsync(DateTimeOffset now, CancellationToken cancellationToken) { // Also waits on the agent's own last attempt: if storing the result fails (the repository), LastDiscoveryAt is // not set, and without this the same instance would be listed again every round (every 2 seconds). var due = (await repository.GetConfiguredInstancesAsync(cancellationToken).ConfigureAwait(false)) .Where(i => !i.Paused && i.CredentialError is null && (i.LastDiscoveryAt is null || now.UtcDateTime - i.LastDiscoveryAt.Value >= DiscoveryInterval)) .Where(i => !_routineAttempts.TryGetValue(i.ConfiguredInstanceId, out var attempted) || now - attempted >= RoutineRetry) .OrderBy(i => i.LastDiscoveryAt ?? DateTime.MinValue) .FirstOrDefault(); if (due is null) { return; } _routineAttempts[due.ConfiguredInstanceId] = now; using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); timeout.CancelAfter(RequestTimeout); try { var result = await DiscoverAsync(due.ConfiguredInstanceId, timeout.Token).ConfigureAwait(false); if (result.New.Count > 0) { LogNewDatabases(logger, due.Name, result.New, result.NewMonitored); } } catch (Exception ex) when (ex is DiscoveryFailedException or CredentialException or KeyNotFoundException || ex is OperationCanceledException && !cancellationToken.IsCancellationRequested) { // Recorded on the instance (or the password is reported by the monitor); the next round tries the next instance. await repository.RecordDiscoveryErrorAsync(due.ConfiguredInstanceId, ex.Message, now, cancellationToken).ConfigureAwait(false); } } /// What to do when the agent could not connect (Danish). public static string Hint(Exception exception, MonitoredInstance instance) { ArgumentNullException.ThrowIfNull(exception); ArgumentNullException.ThrowIfNull(instance); var number = exception.GetType().GetProperty("Number")?.GetValue(exception) as int?; var account = AgentCredentialKey.CurrentAccount(); return number switch { 18456 when instance.Authentication == AuthenticationMode.Windows => $"Agentens konto ({account}, på en anden server typisk computerkontoen DOMÆNE\\{Environment.MachineName}$) kan ikke logge ind. Kør GRANT-scriptet nedenfor som sysadmin; det opretter login'et.", 18456 => "SQL-login'et eller adgangskoden blev afvist. Tjek dem, og at instansen tillader SQL Server-godkendelse.", 4060 => "Login'et findes, men kan ikke åbne master. Kør GRANT-scriptet nedenfor.", 2 or 53 or -1 or 258 => "Instansen kunne ikke nås fra agentens server. Tjek servernavnet, at instansen kører, og firewallen (TCP 1433, eller UDP 1434 og instansens port for en navngiven instans).", -2146893019 => "SQL Server bruger et certifikat, agenten ikke har tillid til. Installér et betroet certifikat, eller slå \"Stol på servercertifikatet\" til.", _ when exception.InnerException is System.Security.Authentication.AuthenticationException => "TLS-forbindelsen fejlede. Gamle instanser kan kun TLS 1.0: vælg kryptering \"Valgfri\", eller installér et betroet certifikat.", _ => "Se fejlteksten. Forbindelsen blev forsøgt fra agentens server som agentens konto.", }; } [LoggerMessage(Level = LogLevel.Information, Message = "Request {RequestId} ({Kind}): {Outcome}")] private static partial void LogHandled(ILogger logger, long requestId, string kind, string outcome); [LoggerMessage(Level = LogLevel.Information, Message = "{Instance}: new databases {Databases} (monitored at once: {Monitored})")] private static partial void LogNewDatabases(ILogger logger, string instance, IReadOnlyList databases, bool monitored); [LoggerMessage(Level = LogLevel.Error, Message = "Handling configuration requests failed; retrying.")] private static partial void LogRoundFailed(ILogger logger, Exception exception); } /// A discovery that could not reach the instance; the message says why and what to do (Danish). public sealed class DiscoveryFailedException : Exception { public DiscoveryFailedException() { } public DiscoveryFailedException(string message) : base(message) { } public DiscoveryFailedException(string message, Exception innerException) : base(message, innerException) { } }