using System.Net; using System.Net.Http.Headers; using System.Text; using Microsoft.Extensions.Options; using SQLInsight.Licensing; using SQLInsight.Repository; namespace SQLInsight.Agent.Hub; /// /// Delivers queued submissions to SQLInsight Hub with retry and exponential backoff. Rejections the hub will keep /// making (invalid license, bad request) are given up rather than retried forever. /// public sealed partial class HubSyncService( IOptions options, InsightRepository repository, IHttpClientFactory httpClientFactory, TimeProvider time, ILogger logger) : BackgroundService { public const string AuthorizationScheme = "SQLInsight-License"; public static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(30); public static readonly TimeSpan MaxBackoff = TimeSpan.FromHours(1); protected override async Task ExecuteAsync(CancellationToken stoppingToken) { using var timer = new PeriodicTimer(PollInterval, time); do { try { await RunOnceAsync(stoppingToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { LogRoundFailed(logger, ex); } } while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); } /// One delivery round. Returns how many submissions were sent. public async Task RunOnceAsync(CancellationToken cancellationToken) { var settings = options.Value; var status = CheckLicense(settings); await repository.SaveHubStatusAsync(new HubStatusRow { Enabled = settings.Enabled, HubUrl = settings.Url?.ToString(), LicenseState = status.State.ToString(), LicenseMessage = status.Message, Customer = status.License?.Customer, LicensePlan = status.License?.Plan.ToString(), LicenseExpires = status.License?.ExpiresAt.UtcDateTime, SendSensitive = settings.SendSensitiveEvidence, UpdatedAt = time.GetUtcNow().UtcDateTime, }, cancellationToken).ConfigureAwait(false); if (!settings.Enabled || settings.Url is null || !status.AllowsFindingsSync) { return 0; } // The license is the credential: never send it in clear text. if (settings.Url.Scheme != Uri.UriSchemeHttps && !settings.Url.IsLoopback) { await repository.SetHubErrorAsync($"Hub-adressen {settings.Url} skal bruge https. Intet er sendt.", time.GetUtcNow(), cancellationToken).ConfigureAwait(false); return 0; } var sent = 0; using var http = httpClientFactory.CreateClient("hub"); foreach (var item in await repository.GetDueSubmissionsAsync(time.GetUtcNow(), cancellationToken: cancellationToken).ConfigureAwait(false)) { var now = time.GetUtcNow(); try { using var request = new HttpRequestMessage(HttpMethod.Post, new Uri(settings.Url, "api/v1/findings")) { Content = new StringContent(item.Payload, Encoding.UTF8, "application/json"), }; request.Headers.Authorization = new AuthenticationHeaderValue(AuthorizationScheme, settings.License); using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false); if (response.IsSuccessStatusCode) { await repository.MarkSubmissionSentAsync(item.OutboundId, now, cancellationToken).ConfigureAwait(false); sent++; continue; } var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); var error = $"{(int)response.StatusCode} {response.ReasonPhrase}: {Truncate(body)}"; var permanent = response.StatusCode is HttpStatusCode.BadRequest or HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden or HttpStatusCode.UnprocessableEntity; await repository.MarkSubmissionFailedAsync(item.OutboundId, error, now, permanent ? null : now + Backoff(item.Attempts), cancellationToken).ConfigureAwait(false); LogRejected(logger, item.InstanceName, error); } catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException && !cancellationToken.IsCancellationRequested) { await repository.MarkSubmissionFailedAsync(item.OutboundId, ex.Message, now, now + Backoff(item.Attempts), cancellationToken).ConfigureAwait(false); LogUnreachable(logger, ex, item.InstanceName); } } if (_insightsFetchedAt is null || time.GetUtcNow() - _insightsFetchedAt >= InsightsInterval) { await FetchInsightsAsync(http, settings, cancellationToken).ConfigureAwait(false); } if (status.License?.IncludesPrioritySupport == true) { await SyncSupportCasesAsync(http, settings, cancellationToken).ConfigureAwait(false); } return sent; } /// How often the status of the customer's support cases is fetched when nothing was sent. public static readonly TimeSpan CasesInterval = TimeSpan.FromMinutes(5); private DateTimeOffset? _casesFetchedAt; /// /// Sends support requests filed in SQLInsight Web and stores the case number the hub gives them; then fetches the /// status of all the customer's cases. Rejections the hub will keep making are given up, like submissions. /// public async Task SyncSupportCasesAsync(HttpClient http, HubOptions settings, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(http); ArgumentNullException.ThrowIfNull(settings); var created = 0; foreach (var item in await repository.GetDueSupportRequestsAsync(time.GetUtcNow(), cancellationToken: cancellationToken).ConfigureAwait(false)) { var now = time.GetUtcNow(); try { var body = new Contracts.SupportCaseRequest(item.ClientRequestId!.Value, item.Title, item.Description, item.ContactName ?? string.Empty, item.ContactEmail ?? string.Empty, item.ContactPhone, item.RequestedBy, item.InstanceName, item.RuleId, item.Subject); using var request = new HttpRequestMessage(HttpMethod.Post, new Uri(settings.Url!, "api/v1/cases")) { Content = new StringContent(System.Text.Json.JsonSerializer.Serialize(body, HubOutbox.Json), Encoding.UTF8, "application/json"), }; request.Headers.Authorization = new AuthenticationHeaderValue(AuthorizationScheme, settings.License); using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false); var text = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); if (response.IsSuccessStatusCode && System.Text.Json.JsonSerializer.Deserialize(text, HubOutbox.Json) is { } dto) { await repository.MarkSupportRequestSentAsync(item.CaseId, dto.Reference, dto.IsOpen, dto.ClosedAt?.UtcDateTime, now, cancellationToken).ConfigureAwait(false); created++; continue; } var error = $"{(int)response.StatusCode} {response.ReasonPhrase}: {Truncate(text)}"; var permanent = response.StatusCode is HttpStatusCode.BadRequest or HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden or HttpStatusCode.UnprocessableEntity; await repository.MarkSupportRequestFailedAsync(item.CaseId, error, now, permanent ? null : now + Backoff(item.Attempts), cancellationToken).ConfigureAwait(false); LogCaseRejected(logger, item.Title, error); } catch (Exception ex) when (ex is HttpRequestException or System.Text.Json.JsonException or TaskCanceledException && !cancellationToken.IsCancellationRequested) { await repository.MarkSupportRequestFailedAsync(item.CaseId, ex.Message, now, now + Backoff(item.Attempts), cancellationToken).ConfigureAwait(false); LogCaseUnreachable(logger, ex.Message); } } if (created == 0 && _casesFetchedAt is { } fetched && time.GetUtcNow() - fetched < CasesInterval) { return; } try { using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(settings.Url!, "api/v1/cases")); request.Headers.Authorization = new AuthenticationHeaderValue(AuthorizationScheme, settings.License); using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false); if (!response.IsSuccessStatusCode) { return; } var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); var cases = System.Text.Json.JsonSerializer.Deserialize>(json, HubOutbox.Json) ?? []; await repository.SyncSupportCasesAsync( cases.Select(c => new HubSupportCase(c.Reference, c.Title, c.IsOpen, c.CreatedAt.UtcDateTime, c.ClosedAt?.UtcDateTime, c.Source, c.ClientRequestId)), time.GetUtcNow(), cancellationToken).ConfigureAwait(false); _casesFetchedAt = time.GetUtcNow(); } catch (Exception ex) when (ex is HttpRequestException or System.Text.Json.JsonException or TaskCanceledException && !cancellationToken.IsCancellationRequested) { LogCaseUnreachable(logger, ex.Message); } } /// How often aggregated experience from other customers is fetched. public static readonly TimeSpan InsightsInterval = TimeSpan.FromHours(6); private DateTimeOffset? _insightsFetchedAt; /// Fetches "fixed elsewhere after updating to …" insights. Failing is harmless: the old ones stay. public async Task FetchInsightsAsync(HttpClient http, HubOptions settings, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(http); ArgumentNullException.ThrowIfNull(settings); try { using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(settings.Url!, "api/v1/insights")); request.Headers.Authorization = new AuthenticationHeaderValue(AuthorizationScheme, settings.License); using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false); if (!response.IsSuccessStatusCode) { return false; } var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); if (System.Text.Json.JsonSerializer.Deserialize(json, System.Text.Json.JsonSerializerOptions.Web) is null) { return false; } await repository.SaveHubInsightsAsync(json, time.GetUtcNow(), cancellationToken).ConfigureAwait(false); _insightsFetchedAt = time.GetUtcNow(); return true; } catch (Exception ex) when (ex is HttpRequestException or System.Text.Json.JsonException or TaskCanceledException && !cancellationToken.IsCancellationRequested) { LogInsightsFailed(logger, ex.Message); return false; } } private LicenseStatus CheckLicense(HubOptions settings) => new LicenseVerifier(TrustedKeys.Embedded().With(settings.TrustedKeys), time).Check(settings.License); /// 1, 2, 4 … minutes, at most an hour. public static TimeSpan Backoff(int attempts) => TimeSpan.FromMinutes(Math.Min(MaxBackoff.TotalMinutes, Math.Pow(2, Math.Min(attempts, 10)))); private static string Truncate(string text) => text.Length <= 500 ? text : text[..500]; [LoggerMessage(Level = LogLevel.Warning, Message = "Hub rejected the submission for {Instance}: {Error}")] private static partial void LogRejected(ILogger logger, string instance, string error); [LoggerMessage(Level = LogLevel.Warning, Message = "Hub unreachable for {Instance}; will retry")] private static partial void LogUnreachable(ILogger logger, Exception exception, string instance); [LoggerMessage(Level = LogLevel.Information, Message = "Could not fetch insights from the hub: {Message}")] private static partial void LogInsightsFailed(ILogger logger, string message); [LoggerMessage(Level = LogLevel.Warning, Message = "Hub rejected the support case {Title}: {Error}")] private static partial void LogCaseRejected(ILogger logger, string title, string error); [LoggerMessage(Level = LogLevel.Information, Message = "Support cases could not be synced with the hub: {Message}")] private static partial void LogCaseUnreachable(ILogger logger, string message); [LoggerMessage(Level = LogLevel.Error, Message = "Hub sync round failed")] private static partial void LogRoundFailed(ILogger logger, Exception exception); }