sqlinsight-pro/src/SQLInsight.Agent/Hub/HubSyncService.cs

using System.Net;
using System.Net.Http.Headers;
using System.Text;
using Microsoft.Extensions.Options;
using SQLInsight.Licensing;
using SQLInsight.Repository;

namespace SQLInsight.Agent.Hub;

/// <summary>
/// Delivers queued submissions to SQLInsight Hub with retry and exponential backoff. Rejections the hub will keep
/// making (invalid license, bad request) are given up rather than retried forever.
/// </summary>
public sealed partial class HubSyncService(
    IOptions<HubOptions> options,
    InsightRepository repository,
    IHttpClientFactory httpClientFactory,
    TimeProvider time,
    ILogger<HubSyncService> logger) : BackgroundService
{
    public const string AuthorizationScheme = "SQLInsight-License";
    public static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(30);
    public static readonly TimeSpan MaxBackoff = TimeSpan.FromHours(1);

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        using var timer = new PeriodicTimer(PollInterval, time);
        do
        {
            try
            {
                await RunOnceAsync(stoppingToken).ConfigureAwait(false);
            }
            catch (Exception ex) when (ex is not OperationCanceledException)
            {
                LogRoundFailed(logger, ex);
            }
        }
        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
    }

    /// <summary>One delivery round. Returns how many submissions were sent.</summary>
    public async Task<int> RunOnceAsync(CancellationToken cancellationToken)
    {
        var settings = options.Value;
        var status = CheckLicense(settings);
        await repository.SaveHubStatusAsync(new HubStatusRow
        {
            Enabled = settings.Enabled,
            HubUrl = settings.Url?.ToString(),
            LicenseState = status.State.ToString(),
            LicenseMessage = status.Message,
            Customer = status.License?.Customer,
            LicensePlan = status.License?.Plan.ToString(),
            LicenseExpires = status.License?.ExpiresAt.UtcDateTime,
            SendSensitive = settings.SendSensitiveEvidence,
            UpdatedAt = time.GetUtcNow().UtcDateTime,
        }, cancellationToken).ConfigureAwait(false);

        if (!settings.Enabled || settings.Url is null || !status.AllowsFindingsSync)
        {
            return 0;
        }

        // The license is the credential: never send it in clear text.
        if (settings.Url.Scheme != Uri.UriSchemeHttps && !settings.Url.IsLoopback)
        {
            await repository.SetHubErrorAsync($"Hub-adressen {settings.Url} skal bruge https. Intet er sendt.", time.GetUtcNow(), cancellationToken).ConfigureAwait(false);
            return 0;
        }

        var sent = 0;
        using var http = httpClientFactory.CreateClient("hub");
        foreach (var item in await repository.GetDueSubmissionsAsync(time.GetUtcNow(), cancellationToken: cancellationToken).ConfigureAwait(false))
        {
            var now = time.GetUtcNow();
            try
            {
                using var request = new HttpRequestMessage(HttpMethod.Post, new Uri(settings.Url, "api/v1/findings"))
                {
                    Content = new StringContent(item.Payload, Encoding.UTF8, "application/json"),
                };
                request.Headers.Authorization = new AuthenticationHeaderValue(AuthorizationScheme, settings.License);
                using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false);

                if (response.IsSuccessStatusCode)
                {
                    await repository.MarkSubmissionSentAsync(item.OutboundId, now, cancellationToken).ConfigureAwait(false);
                    sent++;
                    continue;
                }

                var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
                var error = $"{(int)response.StatusCode} {response.ReasonPhrase}: {Truncate(body)}";
                var permanent = response.StatusCode is HttpStatusCode.BadRequest or HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden or HttpStatusCode.UnprocessableEntity;
                await repository.MarkSubmissionFailedAsync(item.OutboundId, error, now, permanent ? null : now + Backoff(item.Attempts), cancellationToken).ConfigureAwait(false);
                LogRejected(logger, item.InstanceName, error);
            }
            catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException && !cancellationToken.IsCancellationRequested)
            {
                await repository.MarkSubmissionFailedAsync(item.OutboundId, ex.Message, now, now + Backoff(item.Attempts), cancellationToken).ConfigureAwait(false);
                LogUnreachable(logger, ex, item.InstanceName);
            }
        }

        if (_insightsFetchedAt is null || time.GetUtcNow() - _insightsFetchedAt >= InsightsInterval)
        {
            await FetchInsightsAsync(http, settings, cancellationToken).ConfigureAwait(false);
        }

        if (status.License?.IncludesPrioritySupport == true)
        {
            await SyncSupportCasesAsync(http, settings, cancellationToken).ConfigureAwait(false);
        }

        return sent;
    }

    /// <summary>How often the status of the customer's support cases is fetched when nothing was sent.</summary>
    public static readonly TimeSpan CasesInterval = TimeSpan.FromMinutes(5);

    private DateTimeOffset? _casesFetchedAt;

    /// <summary>
    /// Sends support requests filed in SQLInsight Web and stores the case number the hub gives them; then fetches the
    /// status of all the customer's cases. Rejections the hub will keep making are given up, like submissions.
    /// </summary>
    public async Task SyncSupportCasesAsync(HttpClient http, HubOptions settings, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(http);
        ArgumentNullException.ThrowIfNull(settings);
        var created = 0;
        foreach (var item in await repository.GetDueSupportRequestsAsync(time.GetUtcNow(), cancellationToken: cancellationToken).ConfigureAwait(false))
        {
            var now = time.GetUtcNow();
            try
            {
                var body = new Contracts.SupportCaseRequest(item.ClientRequestId!.Value, item.Title, item.Description, item.ContactName ?? string.Empty,
                    item.ContactEmail ?? string.Empty, item.ContactPhone, item.RequestedBy, item.InstanceName, item.RuleId, item.Subject);
                using var request = new HttpRequestMessage(HttpMethod.Post, new Uri(settings.Url!, "api/v1/cases"))
                {
                    Content = new StringContent(System.Text.Json.JsonSerializer.Serialize(body, HubOutbox.Json), Encoding.UTF8, "application/json"),
                };
                request.Headers.Authorization = new AuthenticationHeaderValue(AuthorizationScheme, settings.License);
                using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false);
                var text = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
                if (response.IsSuccessStatusCode && System.Text.Json.JsonSerializer.Deserialize<Contracts.SupportCaseDto>(text, HubOutbox.Json) is { } dto)
                {
                    await repository.MarkSupportRequestSentAsync(item.CaseId, dto.Reference, dto.IsOpen, dto.ClosedAt?.UtcDateTime, now, cancellationToken).ConfigureAwait(false);
                    created++;
                    continue;
                }

                var error = $"{(int)response.StatusCode} {response.ReasonPhrase}: {Truncate(text)}";
                var permanent = response.StatusCode is HttpStatusCode.BadRequest or HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden or HttpStatusCode.UnprocessableEntity;
                await repository.MarkSupportRequestFailedAsync(item.CaseId, error, now, permanent ? null : now + Backoff(item.Attempts), cancellationToken).ConfigureAwait(false);
                LogCaseRejected(logger, item.Title, error);
            }
            catch (Exception ex) when (ex is HttpRequestException or System.Text.Json.JsonException or TaskCanceledException && !cancellationToken.IsCancellationRequested)
            {
                await repository.MarkSupportRequestFailedAsync(item.CaseId, ex.Message, now, now + Backoff(item.Attempts), cancellationToken).ConfigureAwait(false);
                LogCaseUnreachable(logger, ex.Message);
            }
        }

        if (created == 0 && _casesFetchedAt is { } fetched && time.GetUtcNow() - fetched < CasesInterval)
        {
            return;
        }

        try
        {
            using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(settings.Url!, "api/v1/cases"));
            request.Headers.Authorization = new AuthenticationHeaderValue(AuthorizationScheme, settings.License);
            using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false);
            if (!response.IsSuccessStatusCode)
            {
                return;
            }

            var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
            var cases = System.Text.Json.JsonSerializer.Deserialize<List<Contracts.SupportCaseDto>>(json, HubOutbox.Json) ?? [];
            await repository.SyncSupportCasesAsync(
                cases.Select(c => new HubSupportCase(c.Reference, c.Title, c.IsOpen, c.CreatedAt.UtcDateTime, c.ClosedAt?.UtcDateTime, c.Source, c.ClientRequestId)),
                time.GetUtcNow(), cancellationToken).ConfigureAwait(false);
            _casesFetchedAt = time.GetUtcNow();
        }
        catch (Exception ex) when (ex is HttpRequestException or System.Text.Json.JsonException or TaskCanceledException && !cancellationToken.IsCancellationRequested)
        {
            LogCaseUnreachable(logger, ex.Message);
        }
    }

    /// <summary>How often aggregated experience from other customers is fetched.</summary>
    public static readonly TimeSpan InsightsInterval = TimeSpan.FromHours(6);

    private DateTimeOffset? _insightsFetchedAt;

    /// <summary>Fetches "fixed elsewhere after updating to …" insights. Failing is harmless: the old ones stay.</summary>
    public async Task<bool> FetchInsightsAsync(HttpClient http, HubOptions settings, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(http);
        ArgumentNullException.ThrowIfNull(settings);
        try
        {
            using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(settings.Url!, "api/v1/insights"));
            request.Headers.Authorization = new AuthenticationHeaderValue(AuthorizationScheme, settings.License);
            using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false);
            if (!response.IsSuccessStatusCode)
            {
                return false;
            }

            var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
            if (System.Text.Json.JsonSerializer.Deserialize<Contracts.HubInsights>(json, System.Text.Json.JsonSerializerOptions.Web) is null)
            {
                return false;
            }

            await repository.SaveHubInsightsAsync(json, time.GetUtcNow(), cancellationToken).ConfigureAwait(false);
            _insightsFetchedAt = time.GetUtcNow();
            return true;
        }
        catch (Exception ex) when (ex is HttpRequestException or System.Text.Json.JsonException or TaskCanceledException && !cancellationToken.IsCancellationRequested)
        {
            LogInsightsFailed(logger, ex.Message);
            return false;
        }
    }

    private LicenseStatus CheckLicense(HubOptions settings) =>
        new LicenseVerifier(TrustedKeys.Embedded().With(settings.TrustedKeys), time).Check(settings.License);

    /// <summary>1, 2, 4 … minutes, at most an hour.</summary>
    public static TimeSpan Backoff(int attempts) =>
        TimeSpan.FromMinutes(Math.Min(MaxBackoff.TotalMinutes, Math.Pow(2, Math.Min(attempts, 10))));

    private static string Truncate(string text) => text.Length <= 500 ? text : text[..500];

    [LoggerMessage(Level = LogLevel.Warning, Message = "Hub rejected the submission for {Instance}: {Error}")]
    private static partial void LogRejected(ILogger logger, string instance, string error);

    [LoggerMessage(Level = LogLevel.Warning, Message = "Hub unreachable for {Instance}; will retry")]
    private static partial void LogUnreachable(ILogger logger, Exception exception, string instance);

    [LoggerMessage(Level = LogLevel.Information, Message = "Could not fetch insights from the hub: {Message}")]
    private static partial void LogInsightsFailed(ILogger logger, string message);

    [LoggerMessage(Level = LogLevel.Warning, Message = "Hub rejected the support case {Title}: {Error}")]
    private static partial void LogCaseRejected(ILogger logger, string title, string error);

    [LoggerMessage(Level = LogLevel.Information, Message = "Support cases could not be synced with the hub: {Message}")]
    private static partial void LogCaseUnreachable(ILogger logger, string message);

    [LoggerMessage(Level = LogLevel.Error, Message = "Hub sync round failed")]
    private static partial void LogRoundFailed(ILogger logger, Exception exception);
}