using System.Net;
using System.Net.Http.Headers;
using System.Text;
using Microsoft.Extensions.Options;
using SQLInsight.Licensing;
using SQLInsight.Repository;
namespace SQLInsight.Agent.Hub;
///
/// Delivers queued submissions to SQLInsight Hub with retry and exponential backoff. Rejections the hub will keep
/// making (invalid license, bad request) are given up rather than retried forever.
///
public sealed partial class HubSyncService(
IOptions options,
InsightRepository repository,
IHttpClientFactory httpClientFactory,
TimeProvider time,
ILogger logger) : BackgroundService
{
public const string AuthorizationScheme = "SQLInsight-License";
public static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(30);
public static readonly TimeSpan MaxBackoff = TimeSpan.FromHours(1);
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
using var timer = new PeriodicTimer(PollInterval, time);
do
{
try
{
await RunOnceAsync(stoppingToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
LogRoundFailed(logger, ex);
}
}
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false));
}
/// One delivery round. Returns how many submissions were sent.
public async Task RunOnceAsync(CancellationToken cancellationToken)
{
var settings = options.Value;
var status = CheckLicense(settings);
await repository.SaveHubStatusAsync(new HubStatusRow
{
Enabled = settings.Enabled,
HubUrl = settings.Url?.ToString(),
LicenseState = status.State.ToString(),
LicenseMessage = status.Message,
Customer = status.License?.Customer,
LicensePlan = status.License?.Plan.ToString(),
LicenseExpires = status.License?.ExpiresAt.UtcDateTime,
SendSensitive = settings.SendSensitiveEvidence,
UpdatedAt = time.GetUtcNow().UtcDateTime,
}, cancellationToken).ConfigureAwait(false);
if (!settings.Enabled || settings.Url is null || !status.AllowsFindingsSync)
{
return 0;
}
// The license is the credential: never send it in clear text.
if (settings.Url.Scheme != Uri.UriSchemeHttps && !settings.Url.IsLoopback)
{
await repository.SetHubErrorAsync($"Hub-adressen {settings.Url} skal bruge https. Intet er sendt.", time.GetUtcNow(), cancellationToken).ConfigureAwait(false);
return 0;
}
var sent = 0;
using var http = httpClientFactory.CreateClient("hub");
foreach (var item in await repository.GetDueSubmissionsAsync(time.GetUtcNow(), cancellationToken: cancellationToken).ConfigureAwait(false))
{
var now = time.GetUtcNow();
try
{
using var request = new HttpRequestMessage(HttpMethod.Post, new Uri(settings.Url, "api/v1/findings"))
{
Content = new StringContent(item.Payload, Encoding.UTF8, "application/json"),
};
request.Headers.Authorization = new AuthenticationHeaderValue(AuthorizationScheme, settings.License);
using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
await repository.MarkSubmissionSentAsync(item.OutboundId, now, cancellationToken).ConfigureAwait(false);
sent++;
continue;
}
var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var error = $"{(int)response.StatusCode} {response.ReasonPhrase}: {Truncate(body)}";
var permanent = response.StatusCode is HttpStatusCode.BadRequest or HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden or HttpStatusCode.UnprocessableEntity;
await repository.MarkSubmissionFailedAsync(item.OutboundId, error, now, permanent ? null : now + Backoff(item.Attempts), cancellationToken).ConfigureAwait(false);
LogRejected(logger, item.InstanceName, error);
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException && !cancellationToken.IsCancellationRequested)
{
await repository.MarkSubmissionFailedAsync(item.OutboundId, ex.Message, now, now + Backoff(item.Attempts), cancellationToken).ConfigureAwait(false);
LogUnreachable(logger, ex, item.InstanceName);
}
}
if (_insightsFetchedAt is null || time.GetUtcNow() - _insightsFetchedAt >= InsightsInterval)
{
await FetchInsightsAsync(http, settings, cancellationToken).ConfigureAwait(false);
}
if (status.License?.IncludesPrioritySupport == true)
{
await SyncSupportCasesAsync(http, settings, cancellationToken).ConfigureAwait(false);
}
return sent;
}
/// How often the status of the customer's support cases is fetched when nothing was sent.
public static readonly TimeSpan CasesInterval = TimeSpan.FromMinutes(5);
private DateTimeOffset? _casesFetchedAt;
///
/// Sends support requests filed in SQLInsight Web and stores the case number the hub gives them; then fetches the
/// status of all the customer's cases. Rejections the hub will keep making are given up, like submissions.
///
public async Task SyncSupportCasesAsync(HttpClient http, HubOptions settings, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(http);
ArgumentNullException.ThrowIfNull(settings);
var created = 0;
foreach (var item in await repository.GetDueSupportRequestsAsync(time.GetUtcNow(), cancellationToken: cancellationToken).ConfigureAwait(false))
{
var now = time.GetUtcNow();
try
{
var body = new Contracts.SupportCaseRequest(item.ClientRequestId!.Value, item.Title, item.Description, item.ContactName ?? string.Empty,
item.ContactEmail ?? string.Empty, item.ContactPhone, item.RequestedBy, item.InstanceName, item.RuleId, item.Subject);
using var request = new HttpRequestMessage(HttpMethod.Post, new Uri(settings.Url!, "api/v1/cases"))
{
Content = new StringContent(System.Text.Json.JsonSerializer.Serialize(body, HubOutbox.Json), Encoding.UTF8, "application/json"),
};
request.Headers.Authorization = new AuthenticationHeaderValue(AuthorizationScheme, settings.License);
using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false);
var text = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode && System.Text.Json.JsonSerializer.Deserialize(text, HubOutbox.Json) is { } dto)
{
await repository.MarkSupportRequestSentAsync(item.CaseId, dto.Reference, dto.IsOpen, dto.ClosedAt?.UtcDateTime, now, cancellationToken).ConfigureAwait(false);
created++;
continue;
}
var error = $"{(int)response.StatusCode} {response.ReasonPhrase}: {Truncate(text)}";
var permanent = response.StatusCode is HttpStatusCode.BadRequest or HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden or HttpStatusCode.UnprocessableEntity;
await repository.MarkSupportRequestFailedAsync(item.CaseId, error, now, permanent ? null : now + Backoff(item.Attempts), cancellationToken).ConfigureAwait(false);
LogCaseRejected(logger, item.Title, error);
}
catch (Exception ex) when (ex is HttpRequestException or System.Text.Json.JsonException or TaskCanceledException && !cancellationToken.IsCancellationRequested)
{
await repository.MarkSupportRequestFailedAsync(item.CaseId, ex.Message, now, now + Backoff(item.Attempts), cancellationToken).ConfigureAwait(false);
LogCaseUnreachable(logger, ex.Message);
}
}
if (created == 0 && _casesFetchedAt is { } fetched && time.GetUtcNow() - fetched < CasesInterval)
{
return;
}
try
{
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(settings.Url!, "api/v1/cases"));
request.Headers.Authorization = new AuthenticationHeaderValue(AuthorizationScheme, settings.License);
using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
return;
}
var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var cases = System.Text.Json.JsonSerializer.Deserialize>(json, HubOutbox.Json) ?? [];
await repository.SyncSupportCasesAsync(
cases.Select(c => new HubSupportCase(c.Reference, c.Title, c.IsOpen, c.CreatedAt.UtcDateTime, c.ClosedAt?.UtcDateTime, c.Source, c.ClientRequestId)),
time.GetUtcNow(), cancellationToken).ConfigureAwait(false);
_casesFetchedAt = time.GetUtcNow();
}
catch (Exception ex) when (ex is HttpRequestException or System.Text.Json.JsonException or TaskCanceledException && !cancellationToken.IsCancellationRequested)
{
LogCaseUnreachable(logger, ex.Message);
}
}
/// How often aggregated experience from other customers is fetched.
public static readonly TimeSpan InsightsInterval = TimeSpan.FromHours(6);
private DateTimeOffset? _insightsFetchedAt;
/// Fetches "fixed elsewhere after updating to …" insights. Failing is harmless: the old ones stay.
public async Task FetchInsightsAsync(HttpClient http, HubOptions settings, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(http);
ArgumentNullException.ThrowIfNull(settings);
try
{
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(settings.Url!, "api/v1/insights"));
request.Headers.Authorization = new AuthenticationHeaderValue(AuthorizationScheme, settings.License);
using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
return false;
}
var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
if (System.Text.Json.JsonSerializer.Deserialize(json, System.Text.Json.JsonSerializerOptions.Web) is null)
{
return false;
}
await repository.SaveHubInsightsAsync(json, time.GetUtcNow(), cancellationToken).ConfigureAwait(false);
_insightsFetchedAt = time.GetUtcNow();
return true;
}
catch (Exception ex) when (ex is HttpRequestException or System.Text.Json.JsonException or TaskCanceledException && !cancellationToken.IsCancellationRequested)
{
LogInsightsFailed(logger, ex.Message);
return false;
}
}
private LicenseStatus CheckLicense(HubOptions settings) =>
new LicenseVerifier(TrustedKeys.Embedded().With(settings.TrustedKeys), time).Check(settings.License);
/// 1, 2, 4 … minutes, at most an hour.
public static TimeSpan Backoff(int attempts) =>
TimeSpan.FromMinutes(Math.Min(MaxBackoff.TotalMinutes, Math.Pow(2, Math.Min(attempts, 10))));
private static string Truncate(string text) => text.Length <= 500 ? text : text[..500];
[LoggerMessage(Level = LogLevel.Warning, Message = "Hub rejected the submission for {Instance}: {Error}")]
private static partial void LogRejected(ILogger logger, string instance, string error);
[LoggerMessage(Level = LogLevel.Warning, Message = "Hub unreachable for {Instance}; will retry")]
private static partial void LogUnreachable(ILogger logger, Exception exception, string instance);
[LoggerMessage(Level = LogLevel.Information, Message = "Could not fetch insights from the hub: {Message}")]
private static partial void LogInsightsFailed(ILogger logger, string message);
[LoggerMessage(Level = LogLevel.Warning, Message = "Hub rejected the support case {Title}: {Error}")]
private static partial void LogCaseRejected(ILogger logger, string title, string error);
[LoggerMessage(Level = LogLevel.Information, Message = "Support cases could not be synced with the hub: {Message}")]
private static partial void LogCaseUnreachable(ILogger logger, string message);
[LoggerMessage(Level = LogLevel.Error, Message = "Hub sync round failed")]
private static partial void LogRoundFailed(ILogger logger, Exception exception);
}