From 4269f6f9e889d01825d0e9ceb057fb2923af3176 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 10:33:37 +0000 Subject: [PATCH] Fase 4: experience across customers The hub records when an instance changes build and works out which findings went away within three days of an update and which were still there a week later (version rules excluded). Staff see every effect in the portal; effects seen at three or more customers, resolved more often than not, are served to licensed agents as anonymous counts (rule, major version, build, CU, customers, instances). The agent fetches them every six hours and the web UI shows them on matching findings: 'at 3 customers this went away after updating to CU21'. Data processing documentation covers the aggregation. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01E3EExwndy8Z22EK2Qpt6BK --- README.md | 4 +- docs/databehandling.md | 13 +- docs/hub.md | 10 + src/SQLInsight.Agent/Hub/HubSyncService.cs | 45 ++ src/SQLInsight.Contracts/HubInsights.cs | 37 ++ src/SQLInsight.Hub/Api/FindingsEndpoint.cs | 43 +- src/SQLInsight.Hub/Data/Entities.cs | 12 + src/SQLInsight.Hub/Data/HubDbContext.cs | 7 + .../20260929102409_VersionChanges.Designer.cs | 540 ++++++++++++++++++ .../20260929102409_VersionChanges.cs | 51 ++ .../Migrations/HubDbContextModelSnapshot.cs | 52 ++ src/SQLInsight.Hub/HubOptions.cs | 3 + .../Insights/InsightCalculator.cs | 85 +++ .../Insights/InsightsEndpoint.cs | 43 ++ .../Maintenance/HubMaintenance.cs | 2 + .../Portal/Pages/Insights.razor | 48 ++ .../Portal/Shared/PortalLayout.razor | 1 + src/SQLInsight.Hub/Program.cs | 3 + .../InsightRepository.Hub.cs | 17 + .../Migrations/0004_hub_insights.sql | 7 + .../Components/Pages/FindingDetail.razor | 31 +- src/SQLInsight.Web/SQLInsight.Web.csproj | 1 + src/SQLInsight.Web/wwwroot/app.css | 1 + .../HubEndToEndTests.cs | 42 ++ .../HubInsightTests.cs | 123 ++++ 25 files changed, 1207 insertions(+), 14 deletions(-) create mode 100644 src/SQLInsight.Contracts/HubInsights.cs create mode 100644 src/SQLInsight.Hub/Data/Migrations/20260929102409_VersionChanges.Designer.cs create mode 100644 src/SQLInsight.Hub/Data/Migrations/20260929102409_VersionChanges.cs create mode 100644 src/SQLInsight.Hub/Insights/InsightCalculator.cs create mode 100644 src/SQLInsight.Hub/Insights/InsightsEndpoint.cs create mode 100644 src/SQLInsight.Hub/Portal/Pages/Insights.razor create mode 100644 src/SQLInsight.Repository/Migrations/0004_hub_insights.sql create mode 100644 tests/SQLInsight.IntegrationTests/HubInsightTests.cs diff --git a/README.md b/README.md index 01c3e2d..e23636c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ SQLInsight Pro er et overvågningsværktøj til Microsoft SQL Server, skrevet i C# på .NET. Det kører hos kunden og forlader ikke netværket, medmindre kunden selv vælger det. Værktøjet nøjes ikke med at vise tal. Det forklarer, hvad der er galt, hvorfor det er et problem, og hvordan det løses. Rådene har samme niveau som dem, man får fra en erfaren DBA. -> **Status:** Fase 1–3 er implementeret: agent, 61 regler (herunder Query Store, plananalyse, indexes, Always On og anomalier), notifikationer, lokalt repository og web-UI. Dertil kommer licenser, signerede opdateringer og SQLInsight Hub med portal til betalende kunder. Se [installation](docs/installation.md), [opdateringer](docs/opdatering.md), [databehandling](docs/databehandling.md), [Roadmap](#roadmap) og [CONTRIBUTING.md](CONTRIBUTING.md). +> **Status:** Fase 1–4 er implementeret: agent, 61 indbyggede regler plus community-regler, notifikationer, lokalt repository og web-UI, licenser, signerede opdateringer, SQLInsight Hub med portal, erfaring på tværs af kunder, PDF-rapporter og integration med Prometheus, Grafana og SCOM. Se [installation](docs/installation.md), [opdateringer](docs/opdatering.md), [integration](docs/integration.md), [rapporter](docs/rapporter.md), [egne regler](docs/regler.md), [databehandling](docs/databehandling.md), [Roadmap](#roadmap) og [CONTRIBUTING.md](CONTRIBUTING.md). --- @@ -203,7 +203,7 @@ Hvert **finding** indeholder: - [ ] Log over medarbejderes adgang og tofaktorlogin i portalen ### Fase 4: Videre -- [ ] Anbefalinger på tværs af kunder: "denne CU har løst problemet hos andre" +- [x] Anbefalinger på tværs af kunder: Hubben ser, hvilke fund der forsvinder efter en opdatering, og agenterne viser "hos 4 kunder forsvandt dette efter CU21" på fundet. Deles kun samlet og anonymt fra mindst 3 kunder. Se [docs/hub.md](docs/hub.md#erfaring-på-tværs-af-kunder) - [x] Rapporter (PDF) til ledelse og compliance: nøgletal, sammenfatning i almindeligt sprog, kontroller, instanser og alvorlige fund. Kan hentes i webbrugerfladen og sendes månedligt på mail. Se [docs/rapporter.md](docs/rapporter.md) - [x] Mulighed for, at community selv bidrager med regler: regler som SQL i JSON (community via pull request og signeret regelpakke, lokale regler hos kunden), med validering og `rules test` mod en instans. Se [docs/regler.md](docs/regler.md) - [x] API til integration med eksisterende monitorering: JSON-API og Prometheus-endpoint med API-tokens, Grafana-dashboard, og Windows' hændelseslog med faste event-id'er til SCOM. Se [docs/integration.md](docs/integration.md) diff --git a/docs/databehandling.md b/docs/databehandling.md index 81b7d2b..83a052a 100644 --- a/docs/databehandling.md +++ b/docs/databehandling.md @@ -39,15 +39,26 @@ Data i hubben bruges kun til: - at gøre SEOSofts support opmærksom på kritiske fund hos kunden - at se, hvilke problemer der går igen på tværs af kunder, så regler, vejledninger og produkt kan forbedres (aggregeret) +- at give alle kunder **samlet, anonym erfaring** tilbage: "hos 4 kunder forsvandt dette fund efter opdatering til CU21" (se nedenfor) + Data sælges ikke og videregives ikke. +### Erfaring på tværs af kunder + +Hubben registrerer, når en instans skifter build (fra og til version, CU-niveau og tidspunkt). Den ser derefter, hvilke fund der forsvandt kort efter, og hvilke der blev ved. Til agenterne deles kun: + +- regel-id, SQL Server-hovedversion, build og CU-niveau +- antal kunder og antal instanser, hvor fundet forsvandt eller blev ved + +Der deles aldrig kundenavne, instans- eller servernavne, databasenavne, målinger eller fundets tekster. En sammenhæng deles kun, når fundet er forsvundet hos mindst 3 forskellige kunder (`SQLInsight:Hub:Insights:MinCustomers`). Oplysningerne kan ikke føres tilbage til en bestemt kunde og er ikke personoplysninger. Det skal alligevel fremgå af databehandleraftalen, at kundens data indgår i denne statistik. + ## Opbevaring | Data | Slettes | |---|---| | Aktive fund | Når de er løst, og derefter efter 365 dage (medmindre de hører til en åben supportsag) | | Log over modtagelser (tidspunkt, antal fund; ikke indholdet) | 90 dage | -| Instanser, der ikke har rapporteret | 180 dage, med deres fund | +| Instanser, der ikke har rapporteret | 180 dage, med deres fund og versionshistorik | | Alt om en kunde | Når aftalen ophører, eller kunden beder om det (`delete-customer`) | Fristerne kan ændres i hubbens konfiguration (se [hub.md](hub.md)). I kundens eget repository styres opbevaringen af `SQLInsight:Retention`. diff --git a/docs/hub.md b/docs/hub.md index 9b2d969..48443c6 100644 --- a/docs/hub.md +++ b/docs/hub.md @@ -51,6 +51,16 @@ dotnet publish src/SQLInsight.Hub -c Release -o /opt/sqlinsight/hub Hubben kan køre som Windows-tjeneste eller systemd-enhed (`Type=notify`) på samme måde som agenten. Databaseskemaet oprettes og opgraderes ved start. +## Erfaring på tværs af kunder + +Hubben registrerer, når en instans skifter build, og beregner, hvilke fund der forsvinder inden for 3 dage efter en opdatering, og hvilke der stadig findes efter 7 dage. Under **Opgraderinger** i portalen ses alle sammenhænge, også dem med få kunder. Sammenhænge med mindst 3 kunder, hvor fundet oftest forsvandt, hentes af agenterne (`GET /api/v1/insights`, med licens) og vises på fundet i kundens webbrugerflade. + +```json +"Insights": { "MinCustomers": 3, "ResolvedWithin": "3.00:00:00", "PersistedAfter": "7.00:00:00" } +``` + +Regler om selve versionen (SQLI-VER-*) er udeladt, fordi de altid forsvinder ved en opdatering. + ## Licenser ```bash diff --git a/src/SQLInsight.Agent/Hub/HubSyncService.cs b/src/SQLInsight.Agent/Hub/HubSyncService.cs index ad31324..73f333b 100644 --- a/src/SQLInsight.Agent/Hub/HubSyncService.cs +++ b/src/SQLInsight.Agent/Hub/HubSyncService.cs @@ -103,9 +103,51 @@ public sealed partial class HubSyncService( } } + if (_insightsFetchedAt is null || time.GetUtcNow() - _insightsFetchedAt >= InsightsInterval) + { + await FetchInsightsAsync(http, settings, cancellationToken).ConfigureAwait(false); + } + return sent; } + /// How often aggregated experience from other customers is fetched. + public static readonly TimeSpan InsightsInterval = TimeSpan.FromHours(6); + + private DateTimeOffset? _insightsFetchedAt; + + /// Fetches "fixed elsewhere after updating to …" insights. Failing is harmless: the old ones stay. + public async Task FetchInsightsAsync(HttpClient http, HubOptions settings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(http); + ArgumentNullException.ThrowIfNull(settings); + try + { + using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(settings.Url!, "api/v1/insights")); + request.Headers.Authorization = new AuthenticationHeaderValue(AuthorizationScheme, settings.License); + using var response = await http.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + return false; + } + + var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + if (System.Text.Json.JsonSerializer.Deserialize(json, System.Text.Json.JsonSerializerOptions.Web) is null) + { + return false; + } + + await repository.SaveHubInsightsAsync(json, time.GetUtcNow(), cancellationToken).ConfigureAwait(false); + _insightsFetchedAt = time.GetUtcNow(); + return true; + } + catch (Exception ex) when (ex is HttpRequestException or System.Text.Json.JsonException or TaskCanceledException && !cancellationToken.IsCancellationRequested) + { + LogInsightsFailed(logger, ex.Message); + return false; + } + } + private LicenseStatus CheckLicense(HubOptions settings) => new LicenseVerifier(TrustedKeys.Embedded().With(settings.TrustedKeys), time).Check(settings.License); @@ -121,6 +163,9 @@ public sealed partial class HubSyncService( [LoggerMessage(Level = LogLevel.Warning, Message = "Hub unreachable for {Instance}; will retry")] private static partial void LogUnreachable(ILogger logger, Exception exception, string instance); + [LoggerMessage(Level = LogLevel.Information, Message = "Could not fetch insights from the hub: {Message}")] + private static partial void LogInsightsFailed(ILogger logger, string message); + [LoggerMessage(Level = LogLevel.Error, Message = "Hub sync round failed")] private static partial void LogRoundFailed(ILogger logger, Exception exception); } diff --git a/src/SQLInsight.Contracts/HubInsights.cs b/src/SQLInsight.Contracts/HubInsights.cs new file mode 100644 index 0000000..36c52b6 --- /dev/null +++ b/src/SQLInsight.Contracts/HubInsights.cs @@ -0,0 +1,37 @@ +namespace SQLInsight.Contracts; + +/// +/// Aggregated experience across SQLInsight Hub customers, sent back to agents. Contains no customer, instance or +/// server names: only counts, and only when at least different customers are behind them. +/// +public sealed record HubInsights(DateTimeOffset GeneratedAt, int MinCustomers, IReadOnlyList Upgrades); + +/// +/// After instances of SQL Server major version were updated to or newer, findings +/// of they had went away within a few days on instances and stayed on +/// . Correlation, not proof of cause. +/// +public sealed record UpgradeInsight(string RuleId, int Major, string ToBuild, string? ToLevel, int Customers, int Resolved, int Persisted) +{ + public double ResolvedShare => Resolved + Persisted == 0 ? 0 : (double)Resolved / (Resolved + Persisted); +} + +public static class HubInsightMatcher +{ + /// + /// Insights relevant to a finding: the same rule, the same major version, and a build newer than the instance's. + /// The one backed by most customers first. + /// + public static IReadOnlyList For(HubInsights? insights, string ruleId, string? productVersion) + { + if (insights is null || productVersion is null || !Version.TryParse(productVersion, out var current)) + { + return []; + } + + return insights.Upgrades + .Where(i => i.RuleId == ruleId && i.Major == current.Major && Version.TryParse(i.ToBuild, out var build) && build > current) + .OrderByDescending(i => i.Customers).ThenBy(i => Version.Parse(i.ToBuild)) + .ToList(); + } +} diff --git a/src/SQLInsight.Hub/Api/FindingsEndpoint.cs b/src/SQLInsight.Hub/Api/FindingsEndpoint.cs index e41d2f3..7d52323 100644 --- a/src/SQLInsight.Hub/Api/FindingsEndpoint.cs +++ b/src/SQLInsight.Hub/Api/FindingsEndpoint.cs @@ -19,35 +19,49 @@ public static class FindingsEndpoint public static void Map(IEndpointRouteBuilder app) => app.MapPost("/api/v1/findings", HandleAsync).DisableAntiforgery().AllowAnonymous(); - internal static async Task HandleAsync( - HttpRequest request, HubDbContext db, IOptions options, TimeProvider time, CancellationToken cancellationToken) + /// The license in the Authorization header, verified and not revoked; otherwise the response to send. + internal static async Task<(License? License, IResult? Rejected)> AuthorizeAsync( + HttpRequest request, HubDbContext db, HubServerOptions options, TimeProvider time, CancellationToken cancellationToken) { - // 1. License: the hub is the authority; the agent's own view is never trusted. var header = request.Headers.Authorization.ToString(); if (!header.StartsWith(AuthorizationScheme + " ", StringComparison.Ordinal)) { - return Problem(401, "Licens mangler", $"Authorization: {AuthorizationScheme} kræves."); + return (null, Problem(401, "Licens mangler", $"Authorization: {AuthorizationScheme} kræves.")); } - var keys = TrustedKeys.Embedded().With(options.Value.TrustedKeys); + var keys = TrustedKeys.Embedded().With(options.TrustedKeys); var status = new LicenseVerifier(keys, time).Check(header[(AuthorizationScheme.Length + 1)..]); if (status.State is LicenseState.Invalid or LicenseState.None) { - return Problem(401, "Ugyldig licens", status.Message); + return (null, Problem(401, "Ugyldig licens", status.Message)); } if (!status.AllowsFindingsSync) { - return Problem(403, "Licensen giver ikke Hub-synkronisering", status.Message); + return (null, Problem(403, "Licensen giver ikke Hub-synkronisering", status.Message)); } - var license = status.License!; - var record = await db.Licenses.FindAsync([license.LicenseId], cancellationToken).ConfigureAwait(false); + var record = await db.Licenses.FindAsync([status.License!.LicenseId], cancellationToken).ConfigureAwait(false); if (record?.RevokedAt is not null) { - return Problem(403, "Licensen er spærret", record.RevokedReason ?? "Kontakt SQLInsight-support."); + return (null, Problem(403, "Licensen er spærret", record.RevokedReason ?? "Kontakt SQLInsight-support.")); } + return (status.License, null); + } + + internal static async Task HandleAsync( + HttpRequest request, HubDbContext db, IOptions options, TimeProvider time, CancellationToken cancellationToken) + { + // 1. License: the hub is the authority; the agent's own view is never trusted. + var (license, rejected) = await AuthorizeAsync(request, db, options.Value, time, cancellationToken).ConfigureAwait(false); + if (license is null) + { + return rejected!; + } + + LicenseRecord? record; + // 2. Body. if (request.ContentLength > options.Value.MaxRequestBytes) { @@ -135,6 +149,15 @@ public static class FindingsEndpoint var active = 0; if (apply) { + if (instance.ProductVersion is { } previous && previous != submission.Instance.ProductVersion) + { + db.VersionChanges.Add(new InstanceVersionChange + { + InstanceId = instance.Id, FromVersion = previous, ToVersion = submission.Instance.ProductVersion, + ToLevel = submission.Instance.ProductUpdateLevel ?? submission.Instance.ProductLevel, ChangedAt = collectedAt, + }); + } + instance.Release = submission.Instance.Release; instance.ProductVersion = submission.Instance.ProductVersion; instance.ProductLevel = submission.Instance.ProductLevel; diff --git a/src/SQLInsight.Hub/Data/Entities.cs b/src/SQLInsight.Hub/Data/Entities.cs index 3c3c2ff..99cc380 100644 --- a/src/SQLInsight.Hub/Data/Entities.cs +++ b/src/SQLInsight.Hub/Data/Entities.cs @@ -118,3 +118,15 @@ public sealed class StaffUser public DateTime CreatedAt { get; set; } public DateTime? LastLoginAt { get; set; } } + +/// An instance moved to another build (a CU, a GDR or an upgrade). The basis for "fixed after upgrading" insights. +public sealed class InstanceVersionChange +{ + public long Id { get; set; } + public int InstanceId { get; set; } + public Instance? Instance { get; set; } + public required string FromVersion { get; set; } + public required string ToVersion { get; set; } + public string? ToLevel { get; set; } + public DateTime ChangedAt { get; set; } +} diff --git a/src/SQLInsight.Hub/Data/HubDbContext.cs b/src/SQLInsight.Hub/Data/HubDbContext.cs index cf72c06..85b67bb 100644 --- a/src/SQLInsight.Hub/Data/HubDbContext.cs +++ b/src/SQLInsight.Hub/Data/HubDbContext.cs @@ -12,6 +12,7 @@ public sealed class HubDbContext(DbContextOptions options) : DbCon public DbSet RuleStates => Set(); public DbSet SupportCases => Set(); public DbSet StaffUsers => Set(); + public DbSet VersionChanges => Set(); protected override void OnModelCreating(ModelBuilder modelBuilder) { @@ -60,6 +61,12 @@ public sealed class HubDbContext(DbContextOptions options) : DbCon e.HasOne(c => c.Customer).WithMany(c => c.SupportCases).HasForeignKey(c => c.CustomerId); }); + modelBuilder.Entity(e => + { + e.HasOne(v => v.Instance).WithMany().HasForeignKey(v => v.InstanceId); + e.HasIndex(v => new { v.InstanceId, v.ChangedAt }); + }); + modelBuilder.Entity(e => { e.HasIndex(u => u.UserName).IsUnique(); diff --git a/src/SQLInsight.Hub/Data/Migrations/20260929102409_VersionChanges.Designer.cs b/src/SQLInsight.Hub/Data/Migrations/20260929102409_VersionChanges.Designer.cs new file mode 100644 index 0000000..ecc8b68 --- /dev/null +++ b/src/SQLInsight.Hub/Data/Migrations/20260929102409_VersionChanges.Designer.cs @@ -0,0 +1,540 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using SQLInsight.Hub.Data; + +#nullable disable + +namespace SQLInsight.Hub.Data.Migrations +{ + [DbContext(typeof(HubDbContext))] + [Migration("20260929102409_VersionChanges")] + partial class VersionChanges + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.12") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("SQLInsight.Hub.Data.Customer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_customers"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("ix_customers_name"); + + b.ToTable("customers", (string)null); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.HubFinding", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("EvidenceJson") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("evidence_json"); + + b.Property("Explanation") + .IsRequired() + .HasColumnType("text") + .HasColumnName("explanation"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("InstanceId") + .HasColumnType("integer") + .HasColumnName("instance_id"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen_at"); + + b.Property("Recommendation") + .IsRequired() + .HasColumnType("text") + .HasColumnName("recommendation"); + + b.Property("ResolvedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("resolved_at"); + + b.Property("RuleId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("rule_id"); + + b.Property("Severity") + .HasColumnType("integer") + .HasColumnName("severity"); + + b.Property("Subject") + .IsRequired() + .HasColumnType("text") + .HasColumnName("subject"); + + b.Property("SupportCaseId") + .HasColumnType("integer") + .HasColumnName("support_case_id"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text") + .HasColumnName("title"); + + b.HasKey("Id") + .HasName("pk_findings"); + + b.HasIndex("SupportCaseId") + .HasDatabaseName("ix_findings_support_case_id"); + + b.HasIndex("Severity", "ResolvedAt") + .HasDatabaseName("ix_findings_severity_resolved_at"); + + b.HasIndex("InstanceId", "RuleId", "Subject") + .IsUnique() + .HasDatabaseName("ix_findings_instance_id_rule_id_subject") + .HasFilter("resolved_at IS NULL"); + + b.ToTable("findings", (string)null); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.Instance", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AgentVersion") + .HasColumnType("text") + .HasColumnName("agent_version"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("Edition") + .HasColumnType("text") + .HasColumnName("edition"); + + b.Property("InstanceKey") + .IsRequired() + .HasColumnType("text") + .HasColumnName("instance_key"); + + b.Property("LastCollectedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_collected_at"); + + b.Property("LastSubmissionAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_submission_at"); + + b.Property("ProductLevel") + .HasColumnType("text") + .HasColumnName("product_level"); + + b.Property("ProductUpdateLevel") + .HasColumnType("text") + .HasColumnName("product_update_level"); + + b.Property("ProductVersion") + .HasColumnType("text") + .HasColumnName("product_version"); + + b.Property("Release") + .HasColumnType("text") + .HasColumnName("release"); + + b.Property("ServerName") + .HasColumnType("text") + .HasColumnName("server_name"); + + b.HasKey("Id") + .HasName("pk_instances"); + + b.HasIndex("CustomerId", "InstanceKey") + .IsUnique() + .HasDatabaseName("ix_instances_customer_id_instance_key"); + + b.ToTable("instances", (string)null); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.InstanceVersionChange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ChangedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("changed_at"); + + b.Property("FromVersion") + .IsRequired() + .HasColumnType("text") + .HasColumnName("from_version"); + + b.Property("InstanceId") + .HasColumnType("integer") + .HasColumnName("instance_id"); + + b.Property("ToLevel") + .HasColumnType("text") + .HasColumnName("to_level"); + + b.Property("ToVersion") + .IsRequired() + .HasColumnType("text") + .HasColumnName("to_version"); + + b.HasKey("Id") + .HasName("pk_version_changes"); + + b.HasIndex("InstanceId", "ChangedAt") + .HasDatabaseName("ix_version_changes_instance_id_changed_at"); + + b.ToTable("version_changes", (string)null); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.LicenseRecord", b => + { + b.Property("LicenseId") + .HasColumnType("text") + .HasColumnName("license_id"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expires_at"); + + b.Property("FirstSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_seen_at"); + + b.Property("IssuedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("issued_at"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen_at"); + + b.Property("MaxInstances") + .HasColumnType("integer") + .HasColumnName("max_instances"); + + b.Property("Plan") + .IsRequired() + .HasColumnType("text") + .HasColumnName("plan"); + + b.Property("RevokedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("revoked_at"); + + b.Property("RevokedReason") + .HasColumnType("text") + .HasColumnName("revoked_reason"); + + b.HasKey("LicenseId") + .HasName("pk_licenses"); + + b.HasIndex("CustomerId") + .HasDatabaseName("ix_licenses_customer_id"); + + b.ToTable("licenses", (string)null); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.RuleState", b => + { + b.Property("InstanceId") + .HasColumnType("integer") + .HasColumnName("instance_id"); + + b.Property("RuleId") + .HasColumnType("text") + .HasColumnName("rule_id"); + + b.Property("Outcome") + .IsRequired() + .HasColumnType("text") + .HasColumnName("outcome"); + + b.Property("Reason") + .HasColumnType("text") + .HasColumnName("reason"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.HasKey("InstanceId", "RuleId") + .HasName("pk_rule_states"); + + b.ToTable("rule_states", (string)null); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.StaffUser", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DisplayName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("display_name"); + + b.Property("IsActive") + .HasColumnType("boolean") + .HasColumnName("is_active"); + + b.Property("LastLoginAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_login_at"); + + b.Property("PasswordHash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("password_hash"); + + b.Property("UserName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("user_name"); + + b.HasKey("Id") + .HasName("pk_staff_users"); + + b.HasIndex("UserName") + .IsUnique() + .HasDatabaseName("ix_staff_users_user_name"); + + b.ToTable("staff_users", (string)null); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.Submission", b => + { + b.Property("SubmissionId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("submission_id"); + + b.Property("Applied") + .HasColumnType("boolean") + .HasColumnName("applied"); + + b.Property("CollectedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("collected_at"); + + b.Property("FindingCount") + .HasColumnType("integer") + .HasColumnName("finding_count"); + + b.Property("InstanceId") + .HasColumnType("integer") + .HasColumnName("instance_id"); + + b.Property("ReceivedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("received_at"); + + b.HasKey("SubmissionId") + .HasName("pk_submissions"); + + b.HasIndex("ReceivedAt") + .HasDatabaseName("ix_submissions_received_at"); + + b.ToTable("submissions", (string)null); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.SupportCase", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ClosedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("closed_at"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CustomerId") + .HasColumnType("integer") + .HasColumnName("customer_id"); + + b.Property("IsOpen") + .HasColumnType("boolean") + .HasColumnName("is_open"); + + b.Property("Note") + .HasColumnType("text") + .HasColumnName("note"); + + b.Property("Reference") + .IsRequired() + .HasColumnType("text") + .HasColumnName("reference"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text") + .HasColumnName("title"); + + b.HasKey("Id") + .HasName("pk_support_cases"); + + b.HasIndex("CustomerId") + .HasDatabaseName("ix_support_cases_customer_id"); + + b.ToTable("support_cases", (string)null); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.HubFinding", b => + { + b.HasOne("SQLInsight.Hub.Data.Instance", "Instance") + .WithMany("Findings") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_findings_instances_instance_id"); + + b.HasOne("SQLInsight.Hub.Data.SupportCase", "SupportCase") + .WithMany("Findings") + .HasForeignKey("SupportCaseId") + .OnDelete(DeleteBehavior.SetNull) + .HasConstraintName("fk_findings_support_cases_support_case_id"); + + b.Navigation("Instance"); + + b.Navigation("SupportCase"); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.Instance", b => + { + b.HasOne("SQLInsight.Hub.Data.Customer", "Customer") + .WithMany("Instances") + .HasForeignKey("CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_instances_customers_customer_id"); + + b.Navigation("Customer"); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.InstanceVersionChange", b => + { + b.HasOne("SQLInsight.Hub.Data.Instance", "Instance") + .WithMany() + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_version_changes_instances_instance_id"); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.LicenseRecord", b => + { + b.HasOne("SQLInsight.Hub.Data.Customer", "Customer") + .WithMany("Licenses") + .HasForeignKey("CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_licenses_customers_customer_id"); + + b.Navigation("Customer"); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.SupportCase", b => + { + b.HasOne("SQLInsight.Hub.Data.Customer", "Customer") + .WithMany("SupportCases") + .HasForeignKey("CustomerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_support_cases_customers_customer_id"); + + b.Navigation("Customer"); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.Customer", b => + { + b.Navigation("Instances"); + + b.Navigation("Licenses"); + + b.Navigation("SupportCases"); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.Instance", b => + { + b.Navigation("Findings"); + }); + + modelBuilder.Entity("SQLInsight.Hub.Data.SupportCase", b => + { + b.Navigation("Findings"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/SQLInsight.Hub/Data/Migrations/20260929102409_VersionChanges.cs b/src/SQLInsight.Hub/Data/Migrations/20260929102409_VersionChanges.cs new file mode 100644 index 0000000..3bec805 --- /dev/null +++ b/src/SQLInsight.Hub/Data/Migrations/20260929102409_VersionChanges.cs @@ -0,0 +1,51 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace SQLInsight.Hub.Data.Migrations +{ + /// + public partial class VersionChanges : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "version_changes", + columns: table => new + { + id = table.Column(type: "bigint", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + instance_id = table.Column(type: "integer", nullable: false), + from_version = table.Column(type: "text", nullable: false), + to_version = table.Column(type: "text", nullable: false), + to_level = table.Column(type: "text", nullable: true), + changed_at = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_version_changes", x => x.id); + table.ForeignKey( + name: "fk_version_changes_instances_instance_id", + column: x => x.instance_id, + principalTable: "instances", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_version_changes_instance_id_changed_at", + table: "version_changes", + columns: new[] { "instance_id", "changed_at" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "version_changes"); + } + } +} diff --git a/src/SQLInsight.Hub/Data/Migrations/HubDbContextModelSnapshot.cs b/src/SQLInsight.Hub/Data/Migrations/HubDbContextModelSnapshot.cs index 0f293c8..263e0dd 100644 --- a/src/SQLInsight.Hub/Data/Migrations/HubDbContextModelSnapshot.cs +++ b/src/SQLInsight.Hub/Data/Migrations/HubDbContextModelSnapshot.cs @@ -194,6 +194,46 @@ namespace SQLInsight.Hub.Data.Migrations b.ToTable("instances", (string)null); }); + modelBuilder.Entity("SQLInsight.Hub.Data.InstanceVersionChange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ChangedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("changed_at"); + + b.Property("FromVersion") + .IsRequired() + .HasColumnType("text") + .HasColumnName("from_version"); + + b.Property("InstanceId") + .HasColumnType("integer") + .HasColumnName("instance_id"); + + b.Property("ToLevel") + .HasColumnType("text") + .HasColumnName("to_level"); + + b.Property("ToVersion") + .IsRequired() + .HasColumnType("text") + .HasColumnName("to_version"); + + b.HasKey("Id") + .HasName("pk_version_changes"); + + b.HasIndex("InstanceId", "ChangedAt") + .HasDatabaseName("ix_version_changes_instance_id_changed_at"); + + b.ToTable("version_changes", (string)null); + }); + modelBuilder.Entity("SQLInsight.Hub.Data.LicenseRecord", b => { b.Property("LicenseId") @@ -437,6 +477,18 @@ namespace SQLInsight.Hub.Data.Migrations b.Navigation("Customer"); }); + modelBuilder.Entity("SQLInsight.Hub.Data.InstanceVersionChange", b => + { + b.HasOne("SQLInsight.Hub.Data.Instance", "Instance") + .WithMany() + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_version_changes_instances_instance_id"); + + b.Navigation("Instance"); + }); + modelBuilder.Entity("SQLInsight.Hub.Data.LicenseRecord", b => { b.HasOne("SQLInsight.Hub.Data.Customer", "Customer") diff --git a/src/SQLInsight.Hub/HubOptions.cs b/src/SQLInsight.Hub/HubOptions.cs index 0abd364..8701374 100644 --- a/src/SQLInsight.Hub/HubOptions.cs +++ b/src/SQLInsight.Hub/HubOptions.cs @@ -35,6 +35,9 @@ public sealed class HubServerOptions public IList KnownProxies { get; } = []; public RetentionOptions Retention { get; set; } = new(); + + /// Upgrade insights shared with agents (aggregated, anonymous). + public Insights.InsightOptions Insights { get; set; } = new(); } /// How long the hub keeps data it no longer needs. Applied once a day. diff --git a/src/SQLInsight.Hub/Insights/InsightCalculator.cs b/src/SQLInsight.Hub/Insights/InsightCalculator.cs new file mode 100644 index 0000000..5089fd2 --- /dev/null +++ b/src/SQLInsight.Hub/Insights/InsightCalculator.cs @@ -0,0 +1,85 @@ +using Microsoft.EntityFrameworkCore; +using SQLInsight.Contracts; +using System.Globalization; +using SQLInsight.Hub.Data; + +namespace SQLInsight.Hub.Insights; + +public sealed class InsightOptions +{ + /// Fewest different customers behind an insight before it is shared with other customers. + public int MinCustomers { get; set; } = 3; + + /// A finding that went away this soon after an update counts as resolved by it. + public TimeSpan ResolvedWithin { get; set; } = TimeSpan.FromDays(3); + + /// A finding still there this long after an update counts as not resolved by it. + public TimeSpan PersistedAfter { get; set; } = TimeSpan.FromDays(7); +} + +/// +/// "This CU made the problem go away elsewhere": for every build change, which of the instance's active findings +/// disappeared shortly after, and which stayed. +/// +public sealed class InsightCalculator(HubDbContext db, InsightOptions options, TimeProvider time) +{ + /// Rules about the version itself are always "fixed" by an update, so they say nothing. + private static readonly string[] Excluded = ["SQLI-VER-"]; + + /// All upgrade effects, also those from fewer customers than the sharing threshold (for staff). + public async Task> UpgradeEffectsAsync(CancellationToken cancellationToken) + { + var now = time.GetUtcNow().UtcDateTime; + var changes = await db.VersionChanges.AsNoTracking() + .Select(v => new { v.InstanceId, v.Instance!.CustomerId, v.ToVersion, v.ToLevel, v.ChangedAt }) + .ToListAsync(cancellationToken).ConfigureAwait(false); + var instanceIds = changes.Select(c => c.InstanceId).Distinct().ToList(); + var findings = await db.Findings.AsNoTracking() + .Where(f => instanceIds.Contains(f.InstanceId)) + .Select(f => new { f.InstanceId, f.RuleId, f.FirstSeenAt, f.ResolvedAt }) + .ToListAsync(cancellationToken).ConfigureAwait(false); + var byInstance = findings.ToLookup(f => f.InstanceId); + + var events = new List<(string RuleId, int Major, string ToBuild, string? ToLevel, int CustomerId, bool Resolved)>(); + foreach (var change in changes) + { + if (!int.TryParse(change.ToVersion.Split('.')[0], NumberStyles.None, CultureInfo.InvariantCulture, out var major)) + { + continue; + } + + var watchedLongEnough = now - change.ChangedAt >= options.PersistedAfter; + foreach (var rule in byInstance[change.InstanceId] + .Where(f => f.FirstSeenAt < change.ChangedAt && (f.ResolvedAt is null || f.ResolvedAt >= change.ChangedAt)) + .Where(f => !Excluded.Any(prefix => f.RuleId.StartsWith(prefix, StringComparison.Ordinal))) + .GroupBy(f => f.RuleId)) + { + // Per instance and rule: resolved when every occurrence active at the update went away soon after. + var resolved = rule.All(f => f.ResolvedAt is { } at && at - change.ChangedAt <= options.ResolvedWithin); + var persisted = rule.Any(f => f.ResolvedAt is null ? watchedLongEnough : f.ResolvedAt - change.ChangedAt >= options.PersistedAfter); + if (resolved || persisted) + { + events.Add((rule.Key, major, change.ToVersion, change.ToLevel, change.CustomerId, resolved)); + } + } + } + + return events + .GroupBy(e => (e.RuleId, e.Major, e.ToBuild)) + .Select(g => new UpgradeInsight( + g.Key.RuleId, g.Key.Major, g.Key.ToBuild, g.Select(e => e.ToLevel).FirstOrDefault(l => l is not null), + g.Where(e => e.Resolved).Select(e => e.CustomerId).Distinct().Count(), + g.Count(e => e.Resolved), g.Count(e => !e.Resolved))) + .OrderBy(i => i.RuleId, StringComparer.Ordinal).ThenBy(i => i.Major).ThenBy(i => i.ToBuild, StringComparer.Ordinal) + .ToList(); + } + + /// What may be shared with customers: enough different customers, and resolved more often than not. + public async Task ForCustomersAsync(CancellationToken cancellationToken) + { + var shared = (await UpgradeEffectsAsync(cancellationToken).ConfigureAwait(false)) + .Where(i => i.Customers >= options.MinCustomers && i.ResolvedShare > 0.5) + .ToList(); + return new HubInsights(time.GetUtcNow(), options.MinCustomers, shared); + } +} diff --git a/src/SQLInsight.Hub/Insights/InsightsEndpoint.cs b/src/SQLInsight.Hub/Insights/InsightsEndpoint.cs new file mode 100644 index 0000000..8cb7332 --- /dev/null +++ b/src/SQLInsight.Hub/Insights/InsightsEndpoint.cs @@ -0,0 +1,43 @@ +using Microsoft.Extensions.Options; +using SQLInsight.Contracts; +using SQLInsight.Hub.Api; +using SQLInsight.Hub.Data; + +namespace SQLInsight.Hub.Insights; + +/// GET /api/v1/insights for licensed agents. Computed at most once an hour. +public static class InsightsEndpoint +{ + public static readonly TimeSpan CacheFor = TimeSpan.FromHours(1); + + private static readonly SemaphoreSlim Gate = new(1, 1); + private static HubInsights? _cached; + + public static void Map(IEndpointRouteBuilder app) => + app.MapGet("/api/v1/insights", async (HttpRequest request, HubDbContext db, IOptions options, InsightCalculator calculator, TimeProvider time, CancellationToken ct) => + { + var (license, rejected) = await FindingsEndpoint.AuthorizeAsync(request, db, options.Value, time, ct).ConfigureAwait(false); + if (license is null) + { + return rejected!; + } + + await Gate.WaitAsync(ct).ConfigureAwait(false); + try + { + if (_cached is null || time.GetUtcNow() - _cached.GeneratedAt > CacheFor) + { + _cached = await calculator.ForCustomersAsync(ct).ConfigureAwait(false); + } + + return Results.Ok(_cached); + } + finally + { + Gate.Release(); + } + }).AllowAnonymous(); + + /// For tests and after a change of settings. + public static void ClearCache() => _cached = null; +} diff --git a/src/SQLInsight.Hub/Maintenance/HubMaintenance.cs b/src/SQLInsight.Hub/Maintenance/HubMaintenance.cs index fd9084e..422c126 100644 --- a/src/SQLInsight.Hub/Maintenance/HubMaintenance.cs +++ b/src/SQLInsight.Hub/Maintenance/HubMaintenance.cs @@ -18,6 +18,7 @@ public sealed class HubMaintenance(HubDbContext db, IOptions o var silentBefore = now - retention.SilentInstances; var silent = db.Instances.Where(i => i.LastSubmissionAt != null && i.LastSubmissionAt < silentBefore).Select(i => i.Id); await db.RuleStates.Where(r => silent.Contains(r.InstanceId)).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); + await db.VersionChanges.Where(v => silent.Contains(v.InstanceId)).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); await db.Submissions.Where(s => silent.Contains(s.InstanceId)).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); await db.Findings.Where(f => silent.Contains(f.InstanceId)).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); var instances = await db.Instances.Where(i => i.LastSubmissionAt != null && i.LastSubmissionAt < silentBefore).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); @@ -49,6 +50,7 @@ public sealed class HubMaintenance(HubDbContext db, IOptions o await using var transaction = await db.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); var instances = db.Instances.Where(i => i.CustomerId == customer.Id).Select(i => i.Id); await db.RuleStates.Where(r => instances.Contains(r.InstanceId)).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); + await db.VersionChanges.Where(v => instances.Contains(v.InstanceId)).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); await db.Submissions.Where(s => instances.Contains(s.InstanceId)).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); await db.Findings.Where(f => instances.Contains(f.InstanceId)).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); await db.Instances.Where(i => i.CustomerId == customer.Id).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); diff --git a/src/SQLInsight.Hub/Portal/Pages/Insights.razor b/src/SQLInsight.Hub/Portal/Pages/Insights.razor new file mode 100644 index 0000000..eec693c --- /dev/null +++ b/src/SQLInsight.Hub/Portal/Pages/Insights.razor @@ -0,0 +1,48 @@ +@page "/portal/indsigt" +@attribute [Authorize] +@using SQLInsight.Contracts +@using SQLInsight.Hub.Insights +@inject InsightCalculator Calculator +@inject InsightOptions Options + +Opgraderinger · SQLInsight Hub +

Hvad forsvinder efter en opdatering?

+

+ For hver instans, der har skiftet build: hvilke fund forsvandt inden for @Options.ResolvedWithin.TotalDays.ToString("0", CultureInfo.InvariantCulture) dage, + og hvilke var der stadig efter @Options.PersistedAfter.TotalDays.ToString("0", CultureInfo.InvariantCulture) dage. Det er sammenhæng, ikke bevis. + Rækker med mindst @Options.MinCustomers kunder, hvor fundet oftest forsvandt, deles anonymt med kundernes agenter. +

+
+ @if (_effects.Count == 0) + { +

Ingen instanser har skiftet build endnu.

+ } + else + { + + + + @foreach (var e in _effects) + { + var shared = e.Customers >= Options.MinCustomers && e.ResolvedShare > 0.5; + + + + + + + + + + + } + +
RegelVersionOpdateret tilKunderForsvandtBlevAndelDeles
@e.RuleId@e.Major@e.ToBuild @(e.ToLevel is { } level ? $"({level})" : "")@e.Customers@e.Resolved@e.Persisted@e.ResolvedShare.ToString("P0", CultureInfo.GetCultureInfo("da-DK"))@(shared ? "Ja" : "Nej")
+ } +
+ +@code { + private IReadOnlyList _effects = []; + + protected override async Task OnInitializedAsync() => _effects = await Calculator.UpgradeEffectsAsync(CancellationToken.None); +} diff --git a/src/SQLInsight.Hub/Portal/Shared/PortalLayout.razor b/src/SQLInsight.Hub/Portal/Shared/PortalLayout.razor index 48b0f36..ac61f8e 100644 --- a/src/SQLInsight.Hub/Portal/Shared/PortalLayout.razor +++ b/src/SQLInsight.Hub/Portal/Shared/PortalLayout.razor @@ -7,6 +7,7 @@ Kunder Overblik Alle fund + Opgraderinger Supportsager Licenser diff --git a/src/SQLInsight.Hub/Program.cs b/src/SQLInsight.Hub/Program.cs index 7a39067..5081ee9 100644 --- a/src/SQLInsight.Hub/Program.cs +++ b/src/SQLInsight.Hub/Program.cs @@ -61,6 +61,8 @@ public static class HubProgram } }); builder.Services.AddScoped(); + builder.Services.AddSingleton(settings.Insights); + builder.Services.AddScoped(); builder.Services.AddHostedService(); builder.Services.AddScoped(); builder.Services.AddScoped(); @@ -124,6 +126,7 @@ public static class HubProgram app.MapGet("/", () => Results.Redirect("/portal")); app.MapGet("/health", () => Results.Ok(new { status = "ok" })); FindingsEndpoint.Map(app); + SQLInsight.Hub.Insights.InsightsEndpoint.Map(app); PortalEndpoints.Map(app); app.MapRazorComponents(); return app; diff --git a/src/SQLInsight.Repository/InsightRepository.Hub.cs b/src/SQLInsight.Repository/InsightRepository.Hub.cs index 41fe2bf..23dcbeb 100644 --- a/src/SQLInsight.Repository/InsightRepository.Hub.cs +++ b/src/SQLInsight.Repository/InsightRepository.Hub.cs @@ -68,6 +68,23 @@ public sealed partial class InsightRepository """, new { outboundId, error, now = Utc(now), next = nextAttemptAt is { } n ? Utc(n) : (DateTime?)null }).ConfigureAwait(false); } + public async Task SaveHubInsightsAsync(string json, DateTimeOffset fetchedAt, CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false); + await connection.ExecuteAsync(""" + MERGE insight.HubInsights AS t USING (SELECT 1 AS Id) AS s ON t.Id = s.Id + WHEN MATCHED THEN UPDATE SET FetchedAt = @at, Json = @json + WHEN NOT MATCHED THEN INSERT (Id, FetchedAt, Json) VALUES (1, @at, @json); + """, new { json, at = Utc(fetchedAt) }).ConfigureAwait(false); + } + + /// The last insights fetched from the hub, as JSON (SQLInsight.Contracts.HubInsights), or null. + public async Task GetHubInsightsJsonAsync(CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false); + return await connection.QuerySingleOrDefaultAsync("SELECT Json FROM insight.HubInsights WHERE Id = 1;").ConfigureAwait(false); + } + public async Task SetHubErrorAsync(string error, DateTimeOffset now, CancellationToken cancellationToken = default) { await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false); diff --git a/src/SQLInsight.Repository/Migrations/0004_hub_insights.sql b/src/SQLInsight.Repository/Migrations/0004_hub_insights.sql new file mode 100644 index 0000000..16af319 --- /dev/null +++ b/src/SQLInsight.Repository/Migrations/0004_hub_insights.sql @@ -0,0 +1,7 @@ +/* Aggregated, anonymous experience from SQLInsight Hub ("fixed elsewhere after updating to …"). One row. */ +CREATE TABLE insight.HubInsights +( + Id tinyint NOT NULL CONSTRAINT PK_HubInsights PRIMARY KEY CONSTRAINT CK_HubInsights_Single CHECK (Id = 1), + FetchedAt datetime2(0) NOT NULL, + Json nvarchar(max) NOT NULL +); diff --git a/src/SQLInsight.Web/Components/Pages/FindingDetail.razor b/src/SQLInsight.Web/Components/Pages/FindingDetail.razor index 13f1de3..9df83c2 100644 --- a/src/SQLInsight.Web/Components/Pages/FindingDetail.razor +++ b/src/SQLInsight.Web/Components/Pages/FindingDetail.razor @@ -54,6 +54,20 @@ } +@if (_elsewhere.Count > 0 && _finding.ResolvedAt is null) +{ + var best = _elsewhere[0]; +
+

Erfaring fra SQLInsight-kunder

+

+ Hos @best.Customers kunder forsvandt dette fund efter opdatering til + @(best.ToLevel is { } level ? $"{level} ({best.ToBuild})" : best.ToBuild) + (@best.Resolved af @(best.Resolved + best.Persisted) instanser). Instansen kører @_version. +

+

Tal fra SQLInsight Hub, samlet anonymt på tværs af mindst @_minCustomers kunder. Det er en sammenhæng, ikke en garanti: læs anbefalingen ovenfor, og test opdateringen først.

+
+} + @if (_finding.References.Count > 0) {
@@ -71,6 +85,21 @@ [Parameter] public long Id { get; set; } private StoredFinding? _finding; + private IReadOnlyList _elsewhere = []; + private string? _version; + private int _minCustomers; - protected override async Task OnInitializedAsync() => _finding = await Repository.GetFindingAsync(Id); + protected override async Task OnInitializedAsync() + { + _finding = await Repository.GetFindingAsync(Id); + if (_finding is null || await Repository.GetHubInsightsJsonAsync() is not { } json) + { + return; + } + + var insights = System.Text.Json.JsonSerializer.Deserialize(json, System.Text.Json.JsonSerializerOptions.Web); + _version = (await Repository.GetInstanceAsync(_finding.InstanceName))?.ProductVersion; + _minCustomers = insights?.MinCustomers ?? 0; + _elsewhere = SQLInsight.Contracts.HubInsightMatcher.For(insights, _finding.RuleId, _version); + } } diff --git a/src/SQLInsight.Web/SQLInsight.Web.csproj b/src/SQLInsight.Web/SQLInsight.Web.csproj index 8d07a7c..67dde4b 100644 --- a/src/SQLInsight.Web/SQLInsight.Web.csproj +++ b/src/SQLInsight.Web/SQLInsight.Web.csproj @@ -6,6 +6,7 @@ + diff --git a/src/SQLInsight.Web/wwwroot/app.css b/src/SQLInsight.Web/wwwroot/app.css index c16a95b..1cb1b95 100644 --- a/src/SQLInsight.Web/wwwroot/app.css +++ b/src/SQLInsight.Web/wwwroot/app.css @@ -160,3 +160,4 @@ details .script pre { max-height: 32rem; } .report-links { display: flex; gap: .6rem; flex-wrap: wrap; } a.button { display: inline-block; background: var(--accent); color: #fff; padding: .5rem 1rem; border-radius: 8px; font-weight: 600; } a.button:hover { filter: brightness(1.08); } +.panel.insight { border-left: 4px solid var(--accent); } diff --git a/tests/SQLInsight.IntegrationTests/HubEndToEndTests.cs b/tests/SQLInsight.IntegrationTests/HubEndToEndTests.cs index f949a39..0509940 100644 --- a/tests/SQLInsight.IntegrationTests/HubEndToEndTests.cs +++ b/tests/SQLInsight.IntegrationTests/HubEndToEndTests.cs @@ -105,4 +105,46 @@ public class HubEndToEndTests(SqlServerFixture sql, HubFixture hub) : IClassFixt Assert.Equal(0, calls); Assert.Contains("https", (await _repository.GetHubStatusAsync())!.LastError, StringComparison.Ordinal); } + + [Fact] + public async Task Experience_from_other_customers_reaches_the_customers_web_ui() + { + for (var i = 0; i < 3; i++) + { + await HubInsightTests.UpgradeAsync(hub, $"Erfaring {i}", ["SQLI-CFG-005"], []); + } + + SQLInsight.Hub.Insights.InsightsEndpoint.ClearCache(); + var options = new HubOptions { Enabled = true, Url = new Uri("http://localhost/"), License = hub.License("Modtager A/S") }; + options.TrustedKeys[HubFixture.KeyId] = hub.PublicKey; + var sync = new HubSyncService(Options.Create(options), _repository, new TestServerClients(() => hub.Factory.CreateClient()), TimeProvider.System, NullLogger.Instance); + Assert.True(await sync.FetchInsightsAsync(hub.Factory.CreateClient(), options, CancellationToken.None)); + + var instanceId = await _repository.UpsertInstanceAsync("prod-sql01", new SQLInsight.Core.Instances.InstanceInfo + { + ServerName = "SQL01", Version = SQLInsight.Core.Instances.SqlServerVersion.Parse(HubInsightTests.OldBuild), ProductLevel = "RTM", ProductUpdateLevel = "CU20", + Edition = "Enterprise Edition", EngineEdition = SQLInsight.Core.Instances.EngineEdition.Enterprise, + }, "SqlClient", false, DateTimeOffset.UtcNow); + await _repository.SaveRuleResultsAsync(instanceId, + [ + new RuleResult + { + RuleId = "SQLI-CFG-005", RuleTitle = "Plan cache", Outcome = RuleOutcome.Findings, + Findings = [new SQLInsight.Core.Findings.Finding { RuleId = "SQLI-CFG-005", Severity = SQLInsight.Core.Findings.Severity.Medium, Title = "Plan cache er fyldt med engangsplaner", Explanation = "e", Recommendation = "r" }], + }, + ], DateTimeOffset.UtcNow); + var findingId = (await _repository.GetActiveFindingsAsync(instanceId)).Single().FindingId; + + await using var web = new Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory().WithWebHostBuilder(b => + { + Microsoft.AspNetCore.Hosting.HostingAbstractionsWebHostBuilderExtensions.UseEnvironment(b, "Development"); + b.UseSetting("SQLInsight:Repository:ConnectionString", sql.DatabaseConnectionString(_database)); + b.UseSetting("SQLInsight:Web:Authentication", "None"); + }); + var html = await web.CreateClient().GetStringAsync($"/fund/{findingId}"); + + Assert.Contains("Erfaring fra SQLInsight-kunder", html, StringComparison.Ordinal); + Assert.Contains("Hos 3 kunder", html, StringComparison.Ordinal); + Assert.Contains("CU21 (15.0.4415.2)", html, StringComparison.Ordinal); + } } diff --git a/tests/SQLInsight.IntegrationTests/HubInsightTests.cs b/tests/SQLInsight.IntegrationTests/HubInsightTests.cs new file mode 100644 index 0000000..cebea44 --- /dev/null +++ b/tests/SQLInsight.IntegrationTests/HubInsightTests.cs @@ -0,0 +1,123 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using SQLInsight.Contracts; +using SQLInsight.Hub.Data; +using SQLInsight.Hub.Insights; + +namespace SQLInsight.IntegrationTests; + +[Trait("Category", "Integration")] +public class HubInsightTests(HubFixture hub) : IClassFixture +{ + public const string OldBuild = "15.0.4385.2"; + public const string NewBuild = "15.0.4415.2"; + + internal static FindingsSubmission Submission(string build, string level, DateTimeOffset at, params string[] rules) => new() + { + SubmissionId = Guid.NewGuid(), + AgentVersion = "1.0.0", + CollectedAt = at, + Instance = new InstanceSnapshot { InstanceKey = "sql01", Release = "SQL Server 2019", ProductVersion = build, ProductUpdateLevel = level }, + Rules = rules.Select(r => new RuleOutcomeDto { RuleId = r, Outcome = "Findings" }).ToList(), + Findings = rules.Select(r => new FindingDto + { + RuleId = r, Subject = "", Severity = "High", Title = r, Explanation = "e", Recommendation = "r", FirstSeenAt = at.AddHours(-1), + }).ToList(), + }; + + internal static async Task SubmitAsync(HttpClient client, string license, FindingsSubmission submission) + { + using var request = new HttpRequestMessage(HttpMethod.Post, "/api/v1/findings") { Content = JsonContent.Create(submission) }; + request.Headers.Authorization = new AuthenticationHeaderValue("SQLInsight-License", license); + (await client.SendAsync(request)).EnsureSuccessStatusCode(); + } + + /// A customer whose instance had the rules' findings on the old build, then updated: went away. + internal static async Task UpgradeAsync(HubFixture hub, string customer, string[] before, string[] after) + { + var client = hub.Factory.CreateClient(); + var license = hub.License(customer); + var t0 = DateTimeOffset.UtcNow.AddMinutes(-30); + await SubmitAsync(client, license, Submission(OldBuild, "CU20", t0, before)); + await SubmitAsync(client, license, Submission(NewBuild, "CU21", t0.AddMinutes(15), after)); + } + + private async Task InsightsAsync(string license) + { + InsightsEndpoint.ClearCache(); + using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v1/insights"); + request.Headers.Authorization = new AuthenticationHeaderValue("SQLInsight-License", license); + var response = await hub.Factory.CreateClient().SendAsync(request); + response.EnsureSuccessStatusCode(); + return (await response.Content.ReadFromJsonAsync())!; + } + + [Fact] + public async Task Findings_that_went_away_after_an_update_at_three_customers_are_shared() + { + var tag = Guid.NewGuid().ToString("N")[..6]; + for (var i = 0; i < 3; i++) + { + await UpgradeAsync(hub, $"Kunde {tag} {i}", ["SQLI-CFG-005", "SQLI-VER-002", "SQLI-IDX-001"], ["SQLI-IDX-001"]); + } + + await UpgradeAsync(hub, $"Kunde {tag} x", ["SQLI-CFG-003"], []); + await UpgradeAsync(hub, $"Kunde {tag} y", ["SQLI-CFG-003"], []); + + var insights = await InsightsAsync(hub.License($"Kunde {tag} 0")); + + var shared = Assert.Single(insights.Upgrades, u => u.RuleId == "SQLI-CFG-005"); + Assert.Equal((15, NewBuild, "CU21", 3, 3, 0), (shared.Major, shared.ToBuild, shared.ToLevel, shared.Customers, shared.Resolved, shared.Persisted)); + Assert.DoesNotContain(insights.Upgrades, u => u.RuleId == "SQLI-VER-002"); + Assert.DoesNotContain(insights.Upgrades, u => u.RuleId == "SQLI-CFG-003"); // only two customers + Assert.DoesNotContain(insights.Upgrades, u => u.RuleId == "SQLI-IDX-001"); // still there + Assert.Equal(3, insights.MinCustomers); + } + + [Fact] + public async Task Findings_still_there_a_week_after_the_update_count_against_it() + { + var tag = Guid.NewGuid().ToString("N")[..6]; + for (var i = 0; i < 3; i++) + { + await UpgradeAsync(hub, $"Stadig {tag} {i}", ["SQLI-STAT-001"], ["SQLI-STAT-001"]); + } + + await using var scope = hub.Factory.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var instances = db.Instances.Where(i => i.Customer!.Name.StartsWith($"Stadig {tag}")).Select(i => i.Id); + await db.VersionChanges.Where(v => instances.Contains(v.InstanceId)).ExecuteUpdateAsync(v => v.SetProperty(x => x.ChangedAt, x => x.ChangedAt.AddDays(-10))); + await db.Findings.Where(f => instances.Contains(f.InstanceId)).ExecuteUpdateAsync(f => f.SetProperty(x => x.FirstSeenAt, x => x.FirstSeenAt.AddDays(-11))); + + var effects = await scope.ServiceProvider.GetRequiredService().UpgradeEffectsAsync(CancellationToken.None); + + var stat = Assert.Single(effects, e => e.RuleId == "SQLI-STAT-001"); + Assert.Equal((0, 3), (stat.Resolved, stat.Persisted)); + Assert.DoesNotContain((await InsightsAsync(hub.License($"Stadig {tag} 0"))).Upgrades, u => u.RuleId == "SQLI-STAT-001"); + } + + [Fact] + public async Task Insights_require_a_license() + { + Assert.Equal(HttpStatusCode.Unauthorized, (await hub.Factory.CreateClient().GetAsync("/api/v1/insights")).StatusCode); + } + + [Fact] + public void Matcher_picks_newer_builds_of_the_same_major_version() + { + var insights = new HubInsights(DateTimeOffset.UtcNow, 3, + [ + new UpgradeInsight("SQLI-CFG-005", 15, "15.0.4415.2", "CU21", 3, 3, 0), + new UpgradeInsight("SQLI-CFG-005", 15, "15.0.4430.1", "CU24", 5, 5, 1), + new UpgradeInsight("SQLI-CFG-005", 16, "16.0.4175.1", "CU20", 9, 9, 0), + ]); + + Assert.Equal(["CU24", "CU21"], HubInsightMatcher.For(insights, "SQLI-CFG-005", "15.0.4385.2").Select(i => i.ToLevel)); + Assert.Equal(["CU24"], HubInsightMatcher.For(insights, "SQLI-CFG-005", "15.0.4420.2").Select(i => i.ToLevel)); + Assert.Empty(HubInsightMatcher.For(insights, "SQLI-CFG-004", "15.0.4385.2")); + Assert.Empty(HubInsightMatcher.For(null, "SQLI-CFG-005", "15.0.4385.2")); + } +}