From 36ad4c06563ecfaa287c3ca6b440d1161edd09b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Eggert=20Lundsteen=20Olsen?= Date: Fri, 2 Oct 2026 13:05:45 +0200 Subject: [PATCH] Repository and agent: Dynamics AX history (migration 0017), seeding, hub opt-in 0017_dynamics_ax adds insight.AxBatchRuns (completed batch runs, EventDays retention) and insight.AxSequenceSamples (one NEXTREC/NEXTVAL sample per sequence per day, HourlyDays retention). The agent reads them back after a restart so job comparisons and forecasts continue, takes its thresholds from SQLInsight:DynamicsAx, and only sends findings of the built-in SQLI-AX rules to the hub for databases with the AX opt-in ticked; AX rule reasons (which can name tables, logins and GRANT text) are replaced by a fixed text. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/SQLInsight.Agent/AgentOptions.cs | 81 +++++++++ src/SQLInsight.Agent/Hub/SubmissionBuilder.cs | 12 +- .../Monitoring/InstanceMonitor.cs | 26 +++ .../Monitoring/MonitoringService.cs | 2 +- src/SQLInsight.Agent/appsettings.json | 9 + .../DynamicsAx/AxBatchCollector.cs | 6 +- .../DynamicsAx/AxSequenceCollector.cs | 7 +- src/SQLInsight.Core/DynamicsAx/AxBatch.cs | 7 + .../InsightRepository.DynamicsAx.cs | 158 ++++++++++++++++++ .../InsightRepository.Maintenance.cs | 6 +- .../InsightRepository.cs | 8 + .../Migrations/0017_dynamics_ax.sql | 37 ++++ .../InstanceConfigurationAgentTests.cs | 23 +++ 13 files changed, 372 insertions(+), 10 deletions(-) create mode 100644 src/SQLInsight.Repository/InsightRepository.DynamicsAx.cs create mode 100644 src/SQLInsight.Repository/Migrations/0017_dynamics_ax.sql diff --git a/src/SQLInsight.Agent/AgentOptions.cs b/src/SQLInsight.Agent/AgentOptions.cs index cbe25e5..122af17 100644 --- a/src/SQLInsight.Agent/AgentOptions.cs +++ b/src/SQLInsight.Agent/AgentOptions.cs @@ -30,6 +30,8 @@ public sealed class AgentOptions public TempdbOptions Tempdb { get; set; } = new(); + public DynamicsAxOptions DynamicsAx { get; set; } = new(); + public IList Instances { get; } = []; } @@ -122,3 +124,82 @@ public sealed class TempdbOptions /// A session using more tempdb than this (MB) for 15 minutes is a finding (SQLI-TEMP-003). public int SessionMegabytes { get; set; } = 1024; } + +/// +/// The Dynamics AX checks (SQLInsight:DynamicsAx). They only run in databases recognised as AX and chosen for monitoring; +/// the data checks need the optional read access (db/permissions/grant-dynamics-ax-read.sql). +/// +public sealed class DynamicsAxOptions +{ + /// A waiting batch job this many minutes past its planned start is late (SQLI-AX-003). + public int BatchLateMinutes { get; set; } = 30; + + /// Late by this many hours: in practice never started (High). + public int BatchNeverStartedHours { get; set; } = 24; + + /// A batch job executing this many hours is stuck (SQLI-AX-004). + public int BatchStuckHours { get; set; } = 6; + + /// Running this many times longer than its own 90th percentile is unusually long (SQLI-AX-002). + public double BatchLongRunningFactor { get; set; } = 2.0; + + /// …but never for runs shorter than this (minutes). + public int BatchLongRunningMinimumMinutes { get; set; } = 10; + + /// Earlier runs needed before a job is compared with its own history. + public int BatchHistoryMinimumRuns { get; set; } = 5; + + /// A run that failed within this many hours is reported (SQLI-AX-001). + public int BatchFailedLookbackHours { get; set; } = 24; + + /// A number sequence used more than this (percent) is a finding (SQLI-AX-005). + public double NumberSequenceWarnPercent { get; set; } = 80; + + /// …and High from this. + public double NumberSequenceCriticalPercent { get; set; } = 95; + + /// Forecast to run out within this many days is a finding (number sequences and 64-bit RecId). + public int ForecastWarnDays { get; set; } = 90; + + /// …and High within this many days. + public int ForecastCriticalDays { get; set; } = 30; + + /// AX 3.0's 32-bit RecId: a finding from this share of the int range (SQLI-AX-009). + public double RecId32WarnPercent { get; set; } = 50; + + /// …and Critical from this. + public double RecId32CriticalPercent { get; set; } = 75; + + /// AX 3.0: forecast to run out within this many days is a finding, Critical within half. + public int RecId32ForecastDays { get; set; } = 730; + + /// 64-bit RecId (or a lowered MAXVAL): a finding from this share. + public double RecId64WarnPercent { get; set; } = 80; + + /// An AX session blocking others this many seconds is reported with its AX user (SQLI-AX-010). + public int BlockingSeconds { get; set; } = 30; + + /// How many days back the forecasts look. + public int ForecastBasisDays { get; set; } = 30; + + public Core.DynamicsAx.AxThresholds ToThresholds() => new() + { + BatchLateMinutes = Math.Max(1, BatchLateMinutes), + BatchNeverStartedHours = Math.Max(1, BatchNeverStartedHours), + BatchStuckHours = Math.Max(1, BatchStuckHours), + BatchLongRunningFactor = Math.Max(1.1, BatchLongRunningFactor), + BatchLongRunningMinimumMinutes = Math.Max(1, BatchLongRunningMinimumMinutes), + BatchHistoryMinimumRuns = Math.Clamp(BatchHistoryMinimumRuns, 1, Core.DynamicsAx.AxBatchHistory.RunsKept), + BatchFailedLookbackHours = Math.Max(1, BatchFailedLookbackHours), + NumberSequenceWarnPercent = Math.Clamp(NumberSequenceWarnPercent, 1, 100), + NumberSequenceCriticalPercent = Math.Clamp(NumberSequenceCriticalPercent, 1, 100), + ForecastWarnDays = Math.Max(1, ForecastWarnDays), + ForecastCriticalDays = Math.Max(1, ForecastCriticalDays), + RecId32WarnPercent = Math.Clamp(RecId32WarnPercent, 1, 100), + RecId32CriticalPercent = Math.Clamp(RecId32CriticalPercent, 1, 100), + RecId32ForecastDays = Math.Max(1, RecId32ForecastDays), + RecId64WarnPercent = Math.Clamp(RecId64WarnPercent, 1, 100), + BlockingSeconds = Math.Max(1, BlockingSeconds), + ForecastBasisDays = Math.Clamp(ForecastBasisDays, 2, 365), + }; +} diff --git a/src/SQLInsight.Agent/Hub/SubmissionBuilder.cs b/src/SQLInsight.Agent/Hub/SubmissionBuilder.cs index 4767856..7d253f2 100644 --- a/src/SQLInsight.Agent/Hub/SubmissionBuilder.cs +++ b/src/SQLInsight.Agent/Hub/SubmissionBuilder.cs @@ -10,6 +10,9 @@ namespace SQLInsight.Agent.Hub; /// public static class SubmissionBuilder { + /// What the hub gets instead of an AX rule's reason (which can name databases, tables and logins). + public const string AxReasonKeptLocally = "Detaljer om Dynamics AX bliver i kundens netværk."; + public static readonly string AgentVersion = typeof(SubmissionBuilder).Assembly.GetName().Version?.ToString() ?? "0.0.0"; @@ -45,7 +48,14 @@ public static class SubmissionBuilder Edition = info.Edition, ServerName = options.SendServerName ? info.ServerName : null, }, - Rules = results.Where(r => axRulesAllowed || !Core.Configuration.DynamicsAxFindings.IsDynamicsAxRule(r.RuleId)).Select(r => new RuleOutcomeDto { RuleId = r.RuleId, Outcome = r.Outcome.ToString(), Reason = r.Reason }).ToList(), + // An AX rule's outcome covers every AX database on the instance, and its reason can name databases, tables, logins + // and the GRANT text: only the outcome goes to the hub, with a fixed reason. + Rules = results.Where(r => axRulesAllowed || !Core.Configuration.DynamicsAxFindings.IsDynamicsAxRule(r.RuleId)).Select(r => new RuleOutcomeDto + { + RuleId = r.RuleId, + Outcome = r.Outcome.ToString(), + Reason = Core.Configuration.DynamicsAxFindings.IsDynamicsAxRule(r.RuleId) && r.Reason is not null ? AxReasonKeptLocally : r.Reason, + }).ToList(), Findings = activeFindings.Where(f => Allowed(f.RuleId, f.Subject)).Select(f => new FindingDto { RuleId = f.RuleId, diff --git a/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs b/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs index 46d148a..fc976cb 100644 --- a/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs +++ b/src/SQLInsight.Agent/Monitoring/InstanceMonitor.cs @@ -267,6 +267,32 @@ public sealed partial class InstanceMonitor( return await repository.GetSnapshotAsync(instanceId, collector.Name, cancellationToken).ConfigureAwait(false); } + // Dynamics AX: each batch job's own run history, the first-seen times of running jobs, the history watermark and the + // sequences' daily samples come back from the repository, so comparisons and forecasts continue after a restart. + if (collector is Collectors.DynamicsAx.AxBatchCollector) + { + var stored = await repository.GetSnapshotAsync(instanceId, collector.Name, cancellationToken).ConfigureAwait(false) + ?? new Core.DynamicsAx.AxBatchSnapshot { CapturedAt = now, Thresholds = new(), Databases = [] }; + var history = await repository.GetAxBatchHistoryAsync(instanceId, now.AddDays(-90), cancellationToken).ConfigureAwait(false); + return stored with + { + History = history, + RecentRunKeys = history.ScopedRunKeys().ToHashSet(StringComparer.Ordinal), + FirstSeenExecuting = stored.Databases + .SelectMany(d => d.Jobs.Where(j => j.FirstSeenExecutingUtc is not null) + .Select(j => (Key: Core.DynamicsAx.AxBatchHistory.Key(d.Database, "#" + j.RecId.ToString(System.Globalization.CultureInfo.InvariantCulture)), At: j.FirstSeenExecutingUtc!.Value))) + .GroupBy(x => x.Key, StringComparer.OrdinalIgnoreCase) + .ToDictionary(g => g.Key, g => g.First().At, StringComparer.OrdinalIgnoreCase), + }; + } + + if (collector is Collectors.DynamicsAx.AxSequenceCollector sequences) + { + var stored = await repository.GetSnapshotAsync(instanceId, collector.Name, cancellationToken).ConfigureAwait(false) + ?? new Core.DynamicsAx.AxSequenceSnapshot { CapturedAt = now, Thresholds = new(), Databases = [] }; + return stored with { History = await repository.GetAxSequenceHistoryAsync(instanceId, now.AddDays(-sequences.HistoryDays), cancellationToken).ConfigureAwait(false) }; + } + if (collector is not Collectors.Concurrency.DeadlockCollector) { return null; diff --git a/src/SQLInsight.Agent/Monitoring/MonitoringService.cs b/src/SQLInsight.Agent/Monitoring/MonitoringService.cs index 00f488b..611bfd6 100644 --- a/src/SQLInsight.Agent/Monitoring/MonitoringService.cs +++ b/src/SQLInsight.Agent/Monitoring/MonitoringService.cs @@ -81,7 +81,7 @@ public sealed partial class MonitoringService( private InstanceMonitor CreateMonitor(MonitoredInstance instance, AgentOptions settings, IReadOnlyDictionary intervals) => new(instance, BuiltInCollectors.Create(customRules, new CollectorSettings(groupResolver, settings.Privileges.MaxSysadminAccounts, settings.TopSql.ActivityThresholdPercent, settings.ErrorLog.LoginFailureBurst, settings.Activity.TransactionMinutes, settings.Activity.TransactionLogMegabytes, settings.Activity.RequestMinutes, - settings.Tempdb.VersionStoreMegabytes, settings.Tempdb.SessionMegabytes)), intervals, connectionFactory, ruleEngine, repository, + settings.Tempdb.VersionStoreMegabytes, settings.Tempdb.SessionMegabytes, settings.DynamicsAx.ToThresholds())), intervals, connectionFactory, ruleEngine, repository, loggerFactory.CreateLogger($"SQLInsight.Instance.{instance.Name}"), notifications, notificationOptions.Value.WebUrl, hubOutbox); /// diff --git a/src/SQLInsight.Agent/appsettings.json b/src/SQLInsight.Agent/appsettings.json index f7a4eee..c5f8190 100644 --- a/src/SQLInsight.Agent/appsettings.json +++ b/src/SQLInsight.Agent/appsettings.json @@ -52,6 +52,15 @@ "EventDays": 90, "ResolvedFindingDays": 365 }, + "DynamicsAx": { + // Dynamics AX-tjek (docs/dynamics-ax.md). Kører kun i databaser genkendt som AX og valgt til overvågning. + "BatchLateMinutes": 30, + "BatchStuckHours": 6, + "BatchLongRunningFactor": 2.0, + "NumberSequenceWarnPercent": 80, + "ForecastWarnDays": 90, + "RecId32WarnPercent": 50 + }, "Instances": [ // Servere tilføjes nemmest i webbrugerfladen (Servere og databaser). Her kan de også stå, fx (se docs/installation.md): // { "Name": "prod-sql01", "Server": "sql01.domæne.local", "Authentication": "Windows", "Provider": "Auto", "Encryption": "Mandatory" } diff --git a/src/SQLInsight.Collectors/DynamicsAx/AxBatchCollector.cs b/src/SQLInsight.Collectors/DynamicsAx/AxBatchCollector.cs index a254f10..f8e863d 100644 --- a/src/SQLInsight.Collectors/DynamicsAx/AxBatchCollector.cs +++ b/src/SQLInsight.Collectors/DynamicsAx/AxBatchCollector.cs @@ -81,7 +81,7 @@ public sealed class AxBatchCollector(AxDatabaseDetector detector, AxThresholds? foreach (var row in historyRows) { watermark = Math.Max(watermark ?? 0, row.RecId); - if (adapter.HistoryRun(row) is { } run && recentKeys.Add(Scoped(name, run.RunKey))) + if (adapter.HistoryRun(row) is { } run && recentKeys.Add(AxBatchHistory.ScopedRunKey(name, run.RunKey))) { newRuns.Add(run); } @@ -103,7 +103,7 @@ public sealed class AxBatchCollector(AxDatabaseDetector detector, AxThresholds? { firstSeen[seenKey] = seenBefore ?? context.Now.UtcDateTime; } - else if (!fromHistoryTable && AxSchemaAdapter.ObservedRun(job, seenBefore) is { } run && recentKeys.Add(Scoped(name, run.RunKey))) + else if (!fromHistoryTable && AxSchemaAdapter.ObservedRun(job, seenBefore) is { } run && recentKeys.Add(AxBatchHistory.ScopedRunKey(name, run.RunKey))) { newRuns.Add(run); } @@ -138,8 +138,6 @@ public sealed class AxBatchCollector(AxDatabaseDetector detector, AxThresholds? }; } - private static string Scoped(string database, string runKey) => database + "\u001f" + runKey; - private static AxBatchJobRow Map(Row r) => new( r.Long(0), r.NullableInt(1), r.NullableString(2), r.NullableString(3), r.NullableString(4), r.Int(5), r.NullableDateTime(6), r.NullableInt(7), r.NullableDateTime(8), r.NullableInt(9), r.NullableDateTime(10), r.NullableInt(11), r.Int(12)); diff --git a/src/SQLInsight.Collectors/DynamicsAx/AxSequenceCollector.cs b/src/SQLInsight.Collectors/DynamicsAx/AxSequenceCollector.cs index 4603a79..63beb4d 100644 --- a/src/SQLInsight.Collectors/DynamicsAx/AxSequenceCollector.cs +++ b/src/SQLInsight.Collectors/DynamicsAx/AxSequenceCollector.cs @@ -14,6 +14,9 @@ public sealed class AxSequenceCollector(AxDatabaseDetector detector, AxThreshold private readonly AxThresholds _thresholds = thresholds ?? new AxThresholds(); public override string Name => "dynamics-ax-sequences"; + + /// How many days of daily samples the forecasts use (and the agent reads back after a restart). + public int HistoryDays => _thresholds.ForecastBasisDays + 1; public override TimeSpan DefaultInterval => TimeSpan.FromHours(1); public override IReadOnlyCollection RequiredCapabilities => [Capability.DynamicManagementViews]; @@ -38,7 +41,7 @@ public sealed class AxSequenceCollector(AxDatabaseDetector detector, AxThreshold { var historyKey = AxSequenceSnapshot.HistoryKey(name, kind, key); var samples = (previous?.History.TryGetValue(historyKey, out var stored) == true ? stored : []) - .Where(s => s.SampledUtc >= now.AddDays(-(_thresholds.ForecastBasisDays + 1)) && s.SampledUtc.Date != now.Date) + .Where(s => s.SampledUtc >= now.AddDays(-HistoryDays) && s.SampledUtc.Date != now.Date) .Append(new AxSample(now, value)) .ToList(); history[historyKey] = samples; @@ -76,7 +79,7 @@ public sealed class AxSequenceCollector(AxDatabaseDetector detector, AxThreshold { if (key.StartsWith(name + "\u001f", StringComparison.Ordinal) && !history.ContainsKey(key)) { - history[key] = samples.Where(s => s.SampledUtc >= now.AddDays(-(_thresholds.ForecastBasisDays + 1))).ToList(); + history[key] = samples.Where(s => s.SampledUtc >= now.AddDays(-HistoryDays)).ToList(); } } } diff --git a/src/SQLInsight.Core/DynamicsAx/AxBatch.cs b/src/SQLInsight.Core/DynamicsAx/AxBatch.cs index 296498f..d78ead0 100644 --- a/src/SQLInsight.Core/DynamicsAx/AxBatch.cs +++ b/src/SQLInsight.Core/DynamicsAx/AxBatch.cs @@ -154,6 +154,13 @@ public sealed class AxBatchHistory public static string Key(string database, string jobKey) => database + "\u001f" + jobKey; + /// A run key made unique across the instance's databases (). + public static string ScopedRunKey(string database, string runKey) => database + "\u001f" + runKey; + + /// The scoped run keys of every run kept. + public IEnumerable ScopedRunKeys() => + _runs.SelectMany(kv => kv.Value.Select(r => ScopedRunKey(kv.Key[..kv.Key.IndexOf('\u001f', StringComparison.Ordinal)], r.RunKey))); + public void Add(string database, AxBatchRun run) { ArgumentNullException.ThrowIfNull(run); diff --git a/src/SQLInsight.Repository/InsightRepository.DynamicsAx.cs b/src/SQLInsight.Repository/InsightRepository.DynamicsAx.cs new file mode 100644 index 0000000..eab4957 --- /dev/null +++ b/src/SQLInsight.Repository/InsightRepository.DynamicsAx.cs @@ -0,0 +1,158 @@ +using System.Text.Json; +using Dapper; +using Microsoft.Data.SqlClient; +using SQLInsight.Core.DynamicsAx; + +namespace SQLInsight.Repository; + +/// A stored completed AX batch run (insight.AxBatchRuns). +public sealed record StoredAxBatchRun(string InstanceName, string DatabaseName, string JobKey, string RunKey, string? Caption, string? Company, + DateTime StartedAt, DateTime EndedAt, AxBatchStatus Status) +{ + public TimeSpan Duration => EndedAt - StartedAt; +} + +/// One stored daily sample of an AX sequence (insight.AxSequenceSamples). +public sealed record StoredAxSequenceSample(string DatabaseName, char Kind, string SequenceKey, DateTime SampledAt, long Value); + +public sealed partial class InsightRepository +{ + /// A job's runs read back after a restart, so it is compared with its own history at once. + public const int AxBatchRunsPerJob = AxBatchHistory.RunsKept; + + private static Task SaveAxBatchRunsAsync(SqlConnection connection, int instanceId, AxBatchSnapshot snapshot) + { + var runs = snapshot.Databases.SelectMany(d => d.NewRuns.Select(r => new + { + Database = d.Database, + r.RunKey, + JobKey = Truncate(r.JobKey, 200), + Caption = r.Caption is null ? null : Truncate(r.Caption, 200), + Company = r.Company is null ? null : Truncate(r.Company, 10), + StartedAt = Iso(r.StartedUtc), + EndedAt = Iso(r.EndedUtc), + Status = (byte)r.Status, + })).Where(r => r.RunKey.Length <= 100).ToList(); + return runs.Count == 0 + ? Task.FromResult(0) + : connection.ExecuteAsync(""" + INSERT insight.AxBatchRuns (InstanceId, DatabaseName, RunKey, JobKey, Caption, Company, StartedAt, EndedAt, Status) + SELECT @instanceId, j.DatabaseName, j.RunKey, j.JobKey, j.Caption, j.Company, j.StartedAt, j.EndedAt, j.Status + FROM (SELECT DISTINCT * FROM OPENJSON(@json) WITH (DatabaseName nvarchar(128) '$.Database', RunKey nvarchar(100), JobKey nvarchar(200), Caption nvarchar(200), + Company nvarchar(10), StartedAt datetime2(0), EndedAt datetime2(0), Status tinyint)) j + WHERE NOT EXISTS (SELECT 1 FROM insight.AxBatchRuns r WHERE r.InstanceId = @instanceId AND r.DatabaseName = j.DatabaseName AND r.RunKey = j.RunKey); + """, new { instanceId, json = JsonSerializer.Serialize(runs, Json) }); + } + + /// Today's sample of every sequence read (the latest of the day wins). + private static Task SaveAxSequenceSamplesAsync(SqlConnection connection, int instanceId, AxSequenceSnapshot snapshot) + { + var at = Utc(snapshot.CapturedAt); + var samples = snapshot.Databases.SelectMany(d => + d.NumberSequences.Select(n => (d.Database, Kind: AxSequenceSnapshot.NumberSequenceKind, n.Key, Value: n.Next)) + .Concat(d.RecIds.Select(r => (d.Database, Kind: AxSequenceSnapshot.RecIdKind, r.Key, Value: r.Next)))) + .Where(s => s.Key.Length <= 100) + .Select(s => new { s.Database, Kind = s.Kind.ToString(), s.Key, s.Value }) + .ToList(); + return samples.Count == 0 + ? Task.FromResult(0) + : connection.ExecuteAsync(""" + MERGE insight.AxSequenceSamples AS t + USING (SELECT DISTINCT DatabaseName, Kind, SequenceKey, Value FROM OPENJSON(@json) + WITH (DatabaseName nvarchar(128) '$.Database', Kind char(1), SequenceKey nvarchar(100) '$.Key', Value bigint)) AS s + ON t.InstanceId = @instanceId AND t.DatabaseName = s.DatabaseName AND t.Kind = s.Kind AND t.SequenceKey = s.SequenceKey AND t.[Day] = @day + WHEN MATCHED THEN UPDATE SET Value = s.Value, SampledAt = @at + WHEN NOT MATCHED THEN INSERT (InstanceId, DatabaseName, Kind, SequenceKey, [Day], SampledAt, Value) + VALUES (@instanceId, s.DatabaseName, s.Kind, s.SequenceKey, @day, @at, s.Value); + """, new { instanceId, at, day = at.Date, json = JsonSerializer.Serialize(samples, Json) }); + } + + private static string Iso(DateTime utc) => utc.ToString("yyyy-MM-ddTHH:mm:ss", System.Globalization.CultureInfo.InvariantCulture); + + private static string Truncate(string value, int length) => value.Length <= length ? value : value[..length]; + + /// + /// The latest runs of every job since : the agent's batch + /// collector starts from them after a restart. + /// + public async Task GetAxBatchHistoryAsync(int instanceId, DateTimeOffset since, CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false); + var rows = await connection.QueryAsync(""" + SELECT DatabaseName, JobKey, RunKey, Caption, Company, StartedAt, EndedAt, Status + FROM (SELECT r.*, ROW_NUMBER() OVER (PARTITION BY r.DatabaseName, r.JobKey ORDER BY r.EndedAt DESC) AS n + FROM insight.AxBatchRuns r WHERE r.InstanceId = @instanceId AND r.EndedAt >= @since) x + WHERE n <= @perJob; + """, new { instanceId, since = Utc(since), perJob = AxBatchRunsPerJob }).ConfigureAwait(false); + var history = new AxBatchHistory(); + foreach (var r in rows) + { + history.Add(r.DatabaseName, new AxBatchRun(r.JobKey, r.RunKey, r.Caption, r.Company, AsUtc(r.StartedAt), AsUtc(r.EndedAt), (AxBatchStatus)r.Status)); + } + + return history; + } + + /// Completed AX batch runs, newest first; for one instance, database and job when given. + public async Task> GetAxBatchRunsAsync(int? instanceId, string? database, string? jobKey, DateTimeOffset from, DateTimeOffset to, int top = 500, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false); + var rows = await connection.QueryAsync(""" + SELECT TOP (@top) i.Name AS InstanceName, r.DatabaseName, r.JobKey, r.RunKey, r.Caption, r.Company, r.StartedAt, r.EndedAt, r.Status + FROM insight.AxBatchRuns r JOIN insight.Instances i ON i.InstanceId = r.InstanceId + WHERE (@instanceId IS NULL OR r.InstanceId = @instanceId) AND (@database IS NULL OR r.DatabaseName = @database) AND (@jobKey IS NULL OR r.JobKey = @jobKey) + AND r.EndedAt >= @from AND r.EndedAt <= @to + ORDER BY r.EndedAt DESC; + """, new { instanceId, database, jobKey, from = Utc(from), to = Utc(to), top }).ConfigureAwait(false); + return rows.Select(r => new StoredAxBatchRun(r.InstanceName ?? string.Empty, r.DatabaseName, r.JobKey, r.RunKey, r.Caption, r.Company, AsUtc(r.StartedAt), AsUtc(r.EndedAt), + (AxBatchStatus)r.Status)).ToList(); + } + + /// The daily samples since , keyed as : the agent's forecasts start from them after a restart. + public async Task>> GetAxSequenceHistoryAsync(int instanceId, DateTimeOffset since, CancellationToken cancellationToken = default) + { + var samples = await GetAxSequenceSamplesAsync(instanceId, null, null, null, since, cancellationToken).ConfigureAwait(false); + return samples.GroupBy(s => AxSequenceSnapshot.HistoryKey(s.DatabaseName, s.Kind, s.SequenceKey), StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => (IReadOnlyList)g.OrderBy(s => s.SampledAt).Select(s => new AxSample(s.SampledAt, s.Value)).ToList(), StringComparer.Ordinal); + } + + /// Daily samples of AX sequences, oldest first; for one database, kind and key when given. + public async Task> GetAxSequenceSamplesAsync(int instanceId, string? database, char? kind, string? key, DateTimeOffset since, + CancellationToken cancellationToken = default) + { + await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false); + var rows = await connection.QueryAsync(""" + SELECT DatabaseName, Kind, SequenceKey, SampledAt, Value + FROM insight.AxSequenceSamples + WHERE InstanceId = @instanceId AND [Day] >= @since + AND (@database IS NULL OR DatabaseName = @database) AND (@kind IS NULL OR Kind = @kind) AND (@key IS NULL OR SequenceKey = @key) + ORDER BY DatabaseName, Kind, SequenceKey, [Day]; + """, new { instanceId, database, kind = kind?.ToString(), key, since = Utc(since).Date }).ConfigureAwait(false); + return rows.Select(r => new StoredAxSequenceSample(r.DatabaseName, r.Kind[0], r.SequenceKey, AsUtc(r.SampledAt), r.Value)).ToList(); + } + + private static DateTime AsUtc(DateTime value) => DateTime.SpecifyKind(value, DateTimeKind.Utc); + + private sealed record AxBatchRunRow + { + public string? InstanceName { get; init; } + public string DatabaseName { get; init; } = string.Empty; + public string JobKey { get; init; } = string.Empty; + public string RunKey { get; init; } = string.Empty; + public string? Caption { get; init; } + public string? Company { get; init; } + public DateTime StartedAt { get; init; } + public DateTime EndedAt { get; init; } + public byte Status { get; init; } + } + + private sealed record AxSampleRow + { + public string DatabaseName { get; init; } = string.Empty; + public string Kind { get; init; } = "N"; + public string SequenceKey { get; init; } = string.Empty; + public DateTime SampledAt { get; init; } + public long Value { get; init; } + } +} diff --git a/src/SQLInsight.Repository/InsightRepository.Maintenance.cs b/src/SQLInsight.Repository/InsightRepository.Maintenance.cs index 876f3b5..c1f8c60 100644 --- a/src/SQLInsight.Repository/InsightRepository.Maintenance.cs +++ b/src/SQLInsight.Repository/InsightRepository.Maintenance.cs @@ -7,10 +7,10 @@ public sealed class RetentionOptions /// Raw samples (5-minute waits, per-minute CPU …). public int RawDays { get; set; } = 14; - /// Hourly aggregates, for long trends. + /// Hourly aggregates, for long trends, and the daily samples of Dynamics AX number sequences and RecIds. public int HourlyDays { get; set; } = 400; - /// Blocking events and deadlock graphs. + /// Blocking events, deadlock graphs and completed Dynamics AX batch runs. public int EventDays { get; set; } = 90; /// Resolved findings. @@ -92,6 +92,8 @@ public sealed partial class InsightRepository await DeleteInBatchesAsync(connection, "insight.IndexUsageDaily", "[Day] < @cutoff", hourly).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.BlockingEvents", "SampledAt < @cutoff", events).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.Deadlocks", "OccurredAt < @cutoff", events).ConfigureAwait(false); + await DeleteInBatchesAsync(connection, "insight.AxBatchRuns", "EndedAt < @cutoff", events).ConfigureAwait(false); + await DeleteInBatchesAsync(connection, "insight.AxSequenceSamples", "[Day] < @cutoff", hourly).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.ErrorLogEvents", "LastAt < @cutoff", events).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.Findings", "ResolvedAt < @cutoff", resolved).ConfigureAwait(false); await DeleteInBatchesAsync(connection, "insight.ConfigurationChanges", "DetectedAt < @cutoff", resolved).ConfigureAwait(false); diff --git a/src/SQLInsight.Repository/InsightRepository.cs b/src/SQLInsight.Repository/InsightRepository.cs index 03a0868..fba4457 100644 --- a/src/SQLInsight.Repository/InsightRepository.cs +++ b/src/SQLInsight.Repository/InsightRepository.cs @@ -289,6 +289,14 @@ public sealed partial class InsightRepository(string connectionString) await SaveLatePrivilegeAuthorsAsync(connection, instanceId, privileges.RecentChanges).ConfigureAwait(false); await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false); return; + case Core.DynamicsAx.AxBatchSnapshot axBatch: + await SaveAxBatchRunsAsync(connection, instanceId, axBatch).ConfigureAwait(false); + await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false); + return; + case Core.DynamicsAx.AxSequenceSnapshot axSequences: + await SaveAxSequenceSamplesAsync(connection, instanceId, axSequences).ConfigureAwait(false); + await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false); + return; default: await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false); return; diff --git a/src/SQLInsight.Repository/Migrations/0017_dynamics_ax.sql b/src/SQLInsight.Repository/Migrations/0017_dynamics_ax.sql new file mode 100644 index 0000000..6ac508d --- /dev/null +++ b/src/SQLInsight.Repository/Migrations/0017_dynamics_ax.sql @@ -0,0 +1,37 @@ +/* 1.4: Dynamics AX. SQLInsight's own history of what it reads from AX, kept in the repository and never in the customer's + AX database (SQLInsight only ever SELECTs from AX, and only with the optional read access). + + AxBatchRuns completed batch job runs: from BATCHJOBHISTORY (AX 2009 and later) or seen by SQLInsight itself + (AX 3.0/4.0, which keep no history). A job's own runs are what "longer than normal" is measured + against. RunKey is unique per database, so a run read twice is stored once. Kept EventDays (90). + AxSequenceSamples one sample a day of each number sequence's NEXTREC (Kind 'N') and each RecId sequence's NEXTVAL + (Kind 'R', SYSTEMSEQUENCES), the latest of the day: the forecasts of when they run out are made from + it. Kept HourlyDays (400). */ +CREATE TABLE insight.AxBatchRuns +( + InstanceId int NOT NULL, + DatabaseName nvarchar(128) NOT NULL, + RunKey nvarchar(100) NOT NULL, + JobKey nvarchar(200) NOT NULL, + Caption nvarchar(200) NULL, + Company nvarchar(10) NULL, + StartedAt datetime2(0) NOT NULL, + EndedAt datetime2(0) NOT NULL, + Status tinyint NOT NULL, + CONSTRAINT PK_AxBatchRuns PRIMARY KEY (InstanceId, DatabaseName, RunKey) +); +CREATE INDEX IX_AxBatchRuns_Job ON insight.AxBatchRuns (InstanceId, DatabaseName, JobKey, EndedAt); +CREATE INDEX IX_AxBatchRuns_EndedAt ON insight.AxBatchRuns (EndedAt); + +CREATE TABLE insight.AxSequenceSamples +( + InstanceId int NOT NULL, + DatabaseName nvarchar(128) NOT NULL, + Kind char(1) NOT NULL CONSTRAINT CK_AxSequenceSamples_Kind CHECK (Kind IN ('N', 'R')), + SequenceKey nvarchar(100) NOT NULL, + [Day] date NOT NULL, + SampledAt datetime2(0) NOT NULL, + Value bigint NOT NULL, + CONSTRAINT PK_AxSequenceSamples PRIMARY KEY (InstanceId, DatabaseName, Kind, SequenceKey, [Day]) +); +CREATE INDEX IX_AxSequenceSamples_Day ON insight.AxSequenceSamples ([Day]); diff --git a/tests/SQLInsight.Agent.Tests/InstanceConfigurationAgentTests.cs b/tests/SQLInsight.Agent.Tests/InstanceConfigurationAgentTests.cs index f938021..298dd6a 100644 --- a/tests/SQLInsight.Agent.Tests/InstanceConfigurationAgentTests.cs +++ b/tests/SQLInsight.Agent.Tests/InstanceConfigurationAgentTests.cs @@ -146,4 +146,27 @@ public sealed class DynamicsAxHubTests Assert.Equal(["database: AX30", "database: AX30"], submission.Findings.Select(f => f.Subject)); Assert.Contains(submission.Rules, r => r.RuleId == "COM-AX-001"); } + + [Fact] + public void The_built_in_ax_rules_follow_the_same_opt_in_and_keep_their_reasons_local() + { + var selection = new DatabaseSelection(true, ["AX30", "AXTEST"], [], ["AX30"]); + RuleResult[] results = + [ + new() { RuleId = "SQLI-AX-001", RuleTitle = "Batchjob fejlet", Outcome = RuleOutcome.Findings }, + new() { RuleId = "SQLI-AX-005", RuleTitle = "Nummerserie", Outcome = RuleOutcome.NotMeasurable, Reason = "Ikke målbar: GRANT SELECT ON [bmssa].[NUMBERSEQUENCETABLE] TO [svc]" }, + ]; + StoredFinding[] findings = + [ + Finding("SQLI-AX-001", "database: AX30 · batch: 1234|dat|NAT"), Finding("SQLI-AX-001", "database: AXTEST · batch: 99"), + ]; + + var none = SubmissionBuilder.Build("ax", Info, results, findings, new HubOptions(), Now); + var allowed = SubmissionBuilder.Build("ax", Info, results, findings, new HubOptions(), Now, selection); + + Assert.Empty(none.Findings); + Assert.Empty(none.Rules); + Assert.Equal(["database: AX30 · batch: 1234|dat|NAT"], allowed.Findings.Select(f => f.Subject)); + Assert.Equal(SubmissionBuilder.AxReasonKeptLocally, allowed.Rules.Single(r => r.RuleId == "SQLI-AX-005").Reason); + } }