Repository and agent: Dynamics AX history (migration 0017), seeding, hub opt-in
0017_dynamics_ax adds insight.AxBatchRuns (completed batch runs, EventDays retention) and insight.AxSequenceSamples (one NEXTREC/NEXTVAL sample per sequence per day, HourlyDays retention). The agent reads them back after a restart so job comparisons and forecasts continue, takes its thresholds from SQLInsight:DynamicsAx, and only sends findings of the built-in SQLI-AX rules to the hub for databases with the AX opt-in ticked; AX rule reasons (which can name tables, logins and GRANT text) are replaced by a fixed text. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
6b945295d0
commit
36ad4c0656
13 changed files with 372 additions and 10 deletions
|
|
@ -30,6 +30,8 @@ public sealed class AgentOptions
|
|||
|
||||
public TempdbOptions Tempdb { get; set; } = new();
|
||||
|
||||
public DynamicsAxOptions DynamicsAx { get; set; } = new();
|
||||
|
||||
public IList<MonitoredInstance> Instances { get; } = [];
|
||||
}
|
||||
|
||||
|
|
@ -122,3 +124,82 @@ public sealed class TempdbOptions
|
|||
/// <summary>A session using more tempdb than this (MB) for 15 minutes is a finding (SQLI-TEMP-003).</summary>
|
||||
public int SessionMegabytes { get; set; } = 1024;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The Dynamics AX checks (SQLInsight:DynamicsAx). They only run in databases recognised as AX and chosen for monitoring;
|
||||
/// the data checks need the optional read access (db/permissions/grant-dynamics-ax-read.sql).
|
||||
/// </summary>
|
||||
public sealed class DynamicsAxOptions
|
||||
{
|
||||
/// <summary>A waiting batch job this many minutes past its planned start is late (SQLI-AX-003).</summary>
|
||||
public int BatchLateMinutes { get; set; } = 30;
|
||||
|
||||
/// <summary>Late by this many hours: in practice never started (High).</summary>
|
||||
public int BatchNeverStartedHours { get; set; } = 24;
|
||||
|
||||
/// <summary>A batch job executing this many hours is stuck (SQLI-AX-004).</summary>
|
||||
public int BatchStuckHours { get; set; } = 6;
|
||||
|
||||
/// <summary>Running this many times longer than its own 90th percentile is unusually long (SQLI-AX-002).</summary>
|
||||
public double BatchLongRunningFactor { get; set; } = 2.0;
|
||||
|
||||
/// <summary>…but never for runs shorter than this (minutes).</summary>
|
||||
public int BatchLongRunningMinimumMinutes { get; set; } = 10;
|
||||
|
||||
/// <summary>Earlier runs needed before a job is compared with its own history.</summary>
|
||||
public int BatchHistoryMinimumRuns { get; set; } = 5;
|
||||
|
||||
/// <summary>A run that failed within this many hours is reported (SQLI-AX-001).</summary>
|
||||
public int BatchFailedLookbackHours { get; set; } = 24;
|
||||
|
||||
/// <summary>A number sequence used more than this (percent) is a finding (SQLI-AX-005).</summary>
|
||||
public double NumberSequenceWarnPercent { get; set; } = 80;
|
||||
|
||||
/// <summary>…and High from this.</summary>
|
||||
public double NumberSequenceCriticalPercent { get; set; } = 95;
|
||||
|
||||
/// <summary>Forecast to run out within this many days is a finding (number sequences and 64-bit RecId).</summary>
|
||||
public int ForecastWarnDays { get; set; } = 90;
|
||||
|
||||
/// <summary>…and High within this many days.</summary>
|
||||
public int ForecastCriticalDays { get; set; } = 30;
|
||||
|
||||
/// <summary>AX 3.0's 32-bit RecId: a finding from this share of the int range (SQLI-AX-009).</summary>
|
||||
public double RecId32WarnPercent { get; set; } = 50;
|
||||
|
||||
/// <summary>…and Critical from this.</summary>
|
||||
public double RecId32CriticalPercent { get; set; } = 75;
|
||||
|
||||
/// <summary>AX 3.0: forecast to run out within this many days is a finding, Critical within half.</summary>
|
||||
public int RecId32ForecastDays { get; set; } = 730;
|
||||
|
||||
/// <summary>64-bit RecId (or a lowered MAXVAL): a finding from this share.</summary>
|
||||
public double RecId64WarnPercent { get; set; } = 80;
|
||||
|
||||
/// <summary>An AX session blocking others this many seconds is reported with its AX user (SQLI-AX-010).</summary>
|
||||
public int BlockingSeconds { get; set; } = 30;
|
||||
|
||||
/// <summary>How many days back the forecasts look.</summary>
|
||||
public int ForecastBasisDays { get; set; } = 30;
|
||||
|
||||
public Core.DynamicsAx.AxThresholds ToThresholds() => new()
|
||||
{
|
||||
BatchLateMinutes = Math.Max(1, BatchLateMinutes),
|
||||
BatchNeverStartedHours = Math.Max(1, BatchNeverStartedHours),
|
||||
BatchStuckHours = Math.Max(1, BatchStuckHours),
|
||||
BatchLongRunningFactor = Math.Max(1.1, BatchLongRunningFactor),
|
||||
BatchLongRunningMinimumMinutes = Math.Max(1, BatchLongRunningMinimumMinutes),
|
||||
BatchHistoryMinimumRuns = Math.Clamp(BatchHistoryMinimumRuns, 1, Core.DynamicsAx.AxBatchHistory.RunsKept),
|
||||
BatchFailedLookbackHours = Math.Max(1, BatchFailedLookbackHours),
|
||||
NumberSequenceWarnPercent = Math.Clamp(NumberSequenceWarnPercent, 1, 100),
|
||||
NumberSequenceCriticalPercent = Math.Clamp(NumberSequenceCriticalPercent, 1, 100),
|
||||
ForecastWarnDays = Math.Max(1, ForecastWarnDays),
|
||||
ForecastCriticalDays = Math.Max(1, ForecastCriticalDays),
|
||||
RecId32WarnPercent = Math.Clamp(RecId32WarnPercent, 1, 100),
|
||||
RecId32CriticalPercent = Math.Clamp(RecId32CriticalPercent, 1, 100),
|
||||
RecId32ForecastDays = Math.Max(1, RecId32ForecastDays),
|
||||
RecId64WarnPercent = Math.Clamp(RecId64WarnPercent, 1, 100),
|
||||
BlockingSeconds = Math.Max(1, BlockingSeconds),
|
||||
ForecastBasisDays = Math.Clamp(ForecastBasisDays, 2, 365),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,9 @@ namespace SQLInsight.Agent.Hub;
|
|||
/// </summary>
|
||||
public static class SubmissionBuilder
|
||||
{
|
||||
/// <summary>What the hub gets instead of an AX rule's reason (which can name databases, tables and logins).</summary>
|
||||
public const string AxReasonKeptLocally = "Detaljer om Dynamics AX bliver i kundens netværk.";
|
||||
|
||||
public static readonly string AgentVersion =
|
||||
typeof(SubmissionBuilder).Assembly.GetName().Version?.ToString() ?? "0.0.0";
|
||||
|
||||
|
|
@ -45,7 +48,14 @@ public static class SubmissionBuilder
|
|||
Edition = info.Edition,
|
||||
ServerName = options.SendServerName ? info.ServerName : null,
|
||||
},
|
||||
Rules = results.Where(r => axRulesAllowed || !Core.Configuration.DynamicsAxFindings.IsDynamicsAxRule(r.RuleId)).Select(r => new RuleOutcomeDto { RuleId = r.RuleId, Outcome = r.Outcome.ToString(), Reason = r.Reason }).ToList(),
|
||||
// An AX rule's outcome covers every AX database on the instance, and its reason can name databases, tables, logins
|
||||
// and the GRANT text: only the outcome goes to the hub, with a fixed reason.
|
||||
Rules = results.Where(r => axRulesAllowed || !Core.Configuration.DynamicsAxFindings.IsDynamicsAxRule(r.RuleId)).Select(r => new RuleOutcomeDto
|
||||
{
|
||||
RuleId = r.RuleId,
|
||||
Outcome = r.Outcome.ToString(),
|
||||
Reason = Core.Configuration.DynamicsAxFindings.IsDynamicsAxRule(r.RuleId) && r.Reason is not null ? AxReasonKeptLocally : r.Reason,
|
||||
}).ToList(),
|
||||
Findings = activeFindings.Where(f => Allowed(f.RuleId, f.Subject)).Select(f => new FindingDto
|
||||
{
|
||||
RuleId = f.RuleId,
|
||||
|
|
|
|||
|
|
@ -267,6 +267,32 @@ public sealed partial class InstanceMonitor(
|
|||
return await repository.GetSnapshotAsync<Core.Data.IndexSnapshot>(instanceId, collector.Name, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Dynamics AX: each batch job's own run history, the first-seen times of running jobs, the history watermark and the
|
||||
// sequences' daily samples come back from the repository, so comparisons and forecasts continue after a restart.
|
||||
if (collector is Collectors.DynamicsAx.AxBatchCollector)
|
||||
{
|
||||
var stored = await repository.GetSnapshotAsync<Core.DynamicsAx.AxBatchSnapshot>(instanceId, collector.Name, cancellationToken).ConfigureAwait(false)
|
||||
?? new Core.DynamicsAx.AxBatchSnapshot { CapturedAt = now, Thresholds = new(), Databases = [] };
|
||||
var history = await repository.GetAxBatchHistoryAsync(instanceId, now.AddDays(-90), cancellationToken).ConfigureAwait(false);
|
||||
return stored with
|
||||
{
|
||||
History = history,
|
||||
RecentRunKeys = history.ScopedRunKeys().ToHashSet(StringComparer.Ordinal),
|
||||
FirstSeenExecuting = stored.Databases
|
||||
.SelectMany(d => d.Jobs.Where(j => j.FirstSeenExecutingUtc is not null)
|
||||
.Select(j => (Key: Core.DynamicsAx.AxBatchHistory.Key(d.Database, "#" + j.RecId.ToString(System.Globalization.CultureInfo.InvariantCulture)), At: j.FirstSeenExecutingUtc!.Value)))
|
||||
.GroupBy(x => x.Key, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(g => g.Key, g => g.First().At, StringComparer.OrdinalIgnoreCase),
|
||||
};
|
||||
}
|
||||
|
||||
if (collector is Collectors.DynamicsAx.AxSequenceCollector sequences)
|
||||
{
|
||||
var stored = await repository.GetSnapshotAsync<Core.DynamicsAx.AxSequenceSnapshot>(instanceId, collector.Name, cancellationToken).ConfigureAwait(false)
|
||||
?? new Core.DynamicsAx.AxSequenceSnapshot { CapturedAt = now, Thresholds = new(), Databases = [] };
|
||||
return stored with { History = await repository.GetAxSequenceHistoryAsync(instanceId, now.AddDays(-sequences.HistoryDays), cancellationToken).ConfigureAwait(false) };
|
||||
}
|
||||
|
||||
if (collector is not Collectors.Concurrency.DeadlockCollector)
|
||||
{
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ public sealed partial class MonitoringService(
|
|||
private InstanceMonitor CreateMonitor(MonitoredInstance instance, AgentOptions settings, IReadOnlyDictionary<string, TimeSpan> intervals) =>
|
||||
new(instance, BuiltInCollectors.Create(customRules, new CollectorSettings(groupResolver, settings.Privileges.MaxSysadminAccounts, settings.TopSql.ActivityThresholdPercent, settings.ErrorLog.LoginFailureBurst,
|
||||
settings.Activity.TransactionMinutes, settings.Activity.TransactionLogMegabytes, settings.Activity.RequestMinutes,
|
||||
settings.Tempdb.VersionStoreMegabytes, settings.Tempdb.SessionMegabytes)), intervals, connectionFactory, ruleEngine, repository,
|
||||
settings.Tempdb.VersionStoreMegabytes, settings.Tempdb.SessionMegabytes, settings.DynamicsAx.ToThresholds())), intervals, connectionFactory, ruleEngine, repository,
|
||||
loggerFactory.CreateLogger($"SQLInsight.Instance.{instance.Name}"), notifications, notificationOptions.Value.WebUrl, hubOutbox);
|
||||
|
||||
/// <summary>
|
||||
|
|
|
|||
|
|
@ -52,6 +52,15 @@
|
|||
"EventDays": 90,
|
||||
"ResolvedFindingDays": 365
|
||||
},
|
||||
"DynamicsAx": {
|
||||
// Dynamics AX-tjek (docs/dynamics-ax.md). Kører kun i databaser genkendt som AX og valgt til overvågning.
|
||||
"BatchLateMinutes": 30,
|
||||
"BatchStuckHours": 6,
|
||||
"BatchLongRunningFactor": 2.0,
|
||||
"NumberSequenceWarnPercent": 80,
|
||||
"ForecastWarnDays": 90,
|
||||
"RecId32WarnPercent": 50
|
||||
},
|
||||
"Instances": [
|
||||
// Servere tilføjes nemmest i webbrugerfladen (Servere og databaser). Her kan de også stå, fx (se docs/installation.md):
|
||||
// { "Name": "prod-sql01", "Server": "sql01.domæne.local", "Authentication": "Windows", "Provider": "Auto", "Encryption": "Mandatory" }
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ public sealed class AxBatchCollector(AxDatabaseDetector detector, AxThresholds?
|
|||
foreach (var row in historyRows)
|
||||
{
|
||||
watermark = Math.Max(watermark ?? 0, row.RecId);
|
||||
if (adapter.HistoryRun(row) is { } run && recentKeys.Add(Scoped(name, run.RunKey)))
|
||||
if (adapter.HistoryRun(row) is { } run && recentKeys.Add(AxBatchHistory.ScopedRunKey(name, run.RunKey)))
|
||||
{
|
||||
newRuns.Add(run);
|
||||
}
|
||||
|
|
@ -103,7 +103,7 @@ public sealed class AxBatchCollector(AxDatabaseDetector detector, AxThresholds?
|
|||
{
|
||||
firstSeen[seenKey] = seenBefore ?? context.Now.UtcDateTime;
|
||||
}
|
||||
else if (!fromHistoryTable && AxSchemaAdapter.ObservedRun(job, seenBefore) is { } run && recentKeys.Add(Scoped(name, run.RunKey)))
|
||||
else if (!fromHistoryTable && AxSchemaAdapter.ObservedRun(job, seenBefore) is { } run && recentKeys.Add(AxBatchHistory.ScopedRunKey(name, run.RunKey)))
|
||||
{
|
||||
newRuns.Add(run);
|
||||
}
|
||||
|
|
@ -138,8 +138,6 @@ public sealed class AxBatchCollector(AxDatabaseDetector detector, AxThresholds?
|
|||
};
|
||||
}
|
||||
|
||||
private static string Scoped(string database, string runKey) => database + "\u001f" + runKey;
|
||||
|
||||
private static AxBatchJobRow Map(Row r) => new(
|
||||
r.Long(0), r.NullableInt(1), r.NullableString(2), r.NullableString(3), r.NullableString(4), r.Int(5),
|
||||
r.NullableDateTime(6), r.NullableInt(7), r.NullableDateTime(8), r.NullableInt(9), r.NullableDateTime(10), r.NullableInt(11), r.Int(12));
|
||||
|
|
|
|||
|
|
@ -14,6 +14,9 @@ public sealed class AxSequenceCollector(AxDatabaseDetector detector, AxThreshold
|
|||
private readonly AxThresholds _thresholds = thresholds ?? new AxThresholds();
|
||||
|
||||
public override string Name => "dynamics-ax-sequences";
|
||||
|
||||
/// <summary>How many days of daily samples the forecasts use (and the agent reads back after a restart).</summary>
|
||||
public int HistoryDays => _thresholds.ForecastBasisDays + 1;
|
||||
public override TimeSpan DefaultInterval => TimeSpan.FromHours(1);
|
||||
public override IReadOnlyCollection<Capability> RequiredCapabilities => [Capability.DynamicManagementViews];
|
||||
|
||||
|
|
@ -38,7 +41,7 @@ public sealed class AxSequenceCollector(AxDatabaseDetector detector, AxThreshold
|
|||
{
|
||||
var historyKey = AxSequenceSnapshot.HistoryKey(name, kind, key);
|
||||
var samples = (previous?.History.TryGetValue(historyKey, out var stored) == true ? stored : [])
|
||||
.Where(s => s.SampledUtc >= now.AddDays(-(_thresholds.ForecastBasisDays + 1)) && s.SampledUtc.Date != now.Date)
|
||||
.Where(s => s.SampledUtc >= now.AddDays(-HistoryDays) && s.SampledUtc.Date != now.Date)
|
||||
.Append(new AxSample(now, value))
|
||||
.ToList();
|
||||
history[historyKey] = samples;
|
||||
|
|
@ -76,7 +79,7 @@ public sealed class AxSequenceCollector(AxDatabaseDetector detector, AxThreshold
|
|||
{
|
||||
if (key.StartsWith(name + "\u001f", StringComparison.Ordinal) && !history.ContainsKey(key))
|
||||
{
|
||||
history[key] = samples.Where(s => s.SampledUtc >= now.AddDays(-(_thresholds.ForecastBasisDays + 1))).ToList();
|
||||
history[key] = samples.Where(s => s.SampledUtc >= now.AddDays(-HistoryDays)).ToList();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -154,6 +154,13 @@ public sealed class AxBatchHistory
|
|||
|
||||
public static string Key(string database, string jobKey) => database + "\u001f" + jobKey;
|
||||
|
||||
/// <summary>A run key made unique across the instance's databases (<see cref="AxBatchSnapshot.RecentRunKeys"/>).</summary>
|
||||
public static string ScopedRunKey(string database, string runKey) => database + "\u001f" + runKey;
|
||||
|
||||
/// <summary>The scoped run keys of every run kept.</summary>
|
||||
public IEnumerable<string> ScopedRunKeys() =>
|
||||
_runs.SelectMany(kv => kv.Value.Select(r => ScopedRunKey(kv.Key[..kv.Key.IndexOf('\u001f', StringComparison.Ordinal)], r.RunKey)));
|
||||
|
||||
public void Add(string database, AxBatchRun run)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(run);
|
||||
|
|
|
|||
158
src/SQLInsight.Repository/InsightRepository.DynamicsAx.cs
Normal file
158
src/SQLInsight.Repository/InsightRepository.DynamicsAx.cs
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
using System.Text.Json;
|
||||
using Dapper;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using SQLInsight.Core.DynamicsAx;
|
||||
|
||||
namespace SQLInsight.Repository;
|
||||
|
||||
/// <summary>A stored completed AX batch run (insight.AxBatchRuns).</summary>
|
||||
public sealed record StoredAxBatchRun(string InstanceName, string DatabaseName, string JobKey, string RunKey, string? Caption, string? Company,
|
||||
DateTime StartedAt, DateTime EndedAt, AxBatchStatus Status)
|
||||
{
|
||||
public TimeSpan Duration => EndedAt - StartedAt;
|
||||
}
|
||||
|
||||
/// <summary>One stored daily sample of an AX sequence (insight.AxSequenceSamples).</summary>
|
||||
public sealed record StoredAxSequenceSample(string DatabaseName, char Kind, string SequenceKey, DateTime SampledAt, long Value);
|
||||
|
||||
public sealed partial class InsightRepository
|
||||
{
|
||||
/// <summary>A job's runs read back after a restart, so it is compared with its own history at once.</summary>
|
||||
public const int AxBatchRunsPerJob = AxBatchHistory.RunsKept;
|
||||
|
||||
private static Task<int> SaveAxBatchRunsAsync(SqlConnection connection, int instanceId, AxBatchSnapshot snapshot)
|
||||
{
|
||||
var runs = snapshot.Databases.SelectMany(d => d.NewRuns.Select(r => new
|
||||
{
|
||||
Database = d.Database,
|
||||
r.RunKey,
|
||||
JobKey = Truncate(r.JobKey, 200),
|
||||
Caption = r.Caption is null ? null : Truncate(r.Caption, 200),
|
||||
Company = r.Company is null ? null : Truncate(r.Company, 10),
|
||||
StartedAt = Iso(r.StartedUtc),
|
||||
EndedAt = Iso(r.EndedUtc),
|
||||
Status = (byte)r.Status,
|
||||
})).Where(r => r.RunKey.Length <= 100).ToList();
|
||||
return runs.Count == 0
|
||||
? Task.FromResult(0)
|
||||
: connection.ExecuteAsync("""
|
||||
INSERT insight.AxBatchRuns (InstanceId, DatabaseName, RunKey, JobKey, Caption, Company, StartedAt, EndedAt, Status)
|
||||
SELECT @instanceId, j.DatabaseName, j.RunKey, j.JobKey, j.Caption, j.Company, j.StartedAt, j.EndedAt, j.Status
|
||||
FROM (SELECT DISTINCT * FROM OPENJSON(@json) WITH (DatabaseName nvarchar(128) '$.Database', RunKey nvarchar(100), JobKey nvarchar(200), Caption nvarchar(200),
|
||||
Company nvarchar(10), StartedAt datetime2(0), EndedAt datetime2(0), Status tinyint)) j
|
||||
WHERE NOT EXISTS (SELECT 1 FROM insight.AxBatchRuns r WHERE r.InstanceId = @instanceId AND r.DatabaseName = j.DatabaseName AND r.RunKey = j.RunKey);
|
||||
""", new { instanceId, json = JsonSerializer.Serialize(runs, Json) });
|
||||
}
|
||||
|
||||
/// <summary>Today's sample of every sequence read (the latest of the day wins).</summary>
|
||||
private static Task<int> SaveAxSequenceSamplesAsync(SqlConnection connection, int instanceId, AxSequenceSnapshot snapshot)
|
||||
{
|
||||
var at = Utc(snapshot.CapturedAt);
|
||||
var samples = snapshot.Databases.SelectMany(d =>
|
||||
d.NumberSequences.Select(n => (d.Database, Kind: AxSequenceSnapshot.NumberSequenceKind, n.Key, Value: n.Next))
|
||||
.Concat(d.RecIds.Select(r => (d.Database, Kind: AxSequenceSnapshot.RecIdKind, r.Key, Value: r.Next))))
|
||||
.Where(s => s.Key.Length <= 100)
|
||||
.Select(s => new { s.Database, Kind = s.Kind.ToString(), s.Key, s.Value })
|
||||
.ToList();
|
||||
return samples.Count == 0
|
||||
? Task.FromResult(0)
|
||||
: connection.ExecuteAsync("""
|
||||
MERGE insight.AxSequenceSamples AS t
|
||||
USING (SELECT DISTINCT DatabaseName, Kind, SequenceKey, Value FROM OPENJSON(@json)
|
||||
WITH (DatabaseName nvarchar(128) '$.Database', Kind char(1), SequenceKey nvarchar(100) '$.Key', Value bigint)) AS s
|
||||
ON t.InstanceId = @instanceId AND t.DatabaseName = s.DatabaseName AND t.Kind = s.Kind AND t.SequenceKey = s.SequenceKey AND t.[Day] = @day
|
||||
WHEN MATCHED THEN UPDATE SET Value = s.Value, SampledAt = @at
|
||||
WHEN NOT MATCHED THEN INSERT (InstanceId, DatabaseName, Kind, SequenceKey, [Day], SampledAt, Value)
|
||||
VALUES (@instanceId, s.DatabaseName, s.Kind, s.SequenceKey, @day, @at, s.Value);
|
||||
""", new { instanceId, at, day = at.Date, json = JsonSerializer.Serialize(samples, Json) });
|
||||
}
|
||||
|
||||
private static string Iso(DateTime utc) => utc.ToString("yyyy-MM-ddTHH:mm:ss", System.Globalization.CultureInfo.InvariantCulture);
|
||||
|
||||
private static string Truncate(string value, int length) => value.Length <= length ? value : value[..length];
|
||||
|
||||
/// <summary>
|
||||
/// The latest <see cref="AxBatchRunsPerJob"/> runs of every job since <paramref name="since"/>: the agent's batch
|
||||
/// collector starts from them after a restart.
|
||||
/// </summary>
|
||||
public async Task<AxBatchHistory> GetAxBatchHistoryAsync(int instanceId, DateTimeOffset since, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
var rows = await connection.QueryAsync<AxBatchRunRow>("""
|
||||
SELECT DatabaseName, JobKey, RunKey, Caption, Company, StartedAt, EndedAt, Status
|
||||
FROM (SELECT r.*, ROW_NUMBER() OVER (PARTITION BY r.DatabaseName, r.JobKey ORDER BY r.EndedAt DESC) AS n
|
||||
FROM insight.AxBatchRuns r WHERE r.InstanceId = @instanceId AND r.EndedAt >= @since) x
|
||||
WHERE n <= @perJob;
|
||||
""", new { instanceId, since = Utc(since), perJob = AxBatchRunsPerJob }).ConfigureAwait(false);
|
||||
var history = new AxBatchHistory();
|
||||
foreach (var r in rows)
|
||||
{
|
||||
history.Add(r.DatabaseName, new AxBatchRun(r.JobKey, r.RunKey, r.Caption, r.Company, AsUtc(r.StartedAt), AsUtc(r.EndedAt), (AxBatchStatus)r.Status));
|
||||
}
|
||||
|
||||
return history;
|
||||
}
|
||||
|
||||
/// <summary>Completed AX batch runs, newest first; for one instance, database and job when given.</summary>
|
||||
public async Task<IReadOnlyList<StoredAxBatchRun>> GetAxBatchRunsAsync(int? instanceId, string? database, string? jobKey, DateTimeOffset from, DateTimeOffset to, int top = 500,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
var rows = await connection.QueryAsync<AxBatchRunRow>("""
|
||||
SELECT TOP (@top) i.Name AS InstanceName, r.DatabaseName, r.JobKey, r.RunKey, r.Caption, r.Company, r.StartedAt, r.EndedAt, r.Status
|
||||
FROM insight.AxBatchRuns r JOIN insight.Instances i ON i.InstanceId = r.InstanceId
|
||||
WHERE (@instanceId IS NULL OR r.InstanceId = @instanceId) AND (@database IS NULL OR r.DatabaseName = @database) AND (@jobKey IS NULL OR r.JobKey = @jobKey)
|
||||
AND r.EndedAt >= @from AND r.EndedAt <= @to
|
||||
ORDER BY r.EndedAt DESC;
|
||||
""", new { instanceId, database, jobKey, from = Utc(from), to = Utc(to), top }).ConfigureAwait(false);
|
||||
return rows.Select(r => new StoredAxBatchRun(r.InstanceName ?? string.Empty, r.DatabaseName, r.JobKey, r.RunKey, r.Caption, r.Company, AsUtc(r.StartedAt), AsUtc(r.EndedAt),
|
||||
(AxBatchStatus)r.Status)).ToList();
|
||||
}
|
||||
|
||||
/// <summary>The daily samples since <paramref name="since"/>, keyed as <see cref="AxSequenceSnapshot.HistoryKey"/>: the agent's forecasts start from them after a restart.</summary>
|
||||
public async Task<IReadOnlyDictionary<string, IReadOnlyList<AxSample>>> GetAxSequenceHistoryAsync(int instanceId, DateTimeOffset since, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var samples = await GetAxSequenceSamplesAsync(instanceId, null, null, null, since, cancellationToken).ConfigureAwait(false);
|
||||
return samples.GroupBy(s => AxSequenceSnapshot.HistoryKey(s.DatabaseName, s.Kind, s.SequenceKey), StringComparer.Ordinal)
|
||||
.ToDictionary(g => g.Key, g => (IReadOnlyList<AxSample>)g.OrderBy(s => s.SampledAt).Select(s => new AxSample(s.SampledAt, s.Value)).ToList(), StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>Daily samples of AX sequences, oldest first; for one database, kind and key when given.</summary>
|
||||
public async Task<IReadOnlyList<StoredAxSequenceSample>> GetAxSequenceSamplesAsync(int instanceId, string? database, char? kind, string? key, DateTimeOffset since,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
var rows = await connection.QueryAsync<AxSampleRow>("""
|
||||
SELECT DatabaseName, Kind, SequenceKey, SampledAt, Value
|
||||
FROM insight.AxSequenceSamples
|
||||
WHERE InstanceId = @instanceId AND [Day] >= @since
|
||||
AND (@database IS NULL OR DatabaseName = @database) AND (@kind IS NULL OR Kind = @kind) AND (@key IS NULL OR SequenceKey = @key)
|
||||
ORDER BY DatabaseName, Kind, SequenceKey, [Day];
|
||||
""", new { instanceId, database, kind = kind?.ToString(), key, since = Utc(since).Date }).ConfigureAwait(false);
|
||||
return rows.Select(r => new StoredAxSequenceSample(r.DatabaseName, r.Kind[0], r.SequenceKey, AsUtc(r.SampledAt), r.Value)).ToList();
|
||||
}
|
||||
|
||||
private static DateTime AsUtc(DateTime value) => DateTime.SpecifyKind(value, DateTimeKind.Utc);
|
||||
|
||||
private sealed record AxBatchRunRow
|
||||
{
|
||||
public string? InstanceName { get; init; }
|
||||
public string DatabaseName { get; init; } = string.Empty;
|
||||
public string JobKey { get; init; } = string.Empty;
|
||||
public string RunKey { get; init; } = string.Empty;
|
||||
public string? Caption { get; init; }
|
||||
public string? Company { get; init; }
|
||||
public DateTime StartedAt { get; init; }
|
||||
public DateTime EndedAt { get; init; }
|
||||
public byte Status { get; init; }
|
||||
}
|
||||
|
||||
private sealed record AxSampleRow
|
||||
{
|
||||
public string DatabaseName { get; init; } = string.Empty;
|
||||
public string Kind { get; init; } = "N";
|
||||
public string SequenceKey { get; init; } = string.Empty;
|
||||
public DateTime SampledAt { get; init; }
|
||||
public long Value { get; init; }
|
||||
}
|
||||
}
|
||||
|
|
@ -7,10 +7,10 @@ public sealed class RetentionOptions
|
|||
/// <summary>Raw samples (5-minute waits, per-minute CPU …).</summary>
|
||||
public int RawDays { get; set; } = 14;
|
||||
|
||||
/// <summary>Hourly aggregates, for long trends.</summary>
|
||||
/// <summary>Hourly aggregates, for long trends, and the daily samples of Dynamics AX number sequences and RecIds.</summary>
|
||||
public int HourlyDays { get; set; } = 400;
|
||||
|
||||
/// <summary>Blocking events and deadlock graphs.</summary>
|
||||
/// <summary>Blocking events, deadlock graphs and completed Dynamics AX batch runs.</summary>
|
||||
public int EventDays { get; set; } = 90;
|
||||
|
||||
/// <summary>Resolved findings.</summary>
|
||||
|
|
@ -92,6 +92,8 @@ public sealed partial class InsightRepository
|
|||
await DeleteInBatchesAsync(connection, "insight.IndexUsageDaily", "[Day] < @cutoff", hourly).ConfigureAwait(false);
|
||||
await DeleteInBatchesAsync(connection, "insight.BlockingEvents", "SampledAt < @cutoff", events).ConfigureAwait(false);
|
||||
await DeleteInBatchesAsync(connection, "insight.Deadlocks", "OccurredAt < @cutoff", events).ConfigureAwait(false);
|
||||
await DeleteInBatchesAsync(connection, "insight.AxBatchRuns", "EndedAt < @cutoff", events).ConfigureAwait(false);
|
||||
await DeleteInBatchesAsync(connection, "insight.AxSequenceSamples", "[Day] < @cutoff", hourly).ConfigureAwait(false);
|
||||
await DeleteInBatchesAsync(connection, "insight.ErrorLogEvents", "LastAt < @cutoff", events).ConfigureAwait(false);
|
||||
await DeleteInBatchesAsync(connection, "insight.Findings", "ResolvedAt < @cutoff", resolved).ConfigureAwait(false);
|
||||
await DeleteInBatchesAsync(connection, "insight.ConfigurationChanges", "DetectedAt < @cutoff", resolved).ConfigureAwait(false);
|
||||
|
|
|
|||
|
|
@ -289,6 +289,14 @@ public sealed partial class InsightRepository(string connectionString)
|
|||
await SaveLatePrivilegeAuthorsAsync(connection, instanceId, privileges.RecentChanges).ConfigureAwait(false);
|
||||
await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false);
|
||||
return;
|
||||
case Core.DynamicsAx.AxBatchSnapshot axBatch:
|
||||
await SaveAxBatchRunsAsync(connection, instanceId, axBatch).ConfigureAwait(false);
|
||||
await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false);
|
||||
return;
|
||||
case Core.DynamicsAx.AxSequenceSnapshot axSequences:
|
||||
await SaveAxSequenceSamplesAsync(connection, instanceId, axSequences).ConfigureAwait(false);
|
||||
await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false);
|
||||
return;
|
||||
default:
|
||||
await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false);
|
||||
return;
|
||||
|
|
|
|||
37
src/SQLInsight.Repository/Migrations/0017_dynamics_ax.sql
Normal file
37
src/SQLInsight.Repository/Migrations/0017_dynamics_ax.sql
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
/* 1.4: Dynamics AX. SQLInsight's own history of what it reads from AX, kept in the repository and never in the customer's
|
||||
AX database (SQLInsight only ever SELECTs from AX, and only with the optional read access).
|
||||
|
||||
AxBatchRuns completed batch job runs: from BATCHJOBHISTORY (AX 2009 and later) or seen by SQLInsight itself
|
||||
(AX 3.0/4.0, which keep no history). A job's own runs are what "longer than normal" is measured
|
||||
against. RunKey is unique per database, so a run read twice is stored once. Kept EventDays (90).
|
||||
AxSequenceSamples one sample a day of each number sequence's NEXTREC (Kind 'N') and each RecId sequence's NEXTVAL
|
||||
(Kind 'R', SYSTEMSEQUENCES), the latest of the day: the forecasts of when they run out are made from
|
||||
it. Kept HourlyDays (400). */
|
||||
CREATE TABLE insight.AxBatchRuns
|
||||
(
|
||||
InstanceId int NOT NULL,
|
||||
DatabaseName nvarchar(128) NOT NULL,
|
||||
RunKey nvarchar(100) NOT NULL,
|
||||
JobKey nvarchar(200) NOT NULL,
|
||||
Caption nvarchar(200) NULL,
|
||||
Company nvarchar(10) NULL,
|
||||
StartedAt datetime2(0) NOT NULL,
|
||||
EndedAt datetime2(0) NOT NULL,
|
||||
Status tinyint NOT NULL,
|
||||
CONSTRAINT PK_AxBatchRuns PRIMARY KEY (InstanceId, DatabaseName, RunKey)
|
||||
);
|
||||
CREATE INDEX IX_AxBatchRuns_Job ON insight.AxBatchRuns (InstanceId, DatabaseName, JobKey, EndedAt);
|
||||
CREATE INDEX IX_AxBatchRuns_EndedAt ON insight.AxBatchRuns (EndedAt);
|
||||
|
||||
CREATE TABLE insight.AxSequenceSamples
|
||||
(
|
||||
InstanceId int NOT NULL,
|
||||
DatabaseName nvarchar(128) NOT NULL,
|
||||
Kind char(1) NOT NULL CONSTRAINT CK_AxSequenceSamples_Kind CHECK (Kind IN ('N', 'R')),
|
||||
SequenceKey nvarchar(100) NOT NULL,
|
||||
[Day] date NOT NULL,
|
||||
SampledAt datetime2(0) NOT NULL,
|
||||
Value bigint NOT NULL,
|
||||
CONSTRAINT PK_AxSequenceSamples PRIMARY KEY (InstanceId, DatabaseName, Kind, SequenceKey, [Day])
|
||||
);
|
||||
CREATE INDEX IX_AxSequenceSamples_Day ON insight.AxSequenceSamples ([Day]);
|
||||
|
|
@ -146,4 +146,27 @@ public sealed class DynamicsAxHubTests
|
|||
Assert.Equal(["database: AX30", "database: AX30"], submission.Findings.Select(f => f.Subject));
|
||||
Assert.Contains(submission.Rules, r => r.RuleId == "COM-AX-001");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_built_in_ax_rules_follow_the_same_opt_in_and_keep_their_reasons_local()
|
||||
{
|
||||
var selection = new DatabaseSelection(true, ["AX30", "AXTEST"], [], ["AX30"]);
|
||||
RuleResult[] results =
|
||||
[
|
||||
new() { RuleId = "SQLI-AX-001", RuleTitle = "Batchjob fejlet", Outcome = RuleOutcome.Findings },
|
||||
new() { RuleId = "SQLI-AX-005", RuleTitle = "Nummerserie", Outcome = RuleOutcome.NotMeasurable, Reason = "Ikke målbar: GRANT SELECT ON [bmssa].[NUMBERSEQUENCETABLE] TO [svc]" },
|
||||
];
|
||||
StoredFinding[] findings =
|
||||
[
|
||||
Finding("SQLI-AX-001", "database: AX30 · batch: 1234|dat|NAT"), Finding("SQLI-AX-001", "database: AXTEST · batch: 99"),
|
||||
];
|
||||
|
||||
var none = SubmissionBuilder.Build("ax", Info, results, findings, new HubOptions(), Now);
|
||||
var allowed = SubmissionBuilder.Build("ax", Info, results, findings, new HubOptions(), Now, selection);
|
||||
|
||||
Assert.Empty(none.Findings);
|
||||
Assert.Empty(none.Rules);
|
||||
Assert.Equal(["database: AX30 · batch: 1234|dat|NAT"], allowed.Findings.Select(f => f.Subject));
|
||||
Assert.Equal(SubmissionBuilder.AxReasonKeptLocally, allowed.Rules.Single(r => r.RuleId == "SQLI-AX-005").Reason);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue