Repository and agent: Dynamics AX history (migration 0017), seeding, hub opt-in
0017_dynamics_ax adds insight.AxBatchRuns (completed batch runs, EventDays retention) and insight.AxSequenceSamples (one NEXTREC/NEXTVAL sample per sequence per day, HourlyDays retention). The agent reads them back after a restart so job comparisons and forecasts continue, takes its thresholds from SQLInsight:DynamicsAx, and only sends findings of the built-in SQLI-AX rules to the hub for databases with the AX opt-in ticked; AX rule reasons (which can name tables, logins and GRANT text) are replaced by a fixed text. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
6b945295d0
commit
36ad4c0656
13 changed files with 372 additions and 10 deletions
|
|
@ -30,6 +30,8 @@ public sealed class AgentOptions
|
||||||
|
|
||||||
public TempdbOptions Tempdb { get; set; } = new();
|
public TempdbOptions Tempdb { get; set; } = new();
|
||||||
|
|
||||||
|
public DynamicsAxOptions DynamicsAx { get; set; } = new();
|
||||||
|
|
||||||
public IList<MonitoredInstance> Instances { get; } = [];
|
public IList<MonitoredInstance> Instances { get; } = [];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -122,3 +124,82 @@ public sealed class TempdbOptions
|
||||||
/// <summary>A session using more tempdb than this (MB) for 15 minutes is a finding (SQLI-TEMP-003).</summary>
|
/// <summary>A session using more tempdb than this (MB) for 15 minutes is a finding (SQLI-TEMP-003).</summary>
|
||||||
public int SessionMegabytes { get; set; } = 1024;
|
public int SessionMegabytes { get; set; } = 1024;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The Dynamics AX checks (SQLInsight:DynamicsAx). They only run in databases recognised as AX and chosen for monitoring;
|
||||||
|
/// the data checks need the optional read access (db/permissions/grant-dynamics-ax-read.sql).
|
||||||
|
/// </summary>
|
||||||
|
public sealed class DynamicsAxOptions
|
||||||
|
{
|
||||||
|
/// <summary>A waiting batch job this many minutes past its planned start is late (SQLI-AX-003).</summary>
|
||||||
|
public int BatchLateMinutes { get; set; } = 30;
|
||||||
|
|
||||||
|
/// <summary>Late by this many hours: in practice never started (High).</summary>
|
||||||
|
public int BatchNeverStartedHours { get; set; } = 24;
|
||||||
|
|
||||||
|
/// <summary>A batch job executing this many hours is stuck (SQLI-AX-004).</summary>
|
||||||
|
public int BatchStuckHours { get; set; } = 6;
|
||||||
|
|
||||||
|
/// <summary>Running this many times longer than its own 90th percentile is unusually long (SQLI-AX-002).</summary>
|
||||||
|
public double BatchLongRunningFactor { get; set; } = 2.0;
|
||||||
|
|
||||||
|
/// <summary>…but never for runs shorter than this (minutes).</summary>
|
||||||
|
public int BatchLongRunningMinimumMinutes { get; set; } = 10;
|
||||||
|
|
||||||
|
/// <summary>Earlier runs needed before a job is compared with its own history.</summary>
|
||||||
|
public int BatchHistoryMinimumRuns { get; set; } = 5;
|
||||||
|
|
||||||
|
/// <summary>A run that failed within this many hours is reported (SQLI-AX-001).</summary>
|
||||||
|
public int BatchFailedLookbackHours { get; set; } = 24;
|
||||||
|
|
||||||
|
/// <summary>A number sequence used more than this (percent) is a finding (SQLI-AX-005).</summary>
|
||||||
|
public double NumberSequenceWarnPercent { get; set; } = 80;
|
||||||
|
|
||||||
|
/// <summary>…and High from this.</summary>
|
||||||
|
public double NumberSequenceCriticalPercent { get; set; } = 95;
|
||||||
|
|
||||||
|
/// <summary>Forecast to run out within this many days is a finding (number sequences and 64-bit RecId).</summary>
|
||||||
|
public int ForecastWarnDays { get; set; } = 90;
|
||||||
|
|
||||||
|
/// <summary>…and High within this many days.</summary>
|
||||||
|
public int ForecastCriticalDays { get; set; } = 30;
|
||||||
|
|
||||||
|
/// <summary>AX 3.0's 32-bit RecId: a finding from this share of the int range (SQLI-AX-009).</summary>
|
||||||
|
public double RecId32WarnPercent { get; set; } = 50;
|
||||||
|
|
||||||
|
/// <summary>…and Critical from this.</summary>
|
||||||
|
public double RecId32CriticalPercent { get; set; } = 75;
|
||||||
|
|
||||||
|
/// <summary>AX 3.0: forecast to run out within this many days is a finding, Critical within half.</summary>
|
||||||
|
public int RecId32ForecastDays { get; set; } = 730;
|
||||||
|
|
||||||
|
/// <summary>64-bit RecId (or a lowered MAXVAL): a finding from this share.</summary>
|
||||||
|
public double RecId64WarnPercent { get; set; } = 80;
|
||||||
|
|
||||||
|
/// <summary>An AX session blocking others this many seconds is reported with its AX user (SQLI-AX-010).</summary>
|
||||||
|
public int BlockingSeconds { get; set; } = 30;
|
||||||
|
|
||||||
|
/// <summary>How many days back the forecasts look.</summary>
|
||||||
|
public int ForecastBasisDays { get; set; } = 30;
|
||||||
|
|
||||||
|
public Core.DynamicsAx.AxThresholds ToThresholds() => new()
|
||||||
|
{
|
||||||
|
BatchLateMinutes = Math.Max(1, BatchLateMinutes),
|
||||||
|
BatchNeverStartedHours = Math.Max(1, BatchNeverStartedHours),
|
||||||
|
BatchStuckHours = Math.Max(1, BatchStuckHours),
|
||||||
|
BatchLongRunningFactor = Math.Max(1.1, BatchLongRunningFactor),
|
||||||
|
BatchLongRunningMinimumMinutes = Math.Max(1, BatchLongRunningMinimumMinutes),
|
||||||
|
BatchHistoryMinimumRuns = Math.Clamp(BatchHistoryMinimumRuns, 1, Core.DynamicsAx.AxBatchHistory.RunsKept),
|
||||||
|
BatchFailedLookbackHours = Math.Max(1, BatchFailedLookbackHours),
|
||||||
|
NumberSequenceWarnPercent = Math.Clamp(NumberSequenceWarnPercent, 1, 100),
|
||||||
|
NumberSequenceCriticalPercent = Math.Clamp(NumberSequenceCriticalPercent, 1, 100),
|
||||||
|
ForecastWarnDays = Math.Max(1, ForecastWarnDays),
|
||||||
|
ForecastCriticalDays = Math.Max(1, ForecastCriticalDays),
|
||||||
|
RecId32WarnPercent = Math.Clamp(RecId32WarnPercent, 1, 100),
|
||||||
|
RecId32CriticalPercent = Math.Clamp(RecId32CriticalPercent, 1, 100),
|
||||||
|
RecId32ForecastDays = Math.Max(1, RecId32ForecastDays),
|
||||||
|
RecId64WarnPercent = Math.Clamp(RecId64WarnPercent, 1, 100),
|
||||||
|
BlockingSeconds = Math.Max(1, BlockingSeconds),
|
||||||
|
ForecastBasisDays = Math.Clamp(ForecastBasisDays, 2, 365),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,9 @@ namespace SQLInsight.Agent.Hub;
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static class SubmissionBuilder
|
public static class SubmissionBuilder
|
||||||
{
|
{
|
||||||
|
/// <summary>What the hub gets instead of an AX rule's reason (which can name databases, tables and logins).</summary>
|
||||||
|
public const string AxReasonKeptLocally = "Detaljer om Dynamics AX bliver i kundens netværk.";
|
||||||
|
|
||||||
public static readonly string AgentVersion =
|
public static readonly string AgentVersion =
|
||||||
typeof(SubmissionBuilder).Assembly.GetName().Version?.ToString() ?? "0.0.0";
|
typeof(SubmissionBuilder).Assembly.GetName().Version?.ToString() ?? "0.0.0";
|
||||||
|
|
||||||
|
|
@ -45,7 +48,14 @@ public static class SubmissionBuilder
|
||||||
Edition = info.Edition,
|
Edition = info.Edition,
|
||||||
ServerName = options.SendServerName ? info.ServerName : null,
|
ServerName = options.SendServerName ? info.ServerName : null,
|
||||||
},
|
},
|
||||||
Rules = results.Where(r => axRulesAllowed || !Core.Configuration.DynamicsAxFindings.IsDynamicsAxRule(r.RuleId)).Select(r => new RuleOutcomeDto { RuleId = r.RuleId, Outcome = r.Outcome.ToString(), Reason = r.Reason }).ToList(),
|
// An AX rule's outcome covers every AX database on the instance, and its reason can name databases, tables, logins
|
||||||
|
// and the GRANT text: only the outcome goes to the hub, with a fixed reason.
|
||||||
|
Rules = results.Where(r => axRulesAllowed || !Core.Configuration.DynamicsAxFindings.IsDynamicsAxRule(r.RuleId)).Select(r => new RuleOutcomeDto
|
||||||
|
{
|
||||||
|
RuleId = r.RuleId,
|
||||||
|
Outcome = r.Outcome.ToString(),
|
||||||
|
Reason = Core.Configuration.DynamicsAxFindings.IsDynamicsAxRule(r.RuleId) && r.Reason is not null ? AxReasonKeptLocally : r.Reason,
|
||||||
|
}).ToList(),
|
||||||
Findings = activeFindings.Where(f => Allowed(f.RuleId, f.Subject)).Select(f => new FindingDto
|
Findings = activeFindings.Where(f => Allowed(f.RuleId, f.Subject)).Select(f => new FindingDto
|
||||||
{
|
{
|
||||||
RuleId = f.RuleId,
|
RuleId = f.RuleId,
|
||||||
|
|
|
||||||
|
|
@ -267,6 +267,32 @@ public sealed partial class InstanceMonitor(
|
||||||
return await repository.GetSnapshotAsync<Core.Data.IndexSnapshot>(instanceId, collector.Name, cancellationToken).ConfigureAwait(false);
|
return await repository.GetSnapshotAsync<Core.Data.IndexSnapshot>(instanceId, collector.Name, cancellationToken).ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Dynamics AX: each batch job's own run history, the first-seen times of running jobs, the history watermark and the
|
||||||
|
// sequences' daily samples come back from the repository, so comparisons and forecasts continue after a restart.
|
||||||
|
if (collector is Collectors.DynamicsAx.AxBatchCollector)
|
||||||
|
{
|
||||||
|
var stored = await repository.GetSnapshotAsync<Core.DynamicsAx.AxBatchSnapshot>(instanceId, collector.Name, cancellationToken).ConfigureAwait(false)
|
||||||
|
?? new Core.DynamicsAx.AxBatchSnapshot { CapturedAt = now, Thresholds = new(), Databases = [] };
|
||||||
|
var history = await repository.GetAxBatchHistoryAsync(instanceId, now.AddDays(-90), cancellationToken).ConfigureAwait(false);
|
||||||
|
return stored with
|
||||||
|
{
|
||||||
|
History = history,
|
||||||
|
RecentRunKeys = history.ScopedRunKeys().ToHashSet(StringComparer.Ordinal),
|
||||||
|
FirstSeenExecuting = stored.Databases
|
||||||
|
.SelectMany(d => d.Jobs.Where(j => j.FirstSeenExecutingUtc is not null)
|
||||||
|
.Select(j => (Key: Core.DynamicsAx.AxBatchHistory.Key(d.Database, "#" + j.RecId.ToString(System.Globalization.CultureInfo.InvariantCulture)), At: j.FirstSeenExecutingUtc!.Value)))
|
||||||
|
.GroupBy(x => x.Key, StringComparer.OrdinalIgnoreCase)
|
||||||
|
.ToDictionary(g => g.Key, g => g.First().At, StringComparer.OrdinalIgnoreCase),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (collector is Collectors.DynamicsAx.AxSequenceCollector sequences)
|
||||||
|
{
|
||||||
|
var stored = await repository.GetSnapshotAsync<Core.DynamicsAx.AxSequenceSnapshot>(instanceId, collector.Name, cancellationToken).ConfigureAwait(false)
|
||||||
|
?? new Core.DynamicsAx.AxSequenceSnapshot { CapturedAt = now, Thresholds = new(), Databases = [] };
|
||||||
|
return stored with { History = await repository.GetAxSequenceHistoryAsync(instanceId, now.AddDays(-sequences.HistoryDays), cancellationToken).ConfigureAwait(false) };
|
||||||
|
}
|
||||||
|
|
||||||
if (collector is not Collectors.Concurrency.DeadlockCollector)
|
if (collector is not Collectors.Concurrency.DeadlockCollector)
|
||||||
{
|
{
|
||||||
return null;
|
return null;
|
||||||
|
|
|
||||||
|
|
@ -81,7 +81,7 @@ public sealed partial class MonitoringService(
|
||||||
private InstanceMonitor CreateMonitor(MonitoredInstance instance, AgentOptions settings, IReadOnlyDictionary<string, TimeSpan> intervals) =>
|
private InstanceMonitor CreateMonitor(MonitoredInstance instance, AgentOptions settings, IReadOnlyDictionary<string, TimeSpan> intervals) =>
|
||||||
new(instance, BuiltInCollectors.Create(customRules, new CollectorSettings(groupResolver, settings.Privileges.MaxSysadminAccounts, settings.TopSql.ActivityThresholdPercent, settings.ErrorLog.LoginFailureBurst,
|
new(instance, BuiltInCollectors.Create(customRules, new CollectorSettings(groupResolver, settings.Privileges.MaxSysadminAccounts, settings.TopSql.ActivityThresholdPercent, settings.ErrorLog.LoginFailureBurst,
|
||||||
settings.Activity.TransactionMinutes, settings.Activity.TransactionLogMegabytes, settings.Activity.RequestMinutes,
|
settings.Activity.TransactionMinutes, settings.Activity.TransactionLogMegabytes, settings.Activity.RequestMinutes,
|
||||||
settings.Tempdb.VersionStoreMegabytes, settings.Tempdb.SessionMegabytes)), intervals, connectionFactory, ruleEngine, repository,
|
settings.Tempdb.VersionStoreMegabytes, settings.Tempdb.SessionMegabytes, settings.DynamicsAx.ToThresholds())), intervals, connectionFactory, ruleEngine, repository,
|
||||||
loggerFactory.CreateLogger($"SQLInsight.Instance.{instance.Name}"), notifications, notificationOptions.Value.WebUrl, hubOutbox);
|
loggerFactory.CreateLogger($"SQLInsight.Instance.{instance.Name}"), notifications, notificationOptions.Value.WebUrl, hubOutbox);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,15 @@
|
||||||
"EventDays": 90,
|
"EventDays": 90,
|
||||||
"ResolvedFindingDays": 365
|
"ResolvedFindingDays": 365
|
||||||
},
|
},
|
||||||
|
"DynamicsAx": {
|
||||||
|
// Dynamics AX-tjek (docs/dynamics-ax.md). Kører kun i databaser genkendt som AX og valgt til overvågning.
|
||||||
|
"BatchLateMinutes": 30,
|
||||||
|
"BatchStuckHours": 6,
|
||||||
|
"BatchLongRunningFactor": 2.0,
|
||||||
|
"NumberSequenceWarnPercent": 80,
|
||||||
|
"ForecastWarnDays": 90,
|
||||||
|
"RecId32WarnPercent": 50
|
||||||
|
},
|
||||||
"Instances": [
|
"Instances": [
|
||||||
// Servere tilføjes nemmest i webbrugerfladen (Servere og databaser). Her kan de også stå, fx (se docs/installation.md):
|
// Servere tilføjes nemmest i webbrugerfladen (Servere og databaser). Her kan de også stå, fx (se docs/installation.md):
|
||||||
// { "Name": "prod-sql01", "Server": "sql01.domæne.local", "Authentication": "Windows", "Provider": "Auto", "Encryption": "Mandatory" }
|
// { "Name": "prod-sql01", "Server": "sql01.domæne.local", "Authentication": "Windows", "Provider": "Auto", "Encryption": "Mandatory" }
|
||||||
|
|
|
||||||
|
|
@ -81,7 +81,7 @@ public sealed class AxBatchCollector(AxDatabaseDetector detector, AxThresholds?
|
||||||
foreach (var row in historyRows)
|
foreach (var row in historyRows)
|
||||||
{
|
{
|
||||||
watermark = Math.Max(watermark ?? 0, row.RecId);
|
watermark = Math.Max(watermark ?? 0, row.RecId);
|
||||||
if (adapter.HistoryRun(row) is { } run && recentKeys.Add(Scoped(name, run.RunKey)))
|
if (adapter.HistoryRun(row) is { } run && recentKeys.Add(AxBatchHistory.ScopedRunKey(name, run.RunKey)))
|
||||||
{
|
{
|
||||||
newRuns.Add(run);
|
newRuns.Add(run);
|
||||||
}
|
}
|
||||||
|
|
@ -103,7 +103,7 @@ public sealed class AxBatchCollector(AxDatabaseDetector detector, AxThresholds?
|
||||||
{
|
{
|
||||||
firstSeen[seenKey] = seenBefore ?? context.Now.UtcDateTime;
|
firstSeen[seenKey] = seenBefore ?? context.Now.UtcDateTime;
|
||||||
}
|
}
|
||||||
else if (!fromHistoryTable && AxSchemaAdapter.ObservedRun(job, seenBefore) is { } run && recentKeys.Add(Scoped(name, run.RunKey)))
|
else if (!fromHistoryTable && AxSchemaAdapter.ObservedRun(job, seenBefore) is { } run && recentKeys.Add(AxBatchHistory.ScopedRunKey(name, run.RunKey)))
|
||||||
{
|
{
|
||||||
newRuns.Add(run);
|
newRuns.Add(run);
|
||||||
}
|
}
|
||||||
|
|
@ -138,8 +138,6 @@ public sealed class AxBatchCollector(AxDatabaseDetector detector, AxThresholds?
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string Scoped(string database, string runKey) => database + "\u001f" + runKey;
|
|
||||||
|
|
||||||
private static AxBatchJobRow Map(Row r) => new(
|
private static AxBatchJobRow Map(Row r) => new(
|
||||||
r.Long(0), r.NullableInt(1), r.NullableString(2), r.NullableString(3), r.NullableString(4), r.Int(5),
|
r.Long(0), r.NullableInt(1), r.NullableString(2), r.NullableString(3), r.NullableString(4), r.Int(5),
|
||||||
r.NullableDateTime(6), r.NullableInt(7), r.NullableDateTime(8), r.NullableInt(9), r.NullableDateTime(10), r.NullableInt(11), r.Int(12));
|
r.NullableDateTime(6), r.NullableInt(7), r.NullableDateTime(8), r.NullableInt(9), r.NullableDateTime(10), r.NullableInt(11), r.Int(12));
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,9 @@ public sealed class AxSequenceCollector(AxDatabaseDetector detector, AxThreshold
|
||||||
private readonly AxThresholds _thresholds = thresholds ?? new AxThresholds();
|
private readonly AxThresholds _thresholds = thresholds ?? new AxThresholds();
|
||||||
|
|
||||||
public override string Name => "dynamics-ax-sequences";
|
public override string Name => "dynamics-ax-sequences";
|
||||||
|
|
||||||
|
/// <summary>How many days of daily samples the forecasts use (and the agent reads back after a restart).</summary>
|
||||||
|
public int HistoryDays => _thresholds.ForecastBasisDays + 1;
|
||||||
public override TimeSpan DefaultInterval => TimeSpan.FromHours(1);
|
public override TimeSpan DefaultInterval => TimeSpan.FromHours(1);
|
||||||
public override IReadOnlyCollection<Capability> RequiredCapabilities => [Capability.DynamicManagementViews];
|
public override IReadOnlyCollection<Capability> RequiredCapabilities => [Capability.DynamicManagementViews];
|
||||||
|
|
||||||
|
|
@ -38,7 +41,7 @@ public sealed class AxSequenceCollector(AxDatabaseDetector detector, AxThreshold
|
||||||
{
|
{
|
||||||
var historyKey = AxSequenceSnapshot.HistoryKey(name, kind, key);
|
var historyKey = AxSequenceSnapshot.HistoryKey(name, kind, key);
|
||||||
var samples = (previous?.History.TryGetValue(historyKey, out var stored) == true ? stored : [])
|
var samples = (previous?.History.TryGetValue(historyKey, out var stored) == true ? stored : [])
|
||||||
.Where(s => s.SampledUtc >= now.AddDays(-(_thresholds.ForecastBasisDays + 1)) && s.SampledUtc.Date != now.Date)
|
.Where(s => s.SampledUtc >= now.AddDays(-HistoryDays) && s.SampledUtc.Date != now.Date)
|
||||||
.Append(new AxSample(now, value))
|
.Append(new AxSample(now, value))
|
||||||
.ToList();
|
.ToList();
|
||||||
history[historyKey] = samples;
|
history[historyKey] = samples;
|
||||||
|
|
@ -76,7 +79,7 @@ public sealed class AxSequenceCollector(AxDatabaseDetector detector, AxThreshold
|
||||||
{
|
{
|
||||||
if (key.StartsWith(name + "\u001f", StringComparison.Ordinal) && !history.ContainsKey(key))
|
if (key.StartsWith(name + "\u001f", StringComparison.Ordinal) && !history.ContainsKey(key))
|
||||||
{
|
{
|
||||||
history[key] = samples.Where(s => s.SampledUtc >= now.AddDays(-(_thresholds.ForecastBasisDays + 1))).ToList();
|
history[key] = samples.Where(s => s.SampledUtc >= now.AddDays(-HistoryDays)).ToList();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -154,6 +154,13 @@ public sealed class AxBatchHistory
|
||||||
|
|
||||||
public static string Key(string database, string jobKey) => database + "\u001f" + jobKey;
|
public static string Key(string database, string jobKey) => database + "\u001f" + jobKey;
|
||||||
|
|
||||||
|
/// <summary>A run key made unique across the instance's databases (<see cref="AxBatchSnapshot.RecentRunKeys"/>).</summary>
|
||||||
|
public static string ScopedRunKey(string database, string runKey) => database + "\u001f" + runKey;
|
||||||
|
|
||||||
|
/// <summary>The scoped run keys of every run kept.</summary>
|
||||||
|
public IEnumerable<string> ScopedRunKeys() =>
|
||||||
|
_runs.SelectMany(kv => kv.Value.Select(r => ScopedRunKey(kv.Key[..kv.Key.IndexOf('\u001f', StringComparison.Ordinal)], r.RunKey)));
|
||||||
|
|
||||||
public void Add(string database, AxBatchRun run)
|
public void Add(string database, AxBatchRun run)
|
||||||
{
|
{
|
||||||
ArgumentNullException.ThrowIfNull(run);
|
ArgumentNullException.ThrowIfNull(run);
|
||||||
|
|
|
||||||
158
src/SQLInsight.Repository/InsightRepository.DynamicsAx.cs
Normal file
158
src/SQLInsight.Repository/InsightRepository.DynamicsAx.cs
Normal file
|
|
@ -0,0 +1,158 @@
|
||||||
|
using System.Text.Json;
|
||||||
|
using Dapper;
|
||||||
|
using Microsoft.Data.SqlClient;
|
||||||
|
using SQLInsight.Core.DynamicsAx;
|
||||||
|
|
||||||
|
namespace SQLInsight.Repository;
|
||||||
|
|
||||||
|
/// <summary>A stored completed AX batch run (insight.AxBatchRuns).</summary>
|
||||||
|
public sealed record StoredAxBatchRun(string InstanceName, string DatabaseName, string JobKey, string RunKey, string? Caption, string? Company,
|
||||||
|
DateTime StartedAt, DateTime EndedAt, AxBatchStatus Status)
|
||||||
|
{
|
||||||
|
public TimeSpan Duration => EndedAt - StartedAt;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>One stored daily sample of an AX sequence (insight.AxSequenceSamples).</summary>
|
||||||
|
public sealed record StoredAxSequenceSample(string DatabaseName, char Kind, string SequenceKey, DateTime SampledAt, long Value);
|
||||||
|
|
||||||
|
public sealed partial class InsightRepository
|
||||||
|
{
|
||||||
|
/// <summary>A job's runs read back after a restart, so it is compared with its own history at once.</summary>
|
||||||
|
public const int AxBatchRunsPerJob = AxBatchHistory.RunsKept;
|
||||||
|
|
||||||
|
private static Task<int> SaveAxBatchRunsAsync(SqlConnection connection, int instanceId, AxBatchSnapshot snapshot)
|
||||||
|
{
|
||||||
|
var runs = snapshot.Databases.SelectMany(d => d.NewRuns.Select(r => new
|
||||||
|
{
|
||||||
|
Database = d.Database,
|
||||||
|
r.RunKey,
|
||||||
|
JobKey = Truncate(r.JobKey, 200),
|
||||||
|
Caption = r.Caption is null ? null : Truncate(r.Caption, 200),
|
||||||
|
Company = r.Company is null ? null : Truncate(r.Company, 10),
|
||||||
|
StartedAt = Iso(r.StartedUtc),
|
||||||
|
EndedAt = Iso(r.EndedUtc),
|
||||||
|
Status = (byte)r.Status,
|
||||||
|
})).Where(r => r.RunKey.Length <= 100).ToList();
|
||||||
|
return runs.Count == 0
|
||||||
|
? Task.FromResult(0)
|
||||||
|
: connection.ExecuteAsync("""
|
||||||
|
INSERT insight.AxBatchRuns (InstanceId, DatabaseName, RunKey, JobKey, Caption, Company, StartedAt, EndedAt, Status)
|
||||||
|
SELECT @instanceId, j.DatabaseName, j.RunKey, j.JobKey, j.Caption, j.Company, j.StartedAt, j.EndedAt, j.Status
|
||||||
|
FROM (SELECT DISTINCT * FROM OPENJSON(@json) WITH (DatabaseName nvarchar(128) '$.Database', RunKey nvarchar(100), JobKey nvarchar(200), Caption nvarchar(200),
|
||||||
|
Company nvarchar(10), StartedAt datetime2(0), EndedAt datetime2(0), Status tinyint)) j
|
||||||
|
WHERE NOT EXISTS (SELECT 1 FROM insight.AxBatchRuns r WHERE r.InstanceId = @instanceId AND r.DatabaseName = j.DatabaseName AND r.RunKey = j.RunKey);
|
||||||
|
""", new { instanceId, json = JsonSerializer.Serialize(runs, Json) });
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Today's sample of every sequence read (the latest of the day wins).</summary>
|
||||||
|
private static Task<int> SaveAxSequenceSamplesAsync(SqlConnection connection, int instanceId, AxSequenceSnapshot snapshot)
|
||||||
|
{
|
||||||
|
var at = Utc(snapshot.CapturedAt);
|
||||||
|
var samples = snapshot.Databases.SelectMany(d =>
|
||||||
|
d.NumberSequences.Select(n => (d.Database, Kind: AxSequenceSnapshot.NumberSequenceKind, n.Key, Value: n.Next))
|
||||||
|
.Concat(d.RecIds.Select(r => (d.Database, Kind: AxSequenceSnapshot.RecIdKind, r.Key, Value: r.Next))))
|
||||||
|
.Where(s => s.Key.Length <= 100)
|
||||||
|
.Select(s => new { s.Database, Kind = s.Kind.ToString(), s.Key, s.Value })
|
||||||
|
.ToList();
|
||||||
|
return samples.Count == 0
|
||||||
|
? Task.FromResult(0)
|
||||||
|
: connection.ExecuteAsync("""
|
||||||
|
MERGE insight.AxSequenceSamples AS t
|
||||||
|
USING (SELECT DISTINCT DatabaseName, Kind, SequenceKey, Value FROM OPENJSON(@json)
|
||||||
|
WITH (DatabaseName nvarchar(128) '$.Database', Kind char(1), SequenceKey nvarchar(100) '$.Key', Value bigint)) AS s
|
||||||
|
ON t.InstanceId = @instanceId AND t.DatabaseName = s.DatabaseName AND t.Kind = s.Kind AND t.SequenceKey = s.SequenceKey AND t.[Day] = @day
|
||||||
|
WHEN MATCHED THEN UPDATE SET Value = s.Value, SampledAt = @at
|
||||||
|
WHEN NOT MATCHED THEN INSERT (InstanceId, DatabaseName, Kind, SequenceKey, [Day], SampledAt, Value)
|
||||||
|
VALUES (@instanceId, s.DatabaseName, s.Kind, s.SequenceKey, @day, @at, s.Value);
|
||||||
|
""", new { instanceId, at, day = at.Date, json = JsonSerializer.Serialize(samples, Json) });
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string Iso(DateTime utc) => utc.ToString("yyyy-MM-ddTHH:mm:ss", System.Globalization.CultureInfo.InvariantCulture);
|
||||||
|
|
||||||
|
private static string Truncate(string value, int length) => value.Length <= length ? value : value[..length];
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The latest <see cref="AxBatchRunsPerJob"/> runs of every job since <paramref name="since"/>: the agent's batch
|
||||||
|
/// collector starts from them after a restart.
|
||||||
|
/// </summary>
|
||||||
|
public async Task<AxBatchHistory> GetAxBatchHistoryAsync(int instanceId, DateTimeOffset since, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
var rows = await connection.QueryAsync<AxBatchRunRow>("""
|
||||||
|
SELECT DatabaseName, JobKey, RunKey, Caption, Company, StartedAt, EndedAt, Status
|
||||||
|
FROM (SELECT r.*, ROW_NUMBER() OVER (PARTITION BY r.DatabaseName, r.JobKey ORDER BY r.EndedAt DESC) AS n
|
||||||
|
FROM insight.AxBatchRuns r WHERE r.InstanceId = @instanceId AND r.EndedAt >= @since) x
|
||||||
|
WHERE n <= @perJob;
|
||||||
|
""", new { instanceId, since = Utc(since), perJob = AxBatchRunsPerJob }).ConfigureAwait(false);
|
||||||
|
var history = new AxBatchHistory();
|
||||||
|
foreach (var r in rows)
|
||||||
|
{
|
||||||
|
history.Add(r.DatabaseName, new AxBatchRun(r.JobKey, r.RunKey, r.Caption, r.Company, AsUtc(r.StartedAt), AsUtc(r.EndedAt), (AxBatchStatus)r.Status));
|
||||||
|
}
|
||||||
|
|
||||||
|
return history;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Completed AX batch runs, newest first; for one instance, database and job when given.</summary>
|
||||||
|
public async Task<IReadOnlyList<StoredAxBatchRun>> GetAxBatchRunsAsync(int? instanceId, string? database, string? jobKey, DateTimeOffset from, DateTimeOffset to, int top = 500,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
var rows = await connection.QueryAsync<AxBatchRunRow>("""
|
||||||
|
SELECT TOP (@top) i.Name AS InstanceName, r.DatabaseName, r.JobKey, r.RunKey, r.Caption, r.Company, r.StartedAt, r.EndedAt, r.Status
|
||||||
|
FROM insight.AxBatchRuns r JOIN insight.Instances i ON i.InstanceId = r.InstanceId
|
||||||
|
WHERE (@instanceId IS NULL OR r.InstanceId = @instanceId) AND (@database IS NULL OR r.DatabaseName = @database) AND (@jobKey IS NULL OR r.JobKey = @jobKey)
|
||||||
|
AND r.EndedAt >= @from AND r.EndedAt <= @to
|
||||||
|
ORDER BY r.EndedAt DESC;
|
||||||
|
""", new { instanceId, database, jobKey, from = Utc(from), to = Utc(to), top }).ConfigureAwait(false);
|
||||||
|
return rows.Select(r => new StoredAxBatchRun(r.InstanceName ?? string.Empty, r.DatabaseName, r.JobKey, r.RunKey, r.Caption, r.Company, AsUtc(r.StartedAt), AsUtc(r.EndedAt),
|
||||||
|
(AxBatchStatus)r.Status)).ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>The daily samples since <paramref name="since"/>, keyed as <see cref="AxSequenceSnapshot.HistoryKey"/>: the agent's forecasts start from them after a restart.</summary>
|
||||||
|
public async Task<IReadOnlyDictionary<string, IReadOnlyList<AxSample>>> GetAxSequenceHistoryAsync(int instanceId, DateTimeOffset since, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var samples = await GetAxSequenceSamplesAsync(instanceId, null, null, null, since, cancellationToken).ConfigureAwait(false);
|
||||||
|
return samples.GroupBy(s => AxSequenceSnapshot.HistoryKey(s.DatabaseName, s.Kind, s.SequenceKey), StringComparer.Ordinal)
|
||||||
|
.ToDictionary(g => g.Key, g => (IReadOnlyList<AxSample>)g.OrderBy(s => s.SampledAt).Select(s => new AxSample(s.SampledAt, s.Value)).ToList(), StringComparer.Ordinal);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Daily samples of AX sequences, oldest first; for one database, kind and key when given.</summary>
|
||||||
|
public async Task<IReadOnlyList<StoredAxSequenceSample>> GetAxSequenceSamplesAsync(int instanceId, string? database, char? kind, string? key, DateTimeOffset since,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
var rows = await connection.QueryAsync<AxSampleRow>("""
|
||||||
|
SELECT DatabaseName, Kind, SequenceKey, SampledAt, Value
|
||||||
|
FROM insight.AxSequenceSamples
|
||||||
|
WHERE InstanceId = @instanceId AND [Day] >= @since
|
||||||
|
AND (@database IS NULL OR DatabaseName = @database) AND (@kind IS NULL OR Kind = @kind) AND (@key IS NULL OR SequenceKey = @key)
|
||||||
|
ORDER BY DatabaseName, Kind, SequenceKey, [Day];
|
||||||
|
""", new { instanceId, database, kind = kind?.ToString(), key, since = Utc(since).Date }).ConfigureAwait(false);
|
||||||
|
return rows.Select(r => new StoredAxSequenceSample(r.DatabaseName, r.Kind[0], r.SequenceKey, AsUtc(r.SampledAt), r.Value)).ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static DateTime AsUtc(DateTime value) => DateTime.SpecifyKind(value, DateTimeKind.Utc);
|
||||||
|
|
||||||
|
private sealed record AxBatchRunRow
|
||||||
|
{
|
||||||
|
public string? InstanceName { get; init; }
|
||||||
|
public string DatabaseName { get; init; } = string.Empty;
|
||||||
|
public string JobKey { get; init; } = string.Empty;
|
||||||
|
public string RunKey { get; init; } = string.Empty;
|
||||||
|
public string? Caption { get; init; }
|
||||||
|
public string? Company { get; init; }
|
||||||
|
public DateTime StartedAt { get; init; }
|
||||||
|
public DateTime EndedAt { get; init; }
|
||||||
|
public byte Status { get; init; }
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed record AxSampleRow
|
||||||
|
{
|
||||||
|
public string DatabaseName { get; init; } = string.Empty;
|
||||||
|
public string Kind { get; init; } = "N";
|
||||||
|
public string SequenceKey { get; init; } = string.Empty;
|
||||||
|
public DateTime SampledAt { get; init; }
|
||||||
|
public long Value { get; init; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -7,10 +7,10 @@ public sealed class RetentionOptions
|
||||||
/// <summary>Raw samples (5-minute waits, per-minute CPU …).</summary>
|
/// <summary>Raw samples (5-minute waits, per-minute CPU …).</summary>
|
||||||
public int RawDays { get; set; } = 14;
|
public int RawDays { get; set; } = 14;
|
||||||
|
|
||||||
/// <summary>Hourly aggregates, for long trends.</summary>
|
/// <summary>Hourly aggregates, for long trends, and the daily samples of Dynamics AX number sequences and RecIds.</summary>
|
||||||
public int HourlyDays { get; set; } = 400;
|
public int HourlyDays { get; set; } = 400;
|
||||||
|
|
||||||
/// <summary>Blocking events and deadlock graphs.</summary>
|
/// <summary>Blocking events, deadlock graphs and completed Dynamics AX batch runs.</summary>
|
||||||
public int EventDays { get; set; } = 90;
|
public int EventDays { get; set; } = 90;
|
||||||
|
|
||||||
/// <summary>Resolved findings.</summary>
|
/// <summary>Resolved findings.</summary>
|
||||||
|
|
@ -92,6 +92,8 @@ public sealed partial class InsightRepository
|
||||||
await DeleteInBatchesAsync(connection, "insight.IndexUsageDaily", "[Day] < @cutoff", hourly).ConfigureAwait(false);
|
await DeleteInBatchesAsync(connection, "insight.IndexUsageDaily", "[Day] < @cutoff", hourly).ConfigureAwait(false);
|
||||||
await DeleteInBatchesAsync(connection, "insight.BlockingEvents", "SampledAt < @cutoff", events).ConfigureAwait(false);
|
await DeleteInBatchesAsync(connection, "insight.BlockingEvents", "SampledAt < @cutoff", events).ConfigureAwait(false);
|
||||||
await DeleteInBatchesAsync(connection, "insight.Deadlocks", "OccurredAt < @cutoff", events).ConfigureAwait(false);
|
await DeleteInBatchesAsync(connection, "insight.Deadlocks", "OccurredAt < @cutoff", events).ConfigureAwait(false);
|
||||||
|
await DeleteInBatchesAsync(connection, "insight.AxBatchRuns", "EndedAt < @cutoff", events).ConfigureAwait(false);
|
||||||
|
await DeleteInBatchesAsync(connection, "insight.AxSequenceSamples", "[Day] < @cutoff", hourly).ConfigureAwait(false);
|
||||||
await DeleteInBatchesAsync(connection, "insight.ErrorLogEvents", "LastAt < @cutoff", events).ConfigureAwait(false);
|
await DeleteInBatchesAsync(connection, "insight.ErrorLogEvents", "LastAt < @cutoff", events).ConfigureAwait(false);
|
||||||
await DeleteInBatchesAsync(connection, "insight.Findings", "ResolvedAt < @cutoff", resolved).ConfigureAwait(false);
|
await DeleteInBatchesAsync(connection, "insight.Findings", "ResolvedAt < @cutoff", resolved).ConfigureAwait(false);
|
||||||
await DeleteInBatchesAsync(connection, "insight.ConfigurationChanges", "DetectedAt < @cutoff", resolved).ConfigureAwait(false);
|
await DeleteInBatchesAsync(connection, "insight.ConfigurationChanges", "DetectedAt < @cutoff", resolved).ConfigureAwait(false);
|
||||||
|
|
|
||||||
|
|
@ -289,6 +289,14 @@ public sealed partial class InsightRepository(string connectionString)
|
||||||
await SaveLatePrivilegeAuthorsAsync(connection, instanceId, privileges.RecentChanges).ConfigureAwait(false);
|
await SaveLatePrivilegeAuthorsAsync(connection, instanceId, privileges.RecentChanges).ConfigureAwait(false);
|
||||||
await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false);
|
await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false);
|
||||||
return;
|
return;
|
||||||
|
case Core.DynamicsAx.AxBatchSnapshot axBatch:
|
||||||
|
await SaveAxBatchRunsAsync(connection, instanceId, axBatch).ConfigureAwait(false);
|
||||||
|
await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false);
|
||||||
|
return;
|
||||||
|
case Core.DynamicsAx.AxSequenceSnapshot axSequences:
|
||||||
|
await SaveAxSequenceSamplesAsync(connection, instanceId, axSequences).ConfigureAwait(false);
|
||||||
|
await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false);
|
||||||
|
return;
|
||||||
default:
|
default:
|
||||||
await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false);
|
await SaveSnapshotJsonAsync(connection, instanceId, kind, data).ConfigureAwait(false);
|
||||||
return;
|
return;
|
||||||
|
|
|
||||||
37
src/SQLInsight.Repository/Migrations/0017_dynamics_ax.sql
Normal file
37
src/SQLInsight.Repository/Migrations/0017_dynamics_ax.sql
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
/* 1.4: Dynamics AX. SQLInsight's own history of what it reads from AX, kept in the repository and never in the customer's
|
||||||
|
AX database (SQLInsight only ever SELECTs from AX, and only with the optional read access).
|
||||||
|
|
||||||
|
AxBatchRuns completed batch job runs: from BATCHJOBHISTORY (AX 2009 and later) or seen by SQLInsight itself
|
||||||
|
(AX 3.0/4.0, which keep no history). A job's own runs are what "longer than normal" is measured
|
||||||
|
against. RunKey is unique per database, so a run read twice is stored once. Kept EventDays (90).
|
||||||
|
AxSequenceSamples one sample a day of each number sequence's NEXTREC (Kind 'N') and each RecId sequence's NEXTVAL
|
||||||
|
(Kind 'R', SYSTEMSEQUENCES), the latest of the day: the forecasts of when they run out are made from
|
||||||
|
it. Kept HourlyDays (400). */
|
||||||
|
CREATE TABLE insight.AxBatchRuns
|
||||||
|
(
|
||||||
|
InstanceId int NOT NULL,
|
||||||
|
DatabaseName nvarchar(128) NOT NULL,
|
||||||
|
RunKey nvarchar(100) NOT NULL,
|
||||||
|
JobKey nvarchar(200) NOT NULL,
|
||||||
|
Caption nvarchar(200) NULL,
|
||||||
|
Company nvarchar(10) NULL,
|
||||||
|
StartedAt datetime2(0) NOT NULL,
|
||||||
|
EndedAt datetime2(0) NOT NULL,
|
||||||
|
Status tinyint NOT NULL,
|
||||||
|
CONSTRAINT PK_AxBatchRuns PRIMARY KEY (InstanceId, DatabaseName, RunKey)
|
||||||
|
);
|
||||||
|
CREATE INDEX IX_AxBatchRuns_Job ON insight.AxBatchRuns (InstanceId, DatabaseName, JobKey, EndedAt);
|
||||||
|
CREATE INDEX IX_AxBatchRuns_EndedAt ON insight.AxBatchRuns (EndedAt);
|
||||||
|
|
||||||
|
CREATE TABLE insight.AxSequenceSamples
|
||||||
|
(
|
||||||
|
InstanceId int NOT NULL,
|
||||||
|
DatabaseName nvarchar(128) NOT NULL,
|
||||||
|
Kind char(1) NOT NULL CONSTRAINT CK_AxSequenceSamples_Kind CHECK (Kind IN ('N', 'R')),
|
||||||
|
SequenceKey nvarchar(100) NOT NULL,
|
||||||
|
[Day] date NOT NULL,
|
||||||
|
SampledAt datetime2(0) NOT NULL,
|
||||||
|
Value bigint NOT NULL,
|
||||||
|
CONSTRAINT PK_AxSequenceSamples PRIMARY KEY (InstanceId, DatabaseName, Kind, SequenceKey, [Day])
|
||||||
|
);
|
||||||
|
CREATE INDEX IX_AxSequenceSamples_Day ON insight.AxSequenceSamples ([Day]);
|
||||||
|
|
@ -146,4 +146,27 @@ public sealed class DynamicsAxHubTests
|
||||||
Assert.Equal(["database: AX30", "database: AX30"], submission.Findings.Select(f => f.Subject));
|
Assert.Equal(["database: AX30", "database: AX30"], submission.Findings.Select(f => f.Subject));
|
||||||
Assert.Contains(submission.Rules, r => r.RuleId == "COM-AX-001");
|
Assert.Contains(submission.Rules, r => r.RuleId == "COM-AX-001");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void The_built_in_ax_rules_follow_the_same_opt_in_and_keep_their_reasons_local()
|
||||||
|
{
|
||||||
|
var selection = new DatabaseSelection(true, ["AX30", "AXTEST"], [], ["AX30"]);
|
||||||
|
RuleResult[] results =
|
||||||
|
[
|
||||||
|
new() { RuleId = "SQLI-AX-001", RuleTitle = "Batchjob fejlet", Outcome = RuleOutcome.Findings },
|
||||||
|
new() { RuleId = "SQLI-AX-005", RuleTitle = "Nummerserie", Outcome = RuleOutcome.NotMeasurable, Reason = "Ikke målbar: GRANT SELECT ON [bmssa].[NUMBERSEQUENCETABLE] TO [svc]" },
|
||||||
|
];
|
||||||
|
StoredFinding[] findings =
|
||||||
|
[
|
||||||
|
Finding("SQLI-AX-001", "database: AX30 · batch: 1234|dat|NAT"), Finding("SQLI-AX-001", "database: AXTEST · batch: 99"),
|
||||||
|
];
|
||||||
|
|
||||||
|
var none = SubmissionBuilder.Build("ax", Info, results, findings, new HubOptions(), Now);
|
||||||
|
var allowed = SubmissionBuilder.Build("ax", Info, results, findings, new HubOptions(), Now, selection);
|
||||||
|
|
||||||
|
Assert.Empty(none.Findings);
|
||||||
|
Assert.Empty(none.Rules);
|
||||||
|
Assert.Equal(["database: AX30 · batch: 1234|dat|NAT"], allowed.Findings.Select(f => f.Subject));
|
||||||
|
Assert.Equal(SubmissionBuilder.AxReasonKeptLocally, allowed.Rules.Single(r => r.RuleId == "SQLI-AX-005").Reason);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue