From 6b945295d0f9f6e6b449ba3667e1ca162fdce5b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Eggert=20Lundsteen=20Olsen?= Date: Fri, 2 Oct 2026 13:00:31 +0200 Subject: [PATCH] Dynamics AX collectors: shared detector, batch jobs, number sequences and RecId dynamics-ax (hourly) lists the monitored AX databases with version, schema, companies and read access; discovery and the AX community rules (now marked dynamicsAx) use the same detection. dynamics-ax-batch (5 min) reads BATCH (3.0/4.0) or BATCHJOB/BATCHJOBHISTORY (2009+), keeps each job's own run history; dynamics-ax-sequences (hourly) reads NUMBERSEQUENCETABLE and SYSTEMSEQUENCES and forecasts from SQLInsight's own daily samples. All reads run through [db].sys.sp_executesql under read committed with a 2 s lock timeout (snapshot when allowed), bounded with TOP, never NOLOCK, never written. Blocking now captures CONTEXT_INFO for the AX user mapping. Co-Authored-By: Claude Opus 5.5 (1M context) --- rules/community/dynamics-ax.json | 31 ++-- .../BuiltInCollectors.cs | 12 +- .../Concurrency/BlockingCollector.cs | 8 +- .../Custom/CustomQueryCollector.cs | 20 ++- src/SQLInsight.Collectors/Db.cs | 50 +++++- .../Discovery/DatabaseDiscovery.cs | 38 ++--- .../DynamicsAx/AxBatchCollector.cs | 149 ++++++++++++++++++ .../DynamicsAx/AxDatabaseDetector.cs | 140 ++++++++++++++++ .../DynamicsAx/AxSequenceCollector.cs | 93 +++++++++++ .../DynamicsAx/DynamicsAxCollector.cs | 19 +++ .../Configuration/AgentRequests.cs | 4 +- .../CustomRules/CustomRuleDefinition.cs | 15 +- .../Custom/DeclarativeRule.cs | 5 + .../DiscoveryTests.cs | 17 +- .../DynamicsAxSqlTests.cs | 56 +++++++ .../SQLInsight.Collectors.Tests.csproj | 4 + 16 files changed, 600 insertions(+), 61 deletions(-) create mode 100644 src/SQLInsight.Collectors/DynamicsAx/AxBatchCollector.cs create mode 100644 src/SQLInsight.Collectors/DynamicsAx/AxDatabaseDetector.cs create mode 100644 src/SQLInsight.Collectors/DynamicsAx/AxSequenceCollector.cs create mode 100644 src/SQLInsight.Collectors/DynamicsAx/DynamicsAxCollector.cs create mode 100644 tests/SQLInsight.Collectors.Tests/DynamicsAxSqlTests.cs diff --git a/rules/community/dynamics-ax.json b/rules/community/dynamics-ax.json index b4e97cf..b3eac4d 100644 --- a/rules/community/dynamics-ax.json +++ b/rules/community/dynamics-ax.json @@ -1,10 +1,10 @@ // Microsoft Dynamics AX (Axapta), with emphasis on AX 3.0. // -// The agent has no data access (grant-least-privilege.sql), so these rules only read metadata and DMVs. They run in -// every user database, so an AX database is recognised without naming its tables directly (that would fail to compile -// elsewhere): the AX system tables SQLDICTIONARY, DATAAREA and SYSCONFIG in sys.tables, in ANY schema: AX 3.0 owns its -// tables as the database user bmssa (bmssa.SQLDICTIONARY), later versions use dbo. AX 3.0 is told apart from AX 4.0 -// and later by RECID being 32-bit (int) instead of 64-bit (bigint). +// The agent has no data access (grant-least-privilege.sql), so these rules only read metadata and DMVs. They are marked +// "dynamicsAx": the agent runs them only in databases its central detection (Core/DynamicsAx/AxDetection.cs) recognises as +// Dynamics AX (SQLDICTIONARY and DATAAREA in one schema: bmssa for AX 3.0, dbo later) and that are chosen for monitoring +// in Servere og databaser, and their findings only reach SQLInsight Hub for databases where AX findings may be sent. AX 3.0 is +// told apart from AX 4.0 and later by RECID being 32-bit (int) instead of 64-bit (bigint). // // Everything is SQL Server 2005 syntax (no CONCAT, IIF or STRING_AGG): AX 3.0 typically runs on 2000 or 2005. [ @@ -13,7 +13,8 @@ "title": "Dynamics AX 3.0 er uden support fra Microsoft", "severity": "Medium", "scope": "Database", - "query": "SELECT CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(128)) AS [sql_version] WHERE EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SQLDICTIONARY') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'DATAAREA') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SYSCONFIG') AND COLUMNPROPERTY((SELECT TOP 1 object_id FROM sys.tables WHERE name = N'SQLDICTIONARY'), N'RECID', 'Precision') = 10", + "dynamicsAx": true, + "query": "SELECT CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(128)) AS [sql_version] WHERE COLUMNPROPERTY((SELECT TOP 1 object_id FROM sys.tables WHERE name = N'SQLDICTIONARY'), N'RECID', 'Precision') = 10", "subject": "database: {database}", "findingTitle": "{database} er en Dynamics AX 3.0-database, og AX 3.0 er uden support", "explanation": "Microsoft Dynamics AX 3.0 er for længst ude af Microsofts support, også den udvidede. Der kommer ingen sikkerhedsopdateringer eller rettelser til AX-kernen, klienten eller AOS, og nyere Windows- og SQL Server-versioner er aldrig testet med AX 3.0. AX 3.0 bruger desuden 32-bit RecId (int), som kan løbe tør på store installationer.", @@ -26,7 +27,8 @@ "title": "Dynamics AX 3.0 kører på en SQL Server-version, der ikke var understøttet til AX 3.0", "severity": "Medium", "scope": "Database", - "query": "SELECT CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(128)) AS [sql_version], d.compatibility_level AS [compatibility_level] FROM sys.databases d WHERE d.database_id = DB_ID() AND CAST(PARSENAME(CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(128)), 4) AS int) > 9 AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SQLDICTIONARY') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'DATAAREA') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SYSCONFIG') AND COLUMNPROPERTY((SELECT TOP 1 object_id FROM sys.tables WHERE name = N'SQLDICTIONARY'), N'RECID', 'Precision') = 10", + "dynamicsAx": true, + "query": "SELECT CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(128)) AS [sql_version], d.compatibility_level AS [compatibility_level] FROM sys.databases d WHERE d.database_id = DB_ID() AND CAST(PARSENAME(CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(128)), 4) AS int) > 9 AND COLUMNPROPERTY((SELECT TOP 1 object_id FROM sys.tables WHERE name = N'SQLDICTIONARY'), N'RECID', 'Precision') = 10", "subject": "database: {database}", "findingTitle": "AX 3.0-databasen {database} ligger på SQL Server {sql_version}", "explanation": "AX 3.0 blev kun frigivet til SQL Server 2000 og 2005. På nyere versioner kan den virke, men optimizeren, låsning og udfasede funktioner opfører sig anderledes end det, AX-kernen blev bygget til, og hverken Microsoft eller en AX-partner kan give support. Fejl viser sig ofte først under belastning, fx ved månedsafslutning. Databasens compatibility level er {compatibility_level}.", @@ -39,7 +41,8 @@ "title": "Dynamics AX-database på en instans, hvor MAXDOP ikke er 1 (SQL Server 2005-2014)", "severity": "Medium", "scope": "Database", - "query": "SELECT CAST(c.value_in_use AS int) AS [maxdop] FROM sys.configurations c WHERE c.name = 'max degree of parallelism' AND CAST(c.value_in_use AS int) <> 1 AND CAST(PARSENAME(CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(128)), 4) AS int) < 13 AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SQLDICTIONARY') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'DATAAREA') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SYSCONFIG')", + "dynamicsAx": true, + "query": "SELECT CAST(c.value_in_use AS int) AS [maxdop] FROM sys.configurations c WHERE c.name = 'max degree of parallelism' AND CAST(c.value_in_use AS int) <> 1 AND CAST(PARSENAME(CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(128)), 4) AS int) < 13", "subject": "database: {database}", "findingTitle": "MAXDOP er {maxdop} på instansen med Dynamics AX-databasen {database}", "explanation": "Dynamics AX sender mange små, hyppige forespørgsler og cursorer fra AOS. Når de får parallelle planer, bruger de flere tråde end nødvendigt, venter på hinanden (CXPACKET) og holder låse længere, så brugerne oplever langsomme formularer og blokering. Microsofts vejledning for Dynamics AX er derfor max degree of parallelism = 1 på instansen.", @@ -55,8 +58,9 @@ "title": "Dynamics AX-database, hvor MAXDOP ikke er 1 (SQL Server 2016 og nyere)", "severity": "Medium", "scope": "Database", + "dynamicsAx": true, "minimumVersion": "13.0", - "query": "SELECT CAST(c.value_in_use AS int) AS [server_maxdop], CAST(s.value AS int) AS [database_maxdop] FROM sys.configurations c CROSS JOIN sys.database_scoped_configurations s WHERE c.name = 'max degree of parallelism' AND s.name = 'MAXDOP' AND (CASE WHEN CAST(s.value AS int) = 0 THEN CAST(c.value_in_use AS int) ELSE CAST(s.value AS int) END) <> 1 AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SQLDICTIONARY') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'DATAAREA') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SYSCONFIG')", + "query": "SELECT CAST(c.value_in_use AS int) AS [server_maxdop], CAST(s.value AS int) AS [database_maxdop] FROM sys.configurations c CROSS JOIN sys.database_scoped_configurations s WHERE c.name = 'max degree of parallelism' AND s.name = 'MAXDOP' AND (CASE WHEN CAST(s.value AS int) = 0 THEN CAST(c.value_in_use AS int) ELSE CAST(s.value AS int) END) <> 1", "subject": "database: {database}", "findingTitle": "Dynamics AX-databasen {database} kører med MAXDOP større end 1", "explanation": "Dynamics AX sender mange små, hyppige forespørgsler og cursorer fra AOS. Når de får parallelle planer, bruger de flere tråde end nødvendigt, venter på hinanden (CXPACKET) og holder låse længere. Microsofts vejledning for Dynamics AX er MAXDOP 1. Instansen har {server_maxdop}, og databasen har {database_maxdop} (0 = instansens værdi gælder).", @@ -72,7 +76,8 @@ "title": "Dynamics AX-tabeller uden unikt RecId-indeks", "severity": "Medium", "scope": "Database", - "query": "SELECT COUNT(*) AS [tables], LEFT(STUFF((SELECT TOP 10 N', ' + t2.name FROM sys.tables t2 WHERE COLUMNPROPERTY(t2.object_id, N'RECID', 'ColumnId') IS NOT NULL AND NOT EXISTS (SELECT 1 FROM sys.indexes i JOIN sys.index_columns ic ON ic.object_id = i.object_id AND ic.index_id = i.index_id AND ic.is_included_column = 0 JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id WHERE i.object_id = t2.object_id AND i.is_unique = 1 AND c.name = N'RECID') ORDER BY t2.name FOR XML PATH('')), 1, 2, ''), 400) AS [examples] FROM sys.tables t WHERE COLUMNPROPERTY(t.object_id, N'RECID', 'ColumnId') IS NOT NULL AND NOT EXISTS (SELECT 1 FROM sys.indexes i JOIN sys.index_columns ic ON ic.object_id = i.object_id AND ic.index_id = i.index_id AND ic.is_included_column = 0 JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id WHERE i.object_id = t.object_id AND i.is_unique = 1 AND c.name = N'RECID') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SQLDICTIONARY') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'DATAAREA') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SYSCONFIG') HAVING COUNT(*) > 0", + "dynamicsAx": true, + "query": "SELECT COUNT(*) AS [tables], LEFT(STUFF((SELECT TOP 10 N', ' + t2.name FROM sys.tables t2 WHERE COLUMNPROPERTY(t2.object_id, N'RECID', 'ColumnId') IS NOT NULL AND NOT EXISTS (SELECT 1 FROM sys.indexes i JOIN sys.index_columns ic ON ic.object_id = i.object_id AND ic.index_id = i.index_id AND ic.is_included_column = 0 JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id WHERE i.object_id = t2.object_id AND i.is_unique = 1 AND c.name = N'RECID') ORDER BY t2.name FOR XML PATH('')), 1, 2, ''), 400) AS [examples] FROM sys.tables t WHERE COLUMNPROPERTY(t.object_id, N'RECID', 'ColumnId') IS NOT NULL AND NOT EXISTS (SELECT 1 FROM sys.indexes i JOIN sys.index_columns ic ON ic.object_id = i.object_id AND ic.index_id = i.index_id AND ic.is_included_column = 0 JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id WHERE i.object_id = t.object_id AND i.is_unique = 1 AND c.name = N'RECID') HAVING COUNT(*) > 0", "subject": "database: {database}", "findingTitle": "{tables} AX-tabeller i {database} har intet unikt indeks på RECID", "explanation": "AX identificerer hver post med RecId og opretter normalt et unikt indeks på den (I_RECID) ved synkronisering. Mangler det, er en synkronisering typisk fejlet eller afbrudt, eller indekset er fjernet manuelt. Uden det bliver opslag på RecId (som AX laver hele tiden, fx ved opdatering og sletning af én post) til scanninger, og dubletter opdages ikke. Eksempler: {examples}.", @@ -85,7 +90,8 @@ "title": "Store Dynamics AX-tabeller uden clustered index (heaps)", "severity": "Medium", "scope": "Database", - "query": "SELECT COUNT(*) AS [tables], SUM(h.row_count) AS [rows], LEFT(STUFF((SELECT TOP 10 N', ' + OBJECT_NAME(h2.object_id) + N' (' + CAST(h2.row_count AS nvarchar(20)) + N')' FROM (SELECT p.object_id, SUM(p.rows) AS row_count FROM sys.partitions p JOIN sys.tables t ON t.object_id = p.object_id WHERE p.index_id = 0 GROUP BY p.object_id) h2 WHERE h2.row_count >= 100000 ORDER BY h2.row_count DESC FOR XML PATH('')), 1, 2, ''), 400) AS [largest] FROM (SELECT p.object_id, SUM(p.rows) AS row_count FROM sys.partitions p JOIN sys.tables t ON t.object_id = p.object_id WHERE p.index_id = 0 GROUP BY p.object_id) h WHERE h.row_count >= 100000 AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SQLDICTIONARY') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'DATAAREA') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SYSCONFIG') HAVING COUNT(*) > 0", + "dynamicsAx": true, + "query": "SELECT COUNT(*) AS [tables], SUM(h.row_count) AS [rows], LEFT(STUFF((SELECT TOP 10 N', ' + OBJECT_NAME(h2.object_id) + N' (' + CAST(h2.row_count AS nvarchar(20)) + N')' FROM (SELECT p.object_id, SUM(p.rows) AS row_count FROM sys.partitions p JOIN sys.tables t ON t.object_id = p.object_id WHERE p.index_id = 0 GROUP BY p.object_id) h2 WHERE h2.row_count >= 100000 ORDER BY h2.row_count DESC FOR XML PATH('')), 1, 2, ''), 400) AS [largest] FROM (SELECT p.object_id, SUM(p.rows) AS row_count FROM sys.partitions p JOIN sys.tables t ON t.object_id = p.object_id WHERE p.index_id = 0 GROUP BY p.object_id) h WHERE h.row_count >= 100000 HAVING COUNT(*) > 0", "subject": "database: {database}", "findingTitle": "{tables} store AX-tabeller i {database} har intet clustered index", "explanation": "En tabel uden clustered index (en heap) får spredte data og 'forwarded records', når poster opdateres, og den kan ikke defragmenteres med en indeks-rebuild. I AX styres det af tabellens ClusterIndex-egenskab i AOT, som i AX 3.0 ofte ikke er sat på standard- og kundetabeller. På store tabeller giver det langsomme opslag og mere I/O. I alt {rows} rækker. Største: {largest}.", @@ -98,7 +104,8 @@ "title": "Automatiske statistikker er slået fra i en Dynamics AX-database", "severity": "High", "scope": "Database", - "query": "SELECT CASE WHEN d.is_auto_create_stats_on = 1 THEN 'til' ELSE 'fra' END AS [auto_create_statistics], CASE WHEN d.is_auto_update_stats_on = 1 THEN 'til' ELSE 'fra' END AS [auto_update_statistics] FROM sys.databases d WHERE d.database_id = DB_ID() AND (d.is_auto_create_stats_on = 0 OR d.is_auto_update_stats_on = 0) AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SQLDICTIONARY') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'DATAAREA') AND EXISTS (SELECT 1 FROM sys.tables WHERE name = N'SYSCONFIG')", + "dynamicsAx": true, + "query": "SELECT CASE WHEN d.is_auto_create_stats_on = 1 THEN 'til' ELSE 'fra' END AS [auto_create_statistics], CASE WHEN d.is_auto_update_stats_on = 1 THEN 'til' ELSE 'fra' END AS [auto_update_statistics] FROM sys.databases d WHERE d.database_id = DB_ID() AND (d.is_auto_create_stats_on = 0 OR d.is_auto_update_stats_on = 0)", "subject": "database: {database}", "findingTitle": "Automatiske statistikker er slået fra i AX-databasen {database}", "explanation": "AX bygger sine forespørgsler dynamisk ud fra brugernes filtre, så optimizeren er helt afhængig af, at statistikkerne er der og er opdaterede. Uden automatiske statistikker vælger SQL Server ofte forkerte planer, især efter store kørsler som lageropgørelse og nummerserier, og formularer og batchjob kan pludselig blive meget langsomme. Auto create statistics: {auto_create_statistics}. Auto update statistics: {auto_update_statistics}.", diff --git a/src/SQLInsight.Collectors/BuiltInCollectors.cs b/src/SQLInsight.Collectors/BuiltInCollectors.cs index aac8e6f..054ca43 100644 --- a/src/SQLInsight.Collectors/BuiltInCollectors.cs +++ b/src/SQLInsight.Collectors/BuiltInCollectors.cs @@ -12,6 +12,9 @@ public static class BuiltInCollectors public static IReadOnlyList Create(Core.CustomRules.CustomRuleCatalog? custom = null, CollectorSettings? settings = null) { settings ??= new CollectorSettings(); + + // One Dynamics AX detection per instance, shared by the AX collectors and the AX community rules. + var ax = new DynamicsAx.AxDatabaseDetector(); return [ new ServerCollector(), @@ -40,7 +43,10 @@ public static class BuiltInCollectors new IndexCollector(), new AvailabilityGroupCollector(), new AgentJobCollector(), - .. custom is null ? Array.Empty() : [new Custom.CustomQueryCollector(custom)], + new DynamicsAx.DynamicsAxCollector(ax), + new DynamicsAx.AxBatchCollector(ax, settings.DynamicsAx), + new DynamicsAx.AxSequenceCollector(ax, settings.DynamicsAx), + .. custom is null ? Array.Empty() : [new Custom.CustomQueryCollector(custom, ax)], ]; } } @@ -55,6 +61,7 @@ public static class BuiltInCollectors /// A statement running longer than this is a finding. /// The version store above this is a finding. /// A session using more tempdb than this for 15 minutes is a finding. +/// When the Dynamics AX checks raise findings (SQLInsight:DynamicsAx). public sealed record CollectorSettings( Core.Data.IGroupMemberResolver? Groups = null, int MaxSysadminAccounts = 3, @@ -64,4 +71,5 @@ public sealed record CollectorSettings( int TransactionLogMegabytes = 1024, int RequestMinutes = 15, int VersionStoreMegabytes = 1024, - int TempdbSessionMegabytes = 1024); + int TempdbSessionMegabytes = 1024, + Core.DynamicsAx.AxThresholds? DynamicsAx = null); diff --git a/src/SQLInsight.Collectors/Concurrency/BlockingCollector.cs b/src/SQLInsight.Collectors/Concurrency/BlockingCollector.cs index 8ffcb40..3df2d28 100644 --- a/src/SQLInsight.Collectors/Concurrency/BlockingCollector.cs +++ b/src/SQLInsight.Collectors/Concurrency/BlockingCollector.cs @@ -29,6 +29,9 @@ public sealed class BlockingCollector : Collector : "(SELECT COUNT(*) FROM sys.dm_tran_session_transactions st WHERE st.session_id = s.session_id)"; var database = version.Major >= 11 ? "COALESCE(r.database_id, s.database_id)" : "r.database_id"; + // CONTEXT_INFO, where Dynamics AX 2009/2012 AOS can put the AX user (sys.dm_exec_sessions has it from 2008). + var contextInfo = version.Major >= 10 ? "s.context_info" : "(SELECT TOP 1 p.context_info FROM sys.sysprocesses p WHERE p.spid = s.session_id AND p.ecid = 0)"; + // blocking_session_id < 0 are special cases (orphaned DTC, deferred recovery, latch) and not a session. return $""" SELECT s.session_id, NULLIF(r.blocking_session_id, 0), COALESCE(r.status, s.status), r.wait_type, r.wait_time, r.wait_resource, @@ -38,7 +41,7 @@ public sealed class BlockingCollector : Collector (CASE WHEN r.statement_end_offset = -1 THEN DATALENGTH(t.text) ELSE r.statement_end_offset END - r.statement_start_offset) / 2 + 1) ELSE t.text END, 4000), - s.last_request_end_time + s.last_request_end_time, {contextInfo} FROM sys.dm_exec_sessions s LEFT JOIN sys.dm_exec_requests r ON r.session_id = s.session_id LEFT JOIN sys.dm_exec_connections c ON c.session_id = s.session_id @@ -52,7 +55,7 @@ public sealed class BlockingCollector : Collector internal const string Sql2000 = """ SELECT p.spid, NULLIF(CASE WHEN p.blocked = p.spid THEN 0 ELSE p.blocked END, 0), RTRIM(p.status), RTRIM(p.lastwaittype), p.waittime, RTRIM(p.waitresource), DB_NAME(p.dbid), RTRIM(p.loginame), RTRIM(p.hostname), RTRIM(p.program_name), - p.open_tran, CAST(NULL AS nvarchar(4000)), p.last_batch + p.open_tran, CAST(NULL AS nvarchar(4000)), p.last_batch, p.context_info FROM master.dbo.sysprocesses p WHERE (p.blocked > 0 AND p.blocked <> p.spid) OR p.spid IN (SELECT blocked FROM master.dbo.sysprocesses WHERE blocked > 0 AND blocked <> spid) @@ -73,6 +76,7 @@ public sealed class BlockingCollector : Collector OpenTransactionCount = r.NullableInt(10) ?? 0, SqlText = r.NullableString(11), LastRequestEndTime = r.NullableDateTime(12), + ContextInfo = r.NullableHex(13), }; private static SessionInfo Map2000(Row r) => Map2005(r); diff --git a/src/SQLInsight.Collectors/Custom/CustomQueryCollector.cs b/src/SQLInsight.Collectors/Custom/CustomQueryCollector.cs index 068baaa..0c600ce 100644 --- a/src/SQLInsight.Collectors/Custom/CustomQueryCollector.cs +++ b/src/SQLInsight.Collectors/Custom/CustomQueryCollector.cs @@ -6,9 +6,10 @@ namespace SQLInsight.Collectors.Custom; /// /// Runs the queries of the custom rules (community and local). Each rule gets its own result or error, so one broken -/// or unpermitted query only affects its own rule. +/// or unpermitted query only affects its own rule. Dynamics AX rules (dynamicsAx) only run in the monitored databases the +/// shared AX detection recognises; elsewhere they send no query at all. /// -public sealed class CustomQueryCollector(CustomRuleCatalog catalog) : Collector +public sealed class CustomQueryCollector(CustomRuleCatalog catalog, DynamicsAx.AxDatabaseDetector? ax = null) : Collector { /// SQL Server error numbers that mean the login lacks a permission or access to a database. private static readonly HashSet PermissionErrors = [229, 230, 262, 297, 300, 916, 4060, 15247, 15151]; @@ -23,6 +24,8 @@ public sealed class CustomQueryCollector(CustomRuleCatalog catalog) : Collector< public override string Name => "custom-rules"; public override TimeSpan DefaultInterval => TimeSpan.FromMinutes(15); + private readonly DynamicsAx.AxDatabaseDetector _ax = ax ?? new DynamicsAx.AxDatabaseDetector(); + protected override async Task CollectAsync(CollectorContext context, CustomQueryResults? previous) { ArgumentNullException.ThrowIfNull(context); @@ -47,6 +50,19 @@ public sealed class CustomQueryCollector(CustomRuleCatalog catalog) : Collector< databases ??= (await Db.QueryAsync(context.Connection, DatabasesSql, r => r.String(0), context.CancellationToken).ConfigureAwait(false)) .Where(context.IncludesDatabase).ToList(); + if (rule.DynamicsAx) + { + // The AX detection needs SQL Server 2005 (sys.columns); AX rules have nowhere to run before that. + var axDatabases = context.Instance.Version.Major >= 9 + ? (await _ax.GetAsync(context).ConfigureAwait(false)).Databases.Select(d => d.Database).ToHashSet(StringComparer.OrdinalIgnoreCase) + : []; + var targets = databases.Where(axDatabases.Contains).ToList(); + results[rule.Id] = targets.Count == 0 + ? new CustomQueryResult([], NotRelevant: "Ikke relevant: ingen af de overvågede databaser på instansen er genkendt som Dynamics AX.") + : await RunPerDatabaseAsync(context, rule, targets).ConfigureAwait(false); + continue; + } + results[rule.Id] = await RunPerDatabaseAsync(context, rule, databases).ConfigureAwait(false); } } diff --git a/src/SQLInsight.Collectors/Db.cs b/src/SQLInsight.Collectors/Db.cs index 3a3620c..6722f88 100644 --- a/src/SQLInsight.Collectors/Db.cs +++ b/src/SQLInsight.Collectors/Db.cs @@ -11,11 +11,14 @@ internal static class Db { public const int CommandTimeoutSeconds = 30; - public static async Task> QueryAsync(DbConnection connection, string sql, Func map, CancellationToken cancellationToken) + public static Task> QueryAsync(DbConnection connection, string sql, Func map, CancellationToken cancellationToken) => + QueryAsync(connection, sql, map, CommandTimeoutSeconds, cancellationToken); + + public static async Task> QueryAsync(DbConnection connection, string sql, Func map, int timeoutSeconds, CancellationToken cancellationToken) { await using var command = connection.CreateCommand(); command.CommandText = sql; - command.CommandTimeout = CommandTimeoutSeconds; + command.CommandTimeout = timeoutSeconds; await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); var rows = new List(); var row = new Row(reader); @@ -37,6 +40,32 @@ internal static class Db /// An N'…' string literal with quotes escaped. public static string Literal(string value) => "N'" + value.Replace("'", "''", StringComparison.Ordinal) + "'"; + /// + /// SQL Server's error number: SqlClient exposes it as Number, ODBC per error as NativeError. Null when unknown + /// (also for a command timeout, which SqlClient reports as -2). + /// + public static int? ErrorNumber(DbException exception) + { + ArgumentNullException.ThrowIfNull(exception); + if (exception.GetType().GetProperty("Number")?.GetValue(exception) is int number) + { + return number; + } + + if (exception.GetType().GetProperty("Errors")?.GetValue(exception) is System.Collections.IEnumerable errors) + { + foreach (var error in errors) + { + if (error.GetType().GetProperty("NativeError")?.GetValue(error) is int native && native != 0) + { + return native; + } + } + } + + return null; + } + /// A [bracketed] identifier with brackets escaped. public static string Identifier(string name) => "[" + name.Replace("]", "]]", StringComparison.Ordinal) + "]"; @@ -105,4 +134,21 @@ internal sealed class Row(DbDataReader reader) public DateTime? NullableDateTime(int ordinal) => reader.IsDBNull(ordinal) ? null : DateTime(ordinal); public int FieldCount => reader.FieldCount; + + /// A binary value as hex without trailing zero bytes (CONTEXT_INFO is padded), or null when null or all zeros. + public string? NullableHex(int ordinal) + { + if (reader.IsDBNull(ordinal) || reader.GetValue(ordinal) is not byte[] bytes) + { + return null; + } + + var length = bytes.Length; + while (length > 0 && bytes[length - 1] == 0) + { + length--; + } + + return length == 0 ? null : Convert.ToHexString(bytes, 0, length); + } } diff --git a/src/SQLInsight.Collectors/Discovery/DatabaseDiscovery.cs b/src/SQLInsight.Collectors/Discovery/DatabaseDiscovery.cs index 9588b41..205a4cb 100644 --- a/src/SQLInsight.Collectors/Discovery/DatabaseDiscovery.cs +++ b/src/SQLInsight.Collectors/Discovery/DatabaseDiscovery.cs @@ -1,5 +1,6 @@ using System.Data.Common; using SQLInsight.Core.Configuration; +using SQLInsight.Core.DynamicsAx; using SQLInsight.Core.Instances; namespace SQLInsight.Collectors.Discovery; @@ -7,7 +8,7 @@ namespace SQLInsight.Collectors.Discovery; /// /// Lists every database the agent can see on an instance, with state, size, recovery model and compatibility level, /// and recognises Dynamics AX databases. Only metadata is read (sys.databases, sys.master_files, sys.tables and -/// sys.columns, or their SQL Server 2000 counterparts), through three-part names: the session never changes database +/// sys.columns, or their SQL Server 2000 counterparts), through three-part names and [db].sys.sp_executesql: the session never changes database /// and nothing is written. Works from SQL Server 2000 (over the legacy ODBC connection) and up. /// public static class DatabaseDiscovery @@ -37,33 +38,16 @@ public static class DatabaseDiscovery """; /// - /// The Dynamics AX signature, the same as the AX community rules use (rules/community/dynamics-ax.json): the system - /// tables SQLDICTIONARY, DATAAREA and SYSCONFIG, in any schema (AX 3.0 owns them as bmssa). RECID on SQLDICTIONARY - /// is int (precision 10) in AX 3.0 and bigint (19) from AX 4.0. + /// The central Dynamics AX detection (Core/DynamicsAx/AxDetection.cs), the same as the AX collectors and the AX + /// community rules use: run in the database through [db].sys.sp_executesql on SQL Server 2005 and later, with three-part + /// names to sysobjects on SQL Server 2000. The session never changes database. /// public static string DynamicsAxSql(string database, SqlServerVersion version) { ArgumentNullException.ThrowIfNull(database); - var db = Db.Identifier(database); - return version.Major >= 9 - ? $""" - SELECT COUNT(DISTINCT t.name), MAX(CASE WHEN t.name = N'SQLDICTIONARY' THEN c.precision END) - FROM {db}.sys.tables t - LEFT JOIN {db}.sys.columns c ON c.object_id = t.object_id AND c.name = N'RECID' - WHERE t.name IN (N'SQLDICTIONARY', N'DATAAREA', N'SYSCONFIG') - """ - : $""" - SELECT COUNT(DISTINCT o.name), MAX(CASE WHEN o.name = N'SQLDICTIONARY' THEN c.xprec END) - FROM {db}.dbo.sysobjects o - LEFT JOIN {db}.dbo.syscolumns c ON c.id = o.id AND c.name = N'RECID' - WHERE o.xtype = 'U' AND o.name IN (N'SQLDICTIONARY', N'DATAAREA', N'SYSCONFIG') - """; + return version.Major >= 9 ? AxSql.InDatabase(database, AxDetection.MetadataSql) : AxDetection.MetadataSql2000(database); } - /// "3.0" or "4.0+" when all three AX system tables are there, else null. - public static string? DynamicsAxVersion(int tables, int? recIdPrecision) => - tables < 3 ? null : recIdPrecision is 10 ? "3.0" : "4.0+"; - public static async Task> DiscoverAsync(DbConnection connection, InstanceInfo instance, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(connection); @@ -82,6 +66,7 @@ public static class DatabaseDiscovery IsSystem = DatabaseSelection.IsSystemDatabase(r.String(0)), }, cancellationToken).ConfigureAwait(false); + var names = databases.Select(d => d.Name).ToList(); for (var i = 0; i < databases.Count; i++) { var database = databases[i]; @@ -92,11 +77,12 @@ public static class DatabaseDiscovery try { - var rows = await Db.QueryAsync(connection, DynamicsAxSql(database.Name, instance.Version), r => (Tables: r.Int(0), Precision: r.NullableInt(1)), cancellationToken) - .ConfigureAwait(false); - if (rows.Count > 0 && DynamicsAxVersion(rows[0].Tables, rows[0].Precision) is { } version) + var rows = await Db.QueryAsync(connection, DynamicsAxSql(database.Name, instance.Version), + r => new AxDetectionRow(r.String(0), r.String(1), r.String(2), r.NullableString(3) ?? "?", r.NullableInt(4) ?? 0, r.NullableInt(5) ?? 0, r.Bool(6), r.NullableString(7)), + AxSql.CommandTimeoutSeconds, cancellationToken).ConfigureAwait(false); + if (AxDetection.Recognise(database.Name, rows, names) is { } ax) { - databases[i] = database with { IsDynamicsAx = true, DynamicsAxVersion = version }; + databases[i] = database with { IsDynamicsAx = true, DynamicsAxVersion = AxVersions.ShortText(ax.Version) }; } } catch (DbException) diff --git a/src/SQLInsight.Collectors/DynamicsAx/AxBatchCollector.cs b/src/SQLInsight.Collectors/DynamicsAx/AxBatchCollector.cs new file mode 100644 index 0000000..a254f10 --- /dev/null +++ b/src/SQLInsight.Collectors/DynamicsAx/AxBatchCollector.cs @@ -0,0 +1,149 @@ +using SQLInsight.Core.DynamicsAx; +using SQLInsight.Core.Instances; + +namespace SQLInsight.Collectors.DynamicsAx; + +/// +/// Batch jobs in every monitored AX database where the agent may read the batch tables (optional AX read access), +/// every 5 minutes: status, planned and actual start, and completed runs (from BATCHJOBHISTORY in AX 2009 and later, +/// from what SQLInsight itself sees in AX 3.0/4.0). Each job is compared with its own history, kept by SQLInsight in +/// its repository (insight.AxBatchRuns). Only standard AX tables are read; bounded with TOP; never written to. +/// +public sealed class AxBatchCollector(AxDatabaseDetector detector, AxThresholds? thresholds = null) : Collector +{ + /// Run keys remembered to avoid storing a run twice; more than this and the set starts over (the repository also ignores duplicates). + private const int RecentRunKeysKept = 20_000; + + /// Finished rows older than this are not kept in the snapshot (AX 3.0 keeps one BATCH row per recurrence). + private static readonly TimeSpan FinishedShown = TimeSpan.FromDays(7); + + private const int JobsShown = 500; + + private readonly AxThresholds _thresholds = thresholds ?? new AxThresholds(); + + public override string Name => "dynamics-ax-batch"; + public override TimeSpan DefaultInterval => TimeSpan.FromMinutes(5); + public override IReadOnlyCollection RequiredCapabilities => [Capability.DynamicManagementViews]; + + protected override async Task CollectAsync(CollectorContext context, AxBatchSnapshot? previous) + { + var ax = await detector.GetAsync(context).ConfigureAwait(false); + var history = previous?.History.Clone() ?? new AxBatchHistory(); + var firstSeen = new Dictionary(StringComparer.OrdinalIgnoreCase); + var recentKeys = previous?.RecentRunKeys is { Count: < RecentRunKeysKept } kept ? new HashSet(kept) : []; + var databases = new List(); + foreach (var database in ax.Databases) + { + databases.Add(await CollectDatabaseAsync(context, database, previous, history, firstSeen, recentKeys).ConfigureAwait(false)); + } + + return new AxBatchSnapshot + { + CapturedAt = context.Now, + Thresholds = _thresholds, + Databases = databases, + History = history, + FirstSeenExecuting = firstSeen, + RecentRunKeys = recentKeys, + }; + } + + private static async Task CollectDatabaseAsync(CollectorContext context, AxDatabase database, AxBatchSnapshot? previous, AxBatchHistory history, + Dictionary firstSeen, HashSet recentKeys) + { + var name = database.Database; + var earlier = previous?.Databases.FirstOrDefault(d => string.Equals(d.Database, name, StringComparison.OrdinalIgnoreCase)); + if (AxSchemaAdapter.For(database.Version) is not { } adapter) + { + return new AxBatchDatabase { Database = name, Version = database.Version, NotMeasurable = "Ikke målbar: AX-versionen kunne ikke bestemmes ud fra skemaet." }; + } + + var query = adapter.BatchJobs(database); + var (rows, problem) = await AxReader.ReadAsync(context, database, query, Map).ConfigureAwait(false); + if (rows is null) + { + // A lock timeout keeps the previous picture (marked as such), so findings do not flap. + return AxReader.IsTransient(problem) && earlier is { NotMeasurable: null } + ? earlier with { NewRuns = [], Notes = [.. earlier.Notes.Where(n => n != AxReader.LockTimeoutText), AxReader.LockTimeoutText] } + : new AxBatchDatabase { Database = name, Version = database.Version, NotMeasurable = problem }; + } + + var notes = new List(query.Notes); + var newRuns = new List(); + long? watermark = earlier?.HistoryWatermark; + + // AX 2009 and later: completed runs from AX's own history, by RECID. + var historyQuery = adapter.BatchHistory(database, watermark); + var (historyRows, historyProblem) = await AxReader.ReadAsync(context, database, historyQuery, MapHistory).ConfigureAwait(false); + var fromHistoryTable = historyRows is not null; + if (historyRows is not null) + { + foreach (var row in historyRows) + { + watermark = Math.Max(watermark ?? 0, row.RecId); + if (adapter.HistoryRun(row) is { } run && recentKeys.Add(Scoped(name, run.RunKey))) + { + newRuns.Add(run); + } + } + } + else if (historyProblem is not null) + { + notes.Add("Kørselshistorik fra BATCHJOBHISTORY: " + historyProblem + " SQLInsight bruger i stedet de kørsler, den selv ser."); + } + + // Pass 1: when jobs were first seen executing, and (AX 3.0/4.0, or no history access) runs SQLInsight saw finish. + var prelim = new List<(AxBatchJobRow Row, AxBatchJob Job)>(); + foreach (var row in rows) + { + var seenKey = AxBatchHistory.Key(name, "#" + row.RecId); + DateTime? seenBefore = previous?.FirstSeenExecuting.TryGetValue(seenKey, out var seen) == true ? seen : null; + var job = adapter.Interpret(row, history, name, null); + if (job.Status == AxBatchStatus.Executing) + { + firstSeen[seenKey] = seenBefore ?? context.Now.UtcDateTime; + } + else if (!fromHistoryTable && AxSchemaAdapter.ObservedRun(job, seenBefore) is { } run && recentKeys.Add(Scoped(name, run.RunKey))) + { + newRuns.Add(run); + } + + prelim.Add((row, job)); + } + + foreach (var run in newRuns) + { + history.Add(name, run); + } + + // Pass 2: the jobs with their history's statistics, the interesting ones first. + var now = context.Now.UtcDateTime; + var jobs = prelim + .Select(p => adapter.Interpret(p.Row, history, name, firstSeen.TryGetValue(AxBatchHistory.Key(name, "#" + p.Row.RecId), out var s) ? s : null)) + .Where(j => j.Status is not (AxBatchStatus.Finished or AxBatchStatus.Canceled) || (j.EndedUtc ?? j.PlannedStartUtc ?? now) >= now - FinishedShown) + .OrderBy(j => j.Status switch { AxBatchStatus.Executing => 0, AxBatchStatus.Error => 1, AxBatchStatus.Waiting => 2, _ => 3 }) + .ThenByDescending(j => j.RecId) + .Take(JobsShown) + .ToList(); + + return new AxBatchDatabase + { + Database = name, + Version = database.Version, + Jobs = jobs, + NewRuns = newRuns, + StatusCounts = rows.GroupBy(r => r.Status).ToDictionary(g => g.Key, g => g.Count()), + Notes = notes, + HistoryWatermark = watermark, + }; + } + + private static string Scoped(string database, string runKey) => database + "\u001f" + runKey; + + private static AxBatchJobRow Map(Row r) => new( + r.Long(0), r.NullableInt(1), r.NullableString(2), r.NullableString(3), r.NullableString(4), r.Int(5), + r.NullableDateTime(6), r.NullableInt(7), r.NullableDateTime(8), r.NullableInt(9), r.NullableDateTime(10), r.NullableInt(11), r.Int(12)); + + private static AxBatchHistoryRow MapHistory(Row r) => new( + r.Long(0), r.NullableLong(1), r.NullableString(2), r.NullableString(3), r.Int(4), r.NullableDateTime(5), r.NullableDateTime(6), r.Int(7)); +} diff --git a/src/SQLInsight.Collectors/DynamicsAx/AxDatabaseDetector.cs b/src/SQLInsight.Collectors/DynamicsAx/AxDatabaseDetector.cs new file mode 100644 index 0000000..a405d17 --- /dev/null +++ b/src/SQLInsight.Collectors/DynamicsAx/AxDatabaseDetector.cs @@ -0,0 +1,140 @@ +using System.Data.Common; +using SQLInsight.Core.DynamicsAx; +using SQLInsight.Core.Instances; + +namespace SQLInsight.Collectors.DynamicsAx; + +/// +/// Finds the Dynamics AX databases on one instance with the central detection () and keeps +/// the answer for , so new and removed AX databases are noticed within the hour. Shared by +/// everything AX-specific on the instance (the AX collectors and the AX community rules). Only databases chosen for +/// monitoring are looked at; reads metadata, plus the company list when the agent may read DATAAREA. Not thread-safe: +/// one instance's collectors run one at a time. +/// +public sealed class AxDatabaseDetector +{ + public static readonly TimeSpan Refresh = TimeSpan.FromHours(1); + + private readonly Dictionary _cache = new(StringComparer.OrdinalIgnoreCase); + private string? _login; + + /// Online user databases the login can enter, with their snapshot settings. + internal const string DatabasesSql = """ + SELECT name, snapshot_isolation_state, is_read_committed_snapshot_on + FROM sys.databases + WHERE database_id > 4 AND state = 0 AND source_database_id IS NULL AND HAS_DBACCESS(name) = 1 + ORDER BY name + """; + + /// The monitored AX databases, from the cache when younger than (unless ). + public async Task GetAsync(CollectorContext context, bool force = false) + { + ArgumentNullException.ThrowIfNull(context); + var connection = context.Connection; + var ct = context.CancellationToken; + var databases = await Db.QueryAsync(connection, DatabasesSql, r => (Name: r.String(0), Snapshot: r.Int(1), Rcsi: r.Bool(2)), ct).ConfigureAwait(false); + _login ??= (await Db.QueryAsync(connection, "SELECT SUSER_SNAME()", r => r.NullableString(0), ct).ConfigureAwait(false)).FirstOrDefault(); + var names = databases.Select(d => d.Name).ToList(); + var found = new List(); + var problems = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var (name, snapshot, rcsi) in databases) + { + if (!context.IncludesDatabase(name) || name.EndsWith("_model", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (force || !_cache.TryGetValue(name, out var cached) || context.Now - cached.At >= Refresh) + { + cached = await DetectAsync(context, name, names).ConfigureAwait(false); + _cache[name] = cached; + } + + if (cached.Problem is { } problem) + { + problems[name] = problem; + } + + if (cached.Database is { } ax) + { + found.Add(ax with + { + AgentLogin = _login, + Isolation = rcsi ? AxReadIsolation.ReadCommittedSnapshot : snapshot == 1 ? AxReadIsolation.Snapshot : AxReadIsolation.ReadCommitted, + }); + } + } + + foreach (var gone in _cache.Keys.Where(k => !names.Contains(k, StringComparer.OrdinalIgnoreCase)).ToList()) + { + _cache.Remove(gone); + } + + return new AxDatabaseSnapshot { CapturedAt = context.Now, Databases = found, AgentLogin = _login, Problems = problems }; + } + + private async Task<(DateTimeOffset, AxDatabase?, string?)> DetectAsync(CollectorContext context, string database, IReadOnlyList names) + { + try + { + var rows = await Db.QueryAsync(context.Connection, AxSql.InDatabase(database, AxDetection.MetadataSql), + r => new AxDetectionRow(r.String(0), r.String(1), r.String(2), r.NullableString(3) ?? "?", r.NullableInt(4) ?? 0, r.NullableInt(5) ?? 0, r.Bool(6), r.NullableString(7)), + AxSql.CommandTimeoutSeconds, context.CancellationToken).ConfigureAwait(false); + if (AxDetection.Recognise(database, rows, names) is not { } ax) + { + return (context.Now, null, null); + } + + ax = ax with { AgentLogin = _login }; + if (AxSchemaAdapter.For(ax.Version) is not null) + { + var (companies, reason) = await AxReader.ReadAsync(context, ax, AxSchemaAdapter.Companies(ax), + r => new AxCompany(r.String(0), r.NullableString(1), r.Bool(2))).ConfigureAwait(false); + ax = ax with { Companies = companies, CompaniesReason = reason }; + } + + return (context.Now, ax, null); + } + catch (DbException ex) + { + // No access to its metadata (e.g. no database user before 2014): it cannot be told apart, and nothing AX-specific runs there. + return (context.Now, null, "AX-genkendelsen kunne ikke læse databasens metadata: " + ex.Message); + } + } +} + +/// Reads an AX query: the rows, or why not (Danish). A lock timeout or a missing permission is a reason, never an exception. +internal static class AxReader +{ + public const string LockTimeoutText = "Ikke målt denne gang: AX-tabellen var låst i mere end 2 sekunder, og SQLInsight venter ikke på AX' låse. Prøves igen ved næste indsamling."; + + public static async Task<(List? Rows, string? Problem)> ReadAsync(CollectorContext context, AxDatabase database, AxQuery? query, Func map) + { + if (query is null) + { + return (null, null); + } + + if (query.Sql is not { } sql) + { + return (null, query.NotMeasurable); + } + + try + { + return (await Db.QueryAsync(context.Connection, AxSql.InDatabase(database.Database, sql, database.Isolation), map, AxSql.CommandTimeoutSeconds, + context.CancellationToken).ConfigureAwait(false), null); + } + catch (DbException ex) + { + return (null, Db.ErrorNumber(ex) switch + { + AxSql.LockTimeoutError => LockTimeoutText, + 229 or 230 or 297 or 916 => "Ikke målbar: adgang nægtet. " + AxReadAccess.NotMeasurable(database, AxReadAccess.Tables(database)), + _ => "Ikke målbar: læsningen fejlede (" + ex.Message + ").", + }); + } + } + + public static bool IsTransient(string? problem) => problem == LockTimeoutText; +} diff --git a/src/SQLInsight.Collectors/DynamicsAx/AxSequenceCollector.cs b/src/SQLInsight.Collectors/DynamicsAx/AxSequenceCollector.cs new file mode 100644 index 0000000..4603a79 --- /dev/null +++ b/src/SQLInsight.Collectors/DynamicsAx/AxSequenceCollector.cs @@ -0,0 +1,93 @@ +using SQLInsight.Core.DynamicsAx; +using SQLInsight.Core.Instances; + +namespace SQLInsight.Collectors.DynamicsAx; + +/// +/// Number sequences (NUMBERSEQUENCETABLE) and RecId allocation (SYSTEMSEQUENCES) in every monitored AX database where +/// the agent may read them, every hour. SQLInsight samples NEXTREC and NEXTVAL once a day into its own repository +/// (insight.AxSequenceSamples, bounded retention) and forecasts from that history when each runs out. Bounded with TOP +/// (the most used first); never written to. +/// +public sealed class AxSequenceCollector(AxDatabaseDetector detector, AxThresholds? thresholds = null) : Collector +{ + private readonly AxThresholds _thresholds = thresholds ?? new AxThresholds(); + + public override string Name => "dynamics-ax-sequences"; + public override TimeSpan DefaultInterval => TimeSpan.FromHours(1); + public override IReadOnlyCollection RequiredCapabilities => [Capability.DynamicManagementViews]; + + protected override async Task CollectAsync(CollectorContext context, AxSequenceSnapshot? previous) + { + var ax = await detector.GetAsync(context).ConfigureAwait(false); + var now = context.Now.UtcDateTime; + var history = new Dictionary>(StringComparer.Ordinal); + var databases = new List(); + foreach (var database in ax.Databases) + { + var name = database.Database; + var earlier = previous?.Databases.FirstOrDefault(d => string.Equals(d.Database, name, StringComparison.OrdinalIgnoreCase)); + if (AxSchemaAdapter.For(database.Version) is not { } adapter) + { + const string unknown = "Ikke målbar: AX-versionen kunne ikke bestemmes ud fra skemaet."; + databases.Add(new AxSequenceDatabase { Database = name, Version = database.Version, NumberSequencesNotMeasurable = unknown, RecIdsNotMeasurable = unknown }); + continue; + } + + IReadOnlyList Samples(char kind, string key, long value) + { + var historyKey = AxSequenceSnapshot.HistoryKey(name, kind, key); + var samples = (previous?.History.TryGetValue(historyKey, out var stored) == true ? stored : []) + .Where(s => s.SampledUtc >= now.AddDays(-(_thresholds.ForecastBasisDays + 1)) && s.SampledUtc.Date != now.Date) + .Append(new AxSample(now, value)) + .ToList(); + history[historyKey] = samples; + return samples; + } + + var notes = new List(); + var numbersQuery = adapter.NumberSequences(database); + notes.AddRange(numbersQuery.Notes); + var (numberRows, numberProblem) = await AxReader.ReadAsync(context, database, numbersQuery, MapNumberSequence).ConfigureAwait(false); + var sequences = numberRows?.Select(r => adapter.Interpret(r, Samples(AxSequenceSnapshot.NumberSequenceKind, adapter.SequenceKey(r), r.Next), now, _thresholds.ForecastBasisDays)) + .ToList(); + + var recIdQuery = adapter.RecIds(database); + notes.AddRange(recIdQuery.Notes); + var nextValIsInt = string.Equals(database.Table("SYSTEMSEQUENCES")?.Column("NEXTVAL")?.Type, "int", StringComparison.OrdinalIgnoreCase); + var (recIdRows, recIdProblem) = await AxReader.ReadAsync(context, database, recIdQuery, MapRecId).ConfigureAwait(false); + var recIds = recIdRows?.Select(r => AxSchemaAdapter.Interpret(r, nextValIsInt, Samples(AxSequenceSnapshot.RecIdKind, AxSchemaAdapter.RecIdKey(r), r.NextVal), now, + _thresholds.ForecastBasisDays)).ToList(); + + // A lock timeout keeps the previous values, so findings do not flap. + databases.Add(new AxSequenceDatabase + { + Database = name, + Version = database.Version, + NumberSequences = sequences ?? (AxReader.IsTransient(numberProblem) ? earlier?.NumberSequences ?? [] : []), + NumberSequencesNotMeasurable = sequences is null && !(AxReader.IsTransient(numberProblem) && earlier?.NumberSequencesNotMeasurable is null && earlier is not null) ? numberProblem : null, + RecIds = recIds ?? (AxReader.IsTransient(recIdProblem) ? earlier?.RecIds ?? [] : []), + RecIdsNotMeasurable = recIds is null && !(AxReader.IsTransient(recIdProblem) && earlier?.RecIdsNotMeasurable is null && earlier is not null) ? recIdProblem : null, + Notes = notes, + }); + + // Sequences not read this time keep their history (a lock timeout must not lose it). + foreach (var (key, samples) in previous?.History ?? new Dictionary>()) + { + if (key.StartsWith(name + "\u001f", StringComparison.Ordinal) && !history.ContainsKey(key)) + { + history[key] = samples.Where(s => s.SampledUtc >= now.AddDays(-(_thresholds.ForecastBasisDays + 1))).ToList(); + } + } + } + + return new AxSequenceSnapshot { CapturedAt = context.Now, Thresholds = _thresholds, Databases = databases, History = history }; + } + + private static AxNumberSequenceRow MapNumberSequence(Row r) => new( + r.Long(0), r.String(1), r.NullableString(2), r.NullableString(3), r.Long(4), r.Long(5), r.Long(6), r.NullableString(7), + r.Bool(8), r.Bool(9), r.Bool(10), r.Bool(11), r.IsNull(12) ? null : r.Bool(12), r.NullableDateTime(13), r.NullableInt(14), + r.IsNull(15) ? null : r.Double(15), r.Int(16)); + + private static AxRecIdRow MapRecId(Row r) => new(r.Int(0), r.NullableString(1), r.Long(2), r.NullableLong(3), r.NullableString(4)); +} diff --git a/src/SQLInsight.Collectors/DynamicsAx/DynamicsAxCollector.cs b/src/SQLInsight.Collectors/DynamicsAx/DynamicsAxCollector.cs new file mode 100644 index 0000000..b11fa8e --- /dev/null +++ b/src/SQLInsight.Collectors/DynamicsAx/DynamicsAxCollector.cs @@ -0,0 +1,19 @@ +using SQLInsight.Core.DynamicsAx; +using SQLInsight.Core.Instances; + +namespace SQLInsight.Collectors.DynamicsAx; + +/// +/// Which monitored databases on the instance are Dynamics AX, with version, schema, companies and the agent's read +/// access, every hour (collector "dynamics-ax"). The AX pages in SQLInsight Web are built on it. +/// +public sealed class DynamicsAxCollector(AxDatabaseDetector detector) : Collector +{ + public override string Name => "dynamics-ax"; + public override TimeSpan DefaultInterval => AxDatabaseDetector.Refresh; + + /// The detection uses sys.tables, sys.columns and HAS_PERMS_BY_NAME: SQL Server 2005 and later. + public override IReadOnlyCollection RequiredCapabilities => [Capability.DynamicManagementViews]; + + protected override Task CollectAsync(CollectorContext context, AxDatabaseSnapshot? previous) => detector.GetAsync(context, force: true); +} diff --git a/src/SQLInsight.Core/Configuration/AgentRequests.cs b/src/SQLInsight.Core/Configuration/AgentRequests.cs index 3e4c5fe..0314734 100644 --- a/src/SQLInsight.Core/Configuration/AgentRequests.cs +++ b/src/SQLInsight.Core/Configuration/AgentRequests.cs @@ -116,10 +116,10 @@ public sealed record DiscoveredDatabase /// The agent can enter the database (HAS_DBACCESS); without it, database-specific checks cannot run there. public bool HasAccess { get; init; } - /// The Dynamics AX system tables (SQLDICTIONARY, DATAAREA, SYSCONFIG) are there. + /// Recognised as Dynamics AX by the central detection (Core/DynamicsAx/AxDetection.cs). public bool IsDynamicsAx { get; init; } - /// "3.0" (32-bit RECID) or "4.0+" (64-bit RECID), when . + /// The AX version as ("3.0", "2009", "2012 R2/R3" …), when . public string? DynamicsAxVersion { get; init; } } diff --git a/src/SQLInsight.Core/CustomRules/CustomRuleDefinition.cs b/src/SQLInsight.Core/CustomRules/CustomRuleDefinition.cs index 1d75df3..69bac89 100644 --- a/src/SQLInsight.Core/CustomRules/CustomRuleDefinition.cs +++ b/src/SQLInsight.Core/CustomRules/CustomRuleDefinition.cs @@ -52,6 +52,12 @@ public sealed partial record CustomRuleDefinition /// Who wrote the rule (name or Forgejo user). Shown in the rule list. public string? Author { get; init; } + /// + /// A Dynamics AX rule (scope Database): runs only in databases the central AX detection recognises (Core/DynamicsAx) and + /// that are chosen for monitoring, and its findings only reach SQLInsight Hub for databases where AX findings may be sent. + /// + public bool DynamicsAx { get; init; } + /// "community" for rules from the signed rule pack, "lokal" for the customer's own. Set when loaded. [JsonIgnore] public string Origin { get; init; } = "lokal"; @@ -92,6 +98,11 @@ public sealed partial record CustomRuleDefinition errors.Add($"minimumVersion '{MinimumVersion}' er ikke en version som 11.0."); } + if (DynamicsAx && Scope != CustomRuleScope.Database) + { + errors.Add("dynamicsAx kræver scope Database."); + } + if (!string.IsNullOrWhiteSpace(Query)) { errors.AddRange(QueryProblems(Query)); @@ -245,4 +256,6 @@ public sealed record CustomQueryResults(IReadOnlyDictionaryColumn name → value (as text) per row, when the query ran. /// Why it did not run. /// The error was a missing permission (the rule is then not measurable, not failing). -public sealed record CustomQueryResult(IReadOnlyList> Rows, string? Error = null, bool PermissionDenied = false, bool Truncated = false); +/// Why the rule had nowhere to run (a Dynamics AX rule on an instance without monitored AX databases). +public sealed record CustomQueryResult(IReadOnlyList> Rows, string? Error = null, bool PermissionDenied = false, bool Truncated = false, + string? NotRelevant = null); diff --git a/src/SQLInsight.Rules/Custom/DeclarativeRule.cs b/src/SQLInsight.Rules/Custom/DeclarativeRule.cs index 5dffb7f..78d9790 100644 --- a/src/SQLInsight.Rules/Custom/DeclarativeRule.cs +++ b/src/SQLInsight.Rules/Custom/DeclarativeRule.cs @@ -29,6 +29,11 @@ public sealed class DeclarativeRule(CustomRuleDefinition definition) : IRule throw new DataUnavailableException("Reglens forespørgsel er ikke kørt endnu."); } + if (result.NotRelevant is { } notRelevant) + { + return RuleEvaluation.NotApplicable(notRelevant); + } + if (result.Error is { } error) { if (result.PermissionDenied) diff --git a/tests/SQLInsight.Collectors.Tests/DiscoveryTests.cs b/tests/SQLInsight.Collectors.Tests/DiscoveryTests.cs index 47cd08d..7c2958e 100644 --- a/tests/SQLInsight.Collectors.Tests/DiscoveryTests.cs +++ b/tests/SQLInsight.Collectors.Tests/DiscoveryTests.cs @@ -12,22 +12,14 @@ public sealed class DiscoveryTests ServerName = "SQL01", Version = SqlServerVersion.Parse(version), Edition = "Standard Edition", EngineEdition = edition, }; - [Theory] - [InlineData(3, 10, "3.0")] - [InlineData(3, 19, "4.0+")] - [InlineData(3, null, "4.0+")] - [InlineData(2, 10, null)] - [InlineData(0, null, null)] - public void Dynamics_ax_is_recognised_by_its_three_system_tables_and_the_recid_type(int tables, int? precision, string? expected) => - Assert.Equal(expected, DatabaseDiscovery.DynamicsAxVersion(tables, precision)); - [Fact] - public void Dynamics_ax_detection_uses_three_part_names_and_catalog_views_by_version() + public void Dynamics_ax_detection_is_the_central_one_run_in_the_database_or_with_three_part_names_on_2000() { var modern = DatabaseDiscovery.DynamicsAxSql("AX]30", SqlServerVersion.Parse("9.0.5000.0")); var legacy = DatabaseDiscovery.DynamicsAxSql("AX30", SqlServerVersion.Parse("8.0.2039.0")); - Assert.Contains("[AX]]30].sys.tables", modern, StringComparison.Ordinal); + Assert.StartsWith("EXEC [AX]]30].sys.sp_executesql", modern, StringComparison.Ordinal); + Assert.Contains("FROM sys.tables", modern, StringComparison.Ordinal); Assert.DoesNotContain("USE ", modern, StringComparison.OrdinalIgnoreCase); Assert.Contains("[AX30].dbo.sysobjects", legacy, StringComparison.Ordinal); Assert.DoesNotContain("sys.", legacy, StringComparison.Ordinal); @@ -61,7 +53,7 @@ public sealed class DiscoveryTests } /// 2008+ constructs that make a batch fail on 2005 (as in Sql2005CompatibilityTests). - private static readonly Regex[] Sql2008Only = + internal static readonly Regex[] Sql2008Only = [ new(@"DECLARE\s+@\w+\s+\w+(\s*\([\d\s,]+\))?\s*=", RegexOptions.IgnoreCase), new(@"\bMERGE\s", RegexOptions.IgnoreCase), @@ -77,6 +69,7 @@ public sealed class DiscoveryTests var batches = new[] { DatabaseDiscovery.Databases2005Sql, DatabaseDiscovery.Databases2000Sql, DatabaseDiscovery.DynamicsAxSql("AX30", sql2005.Version), + DynamicsAx.AxDatabaseDetector.DatabasesSql, PermissionCheck.BuildSql(PermissionCheck.ChecksFor(sql2005)), PermissionCheck.DatabasesWithoutAccessSql, }; diff --git a/tests/SQLInsight.Collectors.Tests/DynamicsAxSqlTests.cs b/tests/SQLInsight.Collectors.Tests/DynamicsAxSqlTests.cs new file mode 100644 index 0000000..b56ea18 --- /dev/null +++ b/tests/SQLInsight.Collectors.Tests/DynamicsAxSqlTests.cs @@ -0,0 +1,56 @@ +using SQLInsight.Core.DynamicsAx; + +namespace SQLInsight.Collectors.Tests; + +/// +/// The SQL the AX collectors send, for every version adapter with a schema file: SQL Server 2005 syntax, bounded, +/// read-only, run in the AX database without changing the session's database, and never NOLOCK. +/// +public sealed class DynamicsAxSqlTests +{ + private static string Directory => Path.Combine(AppContext.BaseDirectory, "DynamicsAx"); + + public static TheoryData Adapters() => [.. AxSchemaAdapter.All.Where(a => AxSchemaFile.Find(Directory, a.SchemaFileName) is not null).Select(a => a.SchemaFileName)]; + + private static (AxSchemaAdapter Adapter, AxDatabase Database) Load(string fileName) + { + var adapter = AxSchemaAdapter.All.First(a => a.SchemaFileName == fileName); + var schema = AxSchemaFile.Find(Directory, fileName)!; + var database = AxDetection.Recognise("AXDB", schema.DetectionRows(), ["AXDB", "AXDB_model"])!; + return (adapter, database with { Isolation = AxReadIsolation.Snapshot }); + } + + [Theory] + [MemberData(nameof(Adapters))] + public void Every_ax_query_is_bounded_read_only_sql_server_2005_syntax(string fileName) + { + var (adapter, database) = Load(fileName); + var queries = new[] + { + adapter.BatchJobs(database), adapter.BatchHistory(database, null), adapter.BatchHistory(database, 123_456), + adapter.NumberSequences(database), adapter.RecIds(database), AxSchemaAdapter.Companies(database), + }.Where(q => q is not null).Select(q => q!).ToList(); + + Assert.All(queries, q => Assert.True(q.IsMeasurable, q.NotMeasurable)); + foreach (var sql in queries.Select(q => AxSql.InDatabase(database.Database, q.Sql!, database.Isolation)).Append(AxSql.InDatabase("AXDB", AxDetection.MetadataSql))) + { + Assert.DoesNotContain(DiscoveryTests.Sql2008Only, p => p.IsMatch(sql)); + Assert.DoesNotContain("NOLOCK", sql, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("PXI_", sql, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("USE ", sql, StringComparison.Ordinal); + Assert.DoesNotMatch(@"\b(INSERT|UPDATE|DELETE|MERGE|TRUNCATE|ALTER|CREATE|DROP|EXEC\s*\()\b", sql.Replace("sp_executesql", string.Empty, StringComparison.Ordinal)); + Assert.StartsWith("EXEC [AXDB].sys.sp_executesql", sql, StringComparison.Ordinal); + Assert.Contains("SET LOCK_TIMEOUT 2000", sql, StringComparison.Ordinal); + } + + Assert.All(queries, q => Assert.Contains("SELECT TOP (", q.Sql!, StringComparison.Ordinal)); + } + + [Fact] + public void Reads_use_snapshot_isolation_when_the_database_allows_it_and_read_committed_otherwise() + { + Assert.Contains("ISOLATION LEVEL SNAPSHOT", AxSql.InDatabase("AX", "SELECT 1", AxReadIsolation.Snapshot), StringComparison.Ordinal); + Assert.Contains("ISOLATION LEVEL READ COMMITTED; SELECT 1", AxSql.InDatabase("AX", "SELECT 1", AxReadIsolation.ReadCommittedSnapshot), StringComparison.Ordinal); + Assert.EndsWith("SET LOCK_TIMEOUT -1; SET TRANSACTION ISOLATION LEVEL READ COMMITTED;", AxSql.InDatabase("AX", "SELECT 1"), StringComparison.Ordinal); + } +} diff --git a/tests/SQLInsight.Collectors.Tests/SQLInsight.Collectors.Tests.csproj b/tests/SQLInsight.Collectors.Tests/SQLInsight.Collectors.Tests.csproj index 22a131c..0d2b2fd 100644 --- a/tests/SQLInsight.Collectors.Tests/SQLInsight.Collectors.Tests.csproj +++ b/tests/SQLInsight.Collectors.Tests/SQLInsight.Collectors.Tests.csproj @@ -11,6 +11,10 @@ + + + +