diff --git a/db/permissions/grant-dynamics-ax-read.sql b/db/permissions/grant-dynamics-ax-read.sql new file mode 100644 index 0000000..032bc72 --- /dev/null +++ b/db/permissions/grant-dynamics-ax-read.sql @@ -0,0 +1,101 @@ +/* + SQLInsight Pro: valgfri læseadgang til Dynamics AX' systemtabeller + ===================================================================== + + TILVALG. Agenten læser normalt kun metadata (grant-least-privilege.sql). Med dette script må den desuden læse de + få AX-systemtabeller, som AX-tjekkene bruger, i ÉN navngiven AX-database, så SQLInsight kan advare om batchjob i + fejl, forsinkede eller hængende batchjob, nummerserier og RecId'er der er ved at løbe tør: + + Axapta 3.0 og AX 4.0: BATCH, NUMBERSEQUENCETABLE, NUMBERSEQUENCEREFERENCE, SYSTEMSEQUENCES, SQLDICTIONARY, DATAAREA + AX 2009: BATCHJOB, BATCHJOBHISTORY, NUMBERSEQUENCETABLE, NUMBERSEQUENCEREFERENCE, SYSTEMSEQUENCES, + SQLDICTIONARY, DATAAREA + AX 2012 og F&O: BATCHJOB, BATCHJOBHISTORY, NUMBERSEQUENCETABLE, NUMBERSEQUENCESCOPE, NUMBERSEQUENCEREFERENCE, + SYSTEMSEQUENCES, SQLDICTIONARY, DATAAREA + + Kun SELECT, kun disse tabeller: ingen kundedata (debitorer, ordrer, posteringer), ingen skriverettigheder. Scriptet + finder selv AX-versionen og skemaet, tabellerne ligger i (bmssa i Axapta 3.0, dbo senere). Tabeller, der ikke findes + i databasen, springes over. Scriptet kan køres igen uden skade. Fjern adgangen igen med revoke-dynamics-ax-read.sql. + + SQLInsight Web viser det samme som færdige GRANT-sætninger for hver AX-database (Dynamics AX → AX-databaser). + AX 2009, 2012 og F&O er baseret på Microsofts dokumenterede skema – ikke verificeret mod kundedata. + + Understøtter SQL Server 2005 og nyere. Kør som sysadmin eller som db_owner/db_securityadmin i AX-databasen. +*/ +SET NOCOUNT ON; + +DECLARE @DatabaseName sysname; +DECLARE @LoginName sysname; +SET @DatabaseName = N'AXDB'; -- <== ret til AX-databasen +SET @LoginName = N'DOMÆNE\svc-sqlinsight$'; -- <== ret til agentens login + +IF DB_ID(@DatabaseName) IS NULL +BEGIN + RAISERROR(N'Databasen %s findes ikke.', 16, 1, @DatabaseName); + RETURN; +END; + +IF SUSER_SID(@LoginName) IS NULL +BEGIN + RAISERROR(N'Login %s findes ikke. Kør grant-least-privilege.sql først.', 16, 1, @LoginName); + RETURN; +END; + +DECLARE @Procedure nvarchar(300); +DECLARE @Sql nvarchar(max); +SET @Procedure = QUOTENAME(@DatabaseName) + N'.sys.sp_executesql'; +SET @Sql = N' +SET NOCOUNT ON; +DECLARE @schema sysname; +DECLARE @precision int; +DECLARE @version nvarchar(10); +DECLARE @tables nvarchar(400); +DECLARE @user sysname; +DECLARE @grant nvarchar(max); +DECLARE @count int; + +SELECT TOP 1 @schema = s.name, @precision = c.precision +FROM sys.tables t +JOIN sys.schemas s ON s.schema_id = t.schema_id +LEFT JOIN sys.columns c ON c.object_id = t.object_id AND UPPER(c.name) = N''RECID'' +WHERE UPPER(t.name) = N''SQLDICTIONARY'' AND OBJECT_ID(QUOTENAME(s.name) + N''.DATAAREA'', N''U'') IS NOT NULL +ORDER BY CASE WHEN s.name = N''dbo'' THEN 0 ELSE 1 END, s.name; + +IF @schema IS NULL +BEGIN + DECLARE @database sysname; + SET @database = DB_NAME(); + RAISERROR(N''%s er ikke en Dynamics AX-database (SQLDICTIONARY og DATAAREA findes ikke i samme skema). Intet er ændret.'', 16, 1, @database); + RETURN; +END; + +/* The same version rules as SQLInsight''s detection (Core/DynamicsAx/AxDetection.cs). */ +IF @precision <= 10 SET @version = N''3.0''; +ELSE IF OBJECT_ID(QUOTENAME(@schema) + N''.BATCHJOB'', N''U'') IS NULL SET @version = N''4.0''; +ELSE IF OBJECT_ID(QUOTENAME(@schema) + N''.NUMBERSEQUENCESCOPE'', N''U'') IS NULL SET @version = N''2009''; +ELSE SET @version = N''2012''; + +/* The tables per version: the same lists as the version adapters'' ReadTables (Core/DynamicsAx/AxSchemaAdapter.cs). */ +SET @tables = CASE + WHEN @version IN (N''3.0'', N''4.0'') THEN N''|BATCH|NUMBERSEQUENCETABLE|NUMBERSEQUENCEREFERENCE|SYSTEMSEQUENCES|SQLDICTIONARY|DATAAREA|'' + WHEN @version = N''2009'' THEN N''|BATCHJOB|BATCHJOBHISTORY|NUMBERSEQUENCETABLE|NUMBERSEQUENCEREFERENCE|SYSTEMSEQUENCES|SQLDICTIONARY|DATAAREA|'' + ELSE N''|BATCHJOB|BATCHJOBHISTORY|NUMBERSEQUENCETABLE|NUMBERSEQUENCESCOPE|NUMBERSEQUENCEREFERENCE|SYSTEMSEQUENCES|SQLDICTIONARY|DATAAREA|'' +END; + +SELECT @user = name FROM sys.database_principals WHERE sid = SUSER_SID(@login); +IF @user IS NULL +BEGIN + SET @user = @login; + SET @grant = N''CREATE USER '' + QUOTENAME(@user) + N'' FOR LOGIN '' + QUOTENAME(@login) + N'';''; + EXEC (@grant); + PRINT N''Bruger oprettet uden rettigheder: '' + @user; +END; + +SET @grant = N''''; +SET @count = 0; +SELECT @grant = @grant + N''GRANT SELECT ON '' + QUOTENAME(s.name) + N''.'' + QUOTENAME(t.name) + N'' TO '' + QUOTENAME(@user) + N''; '', @count = @count + 1 +FROM sys.tables t +JOIN sys.schemas s ON s.schema_id = t.schema_id +WHERE s.name = @schema AND CHARINDEX(N''|'' + UPPER(t.name) + N''|'', @tables) > 0; +IF @count > 0 EXEC (@grant); +PRINT N''Dynamics AX '' + @version + N'' i skemaet '' + @schema + N'': SELECT givet på '' + CAST(@count AS nvarchar(10)) + N'' tabeller i '' + DB_NAME() + N'' til '' + @user + N''.'';'; +EXEC @Procedure @Sql, N'@login sysname', @login = @LoginName; diff --git a/db/permissions/revoke-dynamics-ax-read.sql b/db/permissions/revoke-dynamics-ax-read.sql new file mode 100644 index 0000000..0e3011d --- /dev/null +++ b/db/permissions/revoke-dynamics-ax-read.sql @@ -0,0 +1,48 @@ +/* + SQLInsight Pro: fjern den valgfri læseadgang til Dynamics AX' systemtabeller + ============================================================================== + + Fjerner det, grant-dynamics-ax-read.sql gav, i én AX-database: SELECT på AX' batch-, nummerserie-, RecId- og + firmatabeller. Agentens database-bruger slettes ikke (den kan være oprettet af grant-least-privilege.sql). Bagefter + viser SQLInsight AX-datatjekkene som "ikke målbar" med den GRANT, der skal til for at slå dem til igen. + + Understøtter SQL Server 2005 og nyere. Kør som sysadmin eller som db_owner/db_securityadmin i AX-databasen. +*/ +SET NOCOUNT ON; + +DECLARE @DatabaseName sysname; +DECLARE @LoginName sysname; +SET @DatabaseName = N'AXDB'; -- <== ret til AX-databasen +SET @LoginName = N'DOMÆNE\svc-sqlinsight$'; -- <== ret til agentens login + +IF DB_ID(@DatabaseName) IS NULL +BEGIN + RAISERROR(N'Databasen %s findes ikke.', 16, 1, @DatabaseName); + RETURN; +END; + +DECLARE @Procedure nvarchar(300); +DECLARE @Sql nvarchar(max); +SET @Procedure = QUOTENAME(@DatabaseName) + N'.sys.sp_executesql'; +SET @Sql = N' +SET NOCOUNT ON; +DECLARE @user sysname; +DECLARE @revoke nvarchar(max); +DECLARE @count int; +SELECT @user = name FROM sys.database_principals WHERE sid = SUSER_SID(@login); +IF @user IS NULL +BEGIN + PRINT N''Login''''et har ingen bruger i '' + DB_NAME() + N''. Intet at fjerne.''; + RETURN; +END; + +SET @revoke = N''''; +SET @count = 0; +SELECT @revoke = @revoke + N''REVOKE SELECT ON '' + QUOTENAME(s.name) + N''.'' + QUOTENAME(t.name) + N'' FROM '' + QUOTENAME(@user) + N''; '', @count = @count + 1 +FROM sys.tables t +JOIN sys.schemas s ON s.schema_id = t.schema_id +WHERE UPPER(t.name) IN (N''BATCH'', N''BATCHJOB'', N''BATCHJOBHISTORY'', N''NUMBERSEQUENCETABLE'', N''NUMBERSEQUENCESCOPE'', + N''NUMBERSEQUENCEREFERENCE'', N''SYSTEMSEQUENCES'', N''SQLDICTIONARY'', N''DATAAREA''); +IF @count > 0 EXEC (@revoke); +PRINT N''SELECT fjernet på '' + CAST(@count AS nvarchar(10)) + N'' AX-tabeller i '' + DB_NAME() + N'' for '' + @user + N''.'';'; +EXEC @Procedure @Sql, N'@login sysname', @login = @LoginName; diff --git a/docs/dynamics-ax/schema-ax2009.documented.tsv b/docs/dynamics-ax/schema-ax2009.documented.tsv new file mode 100644 index 0000000..beafa87 --- /dev/null +++ b/docs/dynamics-ax/schema-ax2009.documented.tsv @@ -0,0 +1,129 @@ +skema tabel kolonne type max_length column_id +# Baseret på Microsofts dokumenterede skema – ikke verificeret mod kundedata. Erstattes af schema-ax2009.tsv, når en kunde leverer det rigtige skema. +dbo SQLDICTIONARY TABLEID int 4 1 +dbo SQLDICTIONARY FIELDID int 4 2 +dbo SQLDICTIONARY ARRAY int 4 3 +dbo SQLDICTIONARY NAME nvarchar 80 4 +dbo SQLDICTIONARY SQLNAME nvarchar 80 5 +dbo SQLDICTIONARY FIELDTYPE int 4 6 +dbo SQLDICTIONARY STRSIZE int 4 7 +dbo SQLDICTIONARY SHADOW int 4 8 +dbo SQLDICTIONARY RIGHTJUSTIFY int 4 9 +dbo SQLDICTIONARY NULLABLE int 4 10 +dbo SQLDICTIONARY FLAGS int 4 11 +dbo SQLDICTIONARY RECVERSION int 4 12 +dbo SQLDICTIONARY RECID bigint 8 13 +dbo DATAAREA ID nvarchar 8 1 +dbo DATAAREA NAME nvarchar 120 2 +dbo DATAAREA ISVIRTUAL int 4 3 +dbo DATAAREA ALWAYSNATIVE int 4 4 +dbo DATAAREA TIMEZONE int 4 5 +dbo DATAAREA RECVERSION int 4 6 +dbo DATAAREA RECID bigint 8 7 +dbo SYSCONFIG CONFIGTYPE int 4 1 +dbo SYSCONFIG ID int 4 2 +dbo SYSCONFIG VALUE nvarchar 160 3 +dbo SYSCONFIG MODIFIEDDATETIME datetime 8 4 +dbo SYSCONFIG MODIFIEDBY nvarchar 10 5 +dbo SYSCONFIG CREATEDDATETIME datetime 8 6 +dbo SYSCONFIG CREATEDBY nvarchar 10 7 +dbo SYSCONFIG RECVERSION int 4 8 +dbo SYSCONFIG RECID bigint 8 9 +dbo BATCHJOB CAPTION nvarchar 200 1 +dbo BATCHJOB STATUS int 4 2 +dbo BATCHJOB ORIGSTARTDATETIME datetime 8 3 +dbo BATCHJOB ORIGSTARTDATETIMETZID int 4 4 +dbo BATCHJOB STARTDATETIME datetime 8 5 +dbo BATCHJOB ENDDATETIME datetime 8 6 +dbo BATCHJOB RECURRENCEDATA image 16 7 +dbo BATCHJOB ALERTSPROCESSED int 4 8 +dbo BATCHJOB CANCELEDBY nvarchar 10 9 +dbo BATCHJOB COMPANY nvarchar 8 10 +dbo BATCHJOB LOGLEVEL int 4 11 +dbo BATCHJOB BATCHCREATEDBY nvarchar 10 12 +dbo BATCHJOB MODIFIEDDATETIME datetime 8 13 +dbo BATCHJOB MODIFIEDBY nvarchar 10 14 +dbo BATCHJOB CREATEDDATETIME datetime 8 15 +dbo BATCHJOB CREATEDBY nvarchar 10 16 +dbo BATCHJOB RECVERSION int 4 17 +dbo BATCHJOB RECID bigint 8 18 +dbo BATCHJOBHISTORY BATCHJOBID bigint 8 1 +dbo BATCHJOBHISTORY CAPTION nvarchar 200 2 +dbo BATCHJOBHISTORY STATUS int 4 3 +dbo BATCHJOBHISTORY ORIGSTARTDATETIME datetime 8 4 +dbo BATCHJOBHISTORY STARTDATETIME datetime 8 5 +dbo BATCHJOBHISTORY ENDDATETIME datetime 8 6 +dbo BATCHJOBHISTORY COMPANY nvarchar 8 7 +dbo BATCHJOBHISTORY CANCELEDBY nvarchar 10 8 +dbo BATCHJOBHISTORY BATCHCREATEDBY nvarchar 10 9 +dbo BATCHJOBHISTORY RECVERSION int 4 10 +dbo BATCHJOBHISTORY RECID bigint 8 11 +dbo BATCH BATCHJOBID bigint 8 1 +dbo BATCH GROUPID nvarchar 20 2 +dbo BATCH STATUS int 4 3 +dbo BATCH CLASSNUMBER int 4 4 +dbo BATCH CAPTION nvarchar 200 5 +dbo BATCH STARTDATETIME datetime 8 6 +dbo BATCH ENDDATETIME datetime 8 7 +dbo BATCH SERVERID nvarchar 40 8 +dbo BATCH COMPANY nvarchar 8 9 +dbo BATCH PARAMETERS image 16 10 +dbo BATCH INFO image 16 11 +dbo BATCH SESSIONIDX int 4 12 +dbo BATCH SESSIONLOGINDATETIME datetime 8 13 +dbo BATCH RECVERSION int 4 14 +dbo BATCH RECID bigint 8 15 +dbo BATCHGROUP GROUP_ nvarchar 20 1 +dbo BATCHGROUP DESCRIPTION nvarchar 120 2 +dbo BATCHGROUP RECVERSION int 4 3 +dbo BATCHGROUP RECID bigint 8 4 +dbo NUMBERSEQUENCETABLE NUMBERSEQUENCE nvarchar 40 1 +dbo NUMBERSEQUENCETABLE TXT nvarchar 120 2 +dbo NUMBERSEQUENCETABLE LOWEST int 4 3 +dbo NUMBERSEQUENCETABLE HIGHEST int 4 4 +dbo NUMBERSEQUENCETABLE NEXTREC int 4 5 +dbo NUMBERSEQUENCETABLE BLOCKED int 4 6 +dbo NUMBERSEQUENCETABLE FORMAT nvarchar 40 7 +dbo NUMBERSEQUENCETABLE CONTINUOUS int 4 8 +dbo NUMBERSEQUENCETABLE CYCLIC int 4 9 +dbo NUMBERSEQUENCETABLE CLEANATACCESS int 4 10 +dbo NUMBERSEQUENCETABLE INUSE int 4 11 +dbo NUMBERSEQUENCETABLE NOINCREMENT int 4 12 +dbo NUMBERSEQUENCETABLE CLEANINTERVAL numeric 13 13 +dbo NUMBERSEQUENCETABLE LATESTCLEANDATETIME datetime 8 14 +dbo NUMBERSEQUENCETABLE ALLOWCHANGEUP int 4 15 +dbo NUMBERSEQUENCETABLE ALLOWCHANGEDOWN int 4 16 +dbo NUMBERSEQUENCETABLE MANUAL int 4 17 +dbo NUMBERSEQUENCETABLE FETCHAHEADQTY int 4 18 +dbo NUMBERSEQUENCETABLE FETCHAHEAD int 4 19 +dbo NUMBERSEQUENCETABLE DATAAREAID nvarchar 8 20 +dbo NUMBERSEQUENCETABLE RECVERSION int 4 21 +dbo NUMBERSEQUENCETABLE RECID bigint 8 22 +dbo NUMBERSEQUENCEREFERENCE NUMBERSEQUENCE nvarchar 40 1 +dbo NUMBERSEQUENCEREFERENCE DATATYPEID int 4 2 +dbo NUMBERSEQUENCEREFERENCE CONFIGURATIONKEYID int 4 3 +dbo NUMBERSEQUENCEREFERENCE MODULE int 4 4 +dbo NUMBERSEQUENCEREFERENCE REFERENCEHELP nvarchar 510 5 +dbo NUMBERSEQUENCEREFERENCE REFERENCELABEL nvarchar 120 6 +dbo NUMBERSEQUENCEREFERENCE DATATYPESAMEASID int 4 7 +dbo NUMBERSEQUENCEREFERENCE WIZARDCONTINUOUS int 4 8 +dbo NUMBERSEQUENCEREFERENCE ALLOWSAMEAS int 4 9 +dbo NUMBERSEQUENCEREFERENCE SORTFIELD int 4 10 +dbo NUMBERSEQUENCEREFERENCE GROUPENABLED int 4 11 +dbo NUMBERSEQUENCEREFERENCE WIZARDHIGHEST int 4 12 +dbo NUMBERSEQUENCEREFERENCE WIZARDLOWEST int 4 13 +dbo NUMBERSEQUENCEREFERENCE WIZARDMANUAL int 4 14 +dbo NUMBERSEQUENCEREFERENCE WIZARDFETCHAHEADQTY int 4 15 +dbo NUMBERSEQUENCEREFERENCE DATAAREAID nvarchar 8 16 +dbo NUMBERSEQUENCEREFERENCE RECVERSION int 4 17 +dbo NUMBERSEQUENCEREFERENCE RECID bigint 8 18 +dbo SYSTEMSEQUENCES ID int 4 1 +dbo SYSTEMSEQUENCES NEXTVAL bigint 8 2 +dbo SYSTEMSEQUENCES MINVAL bigint 8 3 +dbo SYSTEMSEQUENCES MAXVAL bigint 8 4 +dbo SYSTEMSEQUENCES CYCLE int 4 5 +dbo SYSTEMSEQUENCES NAME nvarchar 40 6 +dbo SYSTEMSEQUENCES TABID int 4 7 +dbo SYSTEMSEQUENCES DATAAREAID nvarchar 8 8 +dbo SYSTEMSEQUENCES RECVERSION int 4 9 +dbo SYSTEMSEQUENCES RECID bigint 8 10 diff --git a/docs/dynamics-ax/schema-ax2012.documented.tsv b/docs/dynamics-ax/schema-ax2012.documented.tsv new file mode 100644 index 0000000..1ad169c --- /dev/null +++ b/docs/dynamics-ax/schema-ax2012.documented.tsv @@ -0,0 +1,135 @@ +skema tabel kolonne type max_length column_id +# Baseret på Microsofts dokumenterede skema – ikke verificeret mod kundedata. Erstattes af schema-ax2012.tsv, når en kunde leverer det rigtige skema. +dbo SQLDICTIONARY TABLEID int 4 1 +dbo SQLDICTIONARY FIELDID int 4 2 +dbo SQLDICTIONARY ARRAY int 4 3 +dbo SQLDICTIONARY NAME nvarchar 80 4 +dbo SQLDICTIONARY SQLNAME nvarchar 80 5 +dbo SQLDICTIONARY FIELDTYPE int 4 6 +dbo SQLDICTIONARY STRSIZE int 4 7 +dbo SQLDICTIONARY SHADOW int 4 8 +dbo SQLDICTIONARY RIGHTJUSTIFY int 4 9 +dbo SQLDICTIONARY NULLABLE int 4 10 +dbo SQLDICTIONARY FLAGS int 4 11 +dbo SQLDICTIONARY RECVERSION int 4 12 +dbo SQLDICTIONARY RECID bigint 8 13 +dbo DATAAREA ID nvarchar 8 1 +dbo DATAAREA NAME nvarchar 120 2 +dbo DATAAREA ISVIRTUAL int 4 3 +dbo DATAAREA ALWAYSNATIVE int 4 4 +dbo DATAAREA TIMEZONE int 4 5 +dbo DATAAREA RECVERSION int 4 6 +dbo DATAAREA PARTITION bigint 8 7 +dbo DATAAREA RECID bigint 8 8 +dbo SYSCONFIG CONFIGTYPE int 4 1 +dbo SYSCONFIG ID int 4 2 +dbo SYSCONFIG VALUE nvarchar 160 3 +dbo SYSCONFIG MODIFIEDDATETIME datetime 8 4 +dbo SYSCONFIG MODIFIEDBY nvarchar 16 5 +dbo SYSCONFIG CREATEDDATETIME datetime 8 6 +dbo SYSCONFIG CREATEDBY nvarchar 16 7 +dbo SYSCONFIG RECVERSION int 4 8 +dbo SYSCONFIG PARTITION bigint 8 9 +dbo SYSCONFIG RECID bigint 8 10 +dbo BATCHJOB CAPTION nvarchar 200 1 +dbo BATCHJOB STATUS int 4 2 +dbo BATCHJOB ORIGSTARTDATETIME datetime 8 3 +dbo BATCHJOB ORIGSTARTDATETIMETZID int 4 4 +dbo BATCHJOB STARTDATETIME datetime 8 5 +dbo BATCHJOB ENDDATETIME datetime 8 6 +dbo BATCHJOB RECURRENCEDATA image 16 7 +dbo BATCHJOB ALERTSPROCESSED int 4 8 +dbo BATCHJOB CANCELEDBY nvarchar 16 9 +dbo BATCHJOB COMPANY nvarchar 8 10 +dbo BATCHJOB LOGLEVEL int 4 11 +dbo BATCHJOB BATCHCREATEDBY nvarchar 16 12 +dbo BATCHJOB RUNTIMEJOB int 4 13 +dbo BATCHJOB MODIFIEDDATETIME datetime 8 14 +dbo BATCHJOB MODIFIEDBY nvarchar 16 15 +dbo BATCHJOB CREATEDDATETIME datetime 8 16 +dbo BATCHJOB CREATEDBY nvarchar 16 17 +dbo BATCHJOB RECVERSION int 4 18 +dbo BATCHJOB PARTITION bigint 8 19 +dbo BATCHJOB RECID bigint 8 20 +dbo BATCHJOBHISTORY BATCHJOBID bigint 8 1 +dbo BATCHJOBHISTORY CAPTION nvarchar 200 2 +dbo BATCHJOBHISTORY STATUS int 4 3 +dbo BATCHJOBHISTORY ORIGSTARTDATETIME datetime 8 4 +dbo BATCHJOBHISTORY STARTDATETIME datetime 8 5 +dbo BATCHJOBHISTORY ENDDATETIME datetime 8 6 +dbo BATCHJOBHISTORY COMPANY nvarchar 8 7 +dbo BATCHJOBHISTORY CANCELEDBY nvarchar 16 8 +dbo BATCHJOBHISTORY BATCHCREATEDBY nvarchar 16 9 +dbo BATCHJOBHISTORY RECVERSION int 4 10 +dbo BATCHJOBHISTORY PARTITION bigint 8 11 +dbo BATCHJOBHISTORY RECID bigint 8 12 +dbo BATCH BATCHJOBID bigint 8 1 +dbo BATCH GROUPID nvarchar 20 2 +dbo BATCH STATUS int 4 3 +dbo BATCH CLASSNUMBER int 4 4 +dbo BATCH CAPTION nvarchar 200 5 +dbo BATCH STARTDATETIME datetime 8 6 +dbo BATCH ENDDATETIME datetime 8 7 +dbo BATCH SERVERID nvarchar 40 8 +dbo BATCH COMPANY nvarchar 8 9 +dbo BATCH PARAMETERS image 16 10 +dbo BATCH INFO image 16 11 +dbo BATCH SESSIONIDX int 4 12 +dbo BATCH SESSIONLOGINDATETIME datetime 8 13 +dbo BATCH RETRYCOUNT int 4 14 +dbo BATCH RECVERSION int 4 15 +dbo BATCH PARTITION bigint 8 16 +dbo BATCH RECID bigint 8 17 +dbo BATCHGROUP GROUP_ nvarchar 20 1 +dbo BATCHGROUP DESCRIPTION nvarchar 120 2 +dbo BATCHGROUP RECVERSION int 4 3 +dbo BATCHGROUP PARTITION bigint 8 4 +dbo BATCHGROUP RECID bigint 8 5 +dbo NUMBERSEQUENCETABLE NUMBERSEQUENCE nvarchar 40 1 +dbo NUMBERSEQUENCETABLE TXT nvarchar 120 2 +dbo NUMBERSEQUENCETABLE LOWEST bigint 8 3 +dbo NUMBERSEQUENCETABLE HIGHEST bigint 8 4 +dbo NUMBERSEQUENCETABLE NEXTREC bigint 8 5 +dbo NUMBERSEQUENCETABLE BLOCKED int 4 6 +dbo NUMBERSEQUENCETABLE FORMAT nvarchar 80 7 +dbo NUMBERSEQUENCETABLE ANNOTATEDFORMAT nvarchar 80 8 +dbo NUMBERSEQUENCETABLE CONTINUOUS int 4 9 +dbo NUMBERSEQUENCETABLE CYCLIC int 4 10 +dbo NUMBERSEQUENCETABLE CLEANATACCESS int 4 11 +dbo NUMBERSEQUENCETABLE INUSE int 4 12 +dbo NUMBERSEQUENCETABLE NOINCREMENT int 4 13 +dbo NUMBERSEQUENCETABLE CLEANINTERVAL numeric 13 14 +dbo NUMBERSEQUENCETABLE LATESTCLEANDATETIME datetime 8 15 +dbo NUMBERSEQUENCETABLE ALLOWCHANGEUP int 4 16 +dbo NUMBERSEQUENCETABLE ALLOWCHANGEDOWN int 4 17 +dbo NUMBERSEQUENCETABLE MANUAL int 4 18 +dbo NUMBERSEQUENCETABLE FETCHAHEADQTY int 4 19 +dbo NUMBERSEQUENCETABLE FETCHAHEAD int 4 20 +dbo NUMBERSEQUENCETABLE NUMBERSEQUENCESCOPE bigint 8 21 +dbo NUMBERSEQUENCETABLE RECVERSION int 4 22 +dbo NUMBERSEQUENCETABLE PARTITION bigint 8 23 +dbo NUMBERSEQUENCETABLE RECID bigint 8 24 +dbo NUMBERSEQUENCESCOPE DATAAREA nvarchar 8 1 +dbo NUMBERSEQUENCESCOPE LEGALENTITY bigint 8 2 +dbo NUMBERSEQUENCESCOPE OPERATINGUNIT bigint 8 3 +dbo NUMBERSEQUENCESCOPE OPERATINGUNITTYPE int 4 4 +dbo NUMBERSEQUENCESCOPE FISCALCALENDARPERIOD bigint 8 5 +dbo NUMBERSEQUENCESCOPE RECVERSION int 4 6 +dbo NUMBERSEQUENCESCOPE PARTITION bigint 8 7 +dbo NUMBERSEQUENCESCOPE RECID bigint 8 8 +dbo NUMBERSEQUENCEREFERENCE NUMBERSEQUENCEDATATYPE bigint 8 1 +dbo NUMBERSEQUENCEREFERENCE NUMBERSEQUENCEID bigint 8 2 +dbo NUMBERSEQUENCEREFERENCE NUMBERSEQUENCESCOPE bigint 8 3 +dbo NUMBERSEQUENCEREFERENCE RECVERSION int 4 4 +dbo NUMBERSEQUENCEREFERENCE PARTITION bigint 8 5 +dbo NUMBERSEQUENCEREFERENCE RECID bigint 8 6 +dbo SYSTEMSEQUENCES ID int 4 1 +dbo SYSTEMSEQUENCES NEXTVAL bigint 8 2 +dbo SYSTEMSEQUENCES MINVAL bigint 8 3 +dbo SYSTEMSEQUENCES MAXVAL bigint 8 4 +dbo SYSTEMSEQUENCES CYCLE int 4 5 +dbo SYSTEMSEQUENCES NAME nvarchar 40 6 +dbo SYSTEMSEQUENCES TABID int 4 7 +dbo SYSTEMSEQUENCES DATAAREAID nvarchar 8 8 +dbo SYSTEMSEQUENCES RECVERSION int 4 9 +dbo SYSTEMSEQUENCES RECID bigint 8 10 diff --git a/src/SQLInsight.Core/Configuration/DatabaseSelection.cs b/src/SQLInsight.Core/Configuration/DatabaseSelection.cs index 2d4d157..0a05ed4 100644 --- a/src/SQLInsight.Core/Configuration/DatabaseSelection.cs +++ b/src/SQLInsight.Core/Configuration/DatabaseSelection.cs @@ -68,15 +68,34 @@ public sealed class DatabaseSelection /// public static class DynamicsAxFindings { + /// The community AX rules (rules/community/dynamics-ax.json). public const string RulePrefix = "COM-AX-"; + /// The built-in AX rules (1.4: batch jobs, number sequences, RecId, AX users behind blocking). + public const string BuiltInRulePrefix = "SQLI-AX-"; + private const string DatabaseSubject = "database: "; - public static bool IsDynamicsAxRule(string ruleId) => ruleId?.StartsWith(RulePrefix, StringComparison.OrdinalIgnoreCase) == true; + /// Separates the database from the rest of an AX finding's subject: database: AX30 · batch: 4711. + public const string SubjectSeparator = " · "; - /// The database a finding is about, from its subject (database: AX30), or null. - public static string? DatabaseOf(string? subject) => - subject is not null && subject.StartsWith(DatabaseSubject, StringComparison.OrdinalIgnoreCase) && subject.Length > DatabaseSubject.Length - ? subject[DatabaseSubject.Length..] - : null; + public static bool IsDynamicsAxRule(string ruleId) => + ruleId?.StartsWith(RulePrefix, StringComparison.OrdinalIgnoreCase) == true || ruleId?.StartsWith(BuiltInRulePrefix, StringComparison.OrdinalIgnoreCase) == true; + + /// The subject of an AX finding about in . + public static string Subject(string database, string? detail = null) => + detail is null ? DatabaseSubject + database : DatabaseSubject + database + SubjectSeparator + detail; + + /// The database a finding is about, from its subject (database: AX30 or database: AX30 · …), or null. + public static string? DatabaseOf(string? subject) + { + if (subject is null || !subject.StartsWith(DatabaseSubject, StringComparison.OrdinalIgnoreCase) || subject.Length <= DatabaseSubject.Length) + { + return null; + } + + var rest = subject[DatabaseSubject.Length..]; + var separator = rest.IndexOf(SubjectSeparator, StringComparison.Ordinal); + return separator > 0 ? rest[..separator] : rest; + } } diff --git a/src/SQLInsight.Core/Data/Blocking.cs b/src/SQLInsight.Core/Data/Blocking.cs index 0245383..c38d38a 100644 --- a/src/SQLInsight.Core/Data/Blocking.cs +++ b/src/SQLInsight.Core/Data/Blocking.cs @@ -45,4 +45,10 @@ public sealed record SessionInfo public string? SqlText { get; init; } public DateTime? LastRequestEndTime { get; init; } + + /// + /// The session's CONTEXT_INFO as hex, without trailing zero bytes, or null when not set. Dynamics AX 2009/2012 AOS + /// can put the AX user and session there (Core/DynamicsAx/AxSessionMapper.cs). Sensitive: it can hold a user id. + /// + public string? ContextInfo { get; init; } } diff --git a/src/SQLInsight.Core/DynamicsAx/AxBatch.cs b/src/SQLInsight.Core/DynamicsAx/AxBatch.cs new file mode 100644 index 0000000..296498f --- /dev/null +++ b/src/SQLInsight.Core/DynamicsAx/AxBatch.cs @@ -0,0 +1,216 @@ +using System.Text.Json.Serialization; + +namespace SQLInsight.Core.DynamicsAx; + +/// A batch job's status, translated from the version's STATUS enum by its adapter. +public enum AxBatchStatus +{ + Unknown, + Hold, + Waiting, + Executing, + Error, + Finished, + Ready, + NotRun, + Cancelling, + Canceled, +} + +/// +/// A batch job row as read from AX, before interpretation: dates and times exactly as stored. AX 3.0/4.0 store a local +/// date (time part 00:00) plus seconds since midnight; AX 2009 and later a UTC datetime and no seconds. +/// +/// DATEDIFF(minute, GETDATE(), GETUTCDATE()) on the server when read: turns local AX times into UTC. +public sealed record AxBatchJobRow( + long RecId, + int? ClassNumber, + string? Caption, + string? Company, + string? Group, + int Status, + DateTime? PlannedDate, + int? PlannedSeconds, + DateTime? StartDate, + int? StartSeconds, + DateTime? EndDate, + int? EndSeconds, + int ServerUtcOffsetMinutes); + +/// A finished run from AX's batch history (BATCHJOBHISTORY, 2009 and later). +public sealed record AxBatchHistoryRow( + long RecId, + long? JobRecId, + string? Caption, + string? Company, + int Status, + DateTime? StartDate, + DateTime? EndDate, + int ServerUtcOffsetMinutes); + +/// A completed run of a batch job: from AX's history, or seen by SQLInsight itself. Stored in insight.AxBatchRuns. +/// Unique per database, so a run seen twice is stored once. +public sealed record AxBatchRun(string JobKey, string RunKey, string? Caption, string? Company, DateTime StartedUtc, DateTime EndedUtc, AxBatchStatus Status) +{ + [JsonIgnore] + public TimeSpan Duration => EndedUtc - StartedUtc; +} + +/// A batch job now, with what its own history says is normal. +public sealed record AxBatchJob +{ + /// Stable per job: the BATCHJOB RECID (2009 and later), or class, company and group (3.0/4.0). + public required string Key { get; init; } + + public long RecId { get; init; } + public required string Caption { get; init; } + public string? Company { get; init; } + public string? Group { get; init; } + public int StatusCode { get; init; } + public AxBatchStatus Status { get; init; } + public DateTime? PlannedStartUtc { get; init; } + public DateTime? StartedUtc { get; init; } + public DateTime? EndedUtc { get; init; } + + /// When SQLInsight first saw the job executing (AX 3.0/4.0 do not store the actual start of a running job). + public DateTime? FirstSeenExecutingUtc { get; init; } + + /// Completed runs in SQLInsight's history of the job (at most ). + public int HistoryRuns { get; init; } + + public double? MedianSeconds { get; init; } + public double? P90Seconds { get; init; } + public double? LastRunSeconds { get; init; } + public AxBatchStatus? LastRunStatus { get; init; } + public DateTime? LastRunEndedUtc { get; init; } + + /// When the job started running, as well as SQLInsight knows it. + [JsonIgnore] + public DateTime? RunningSince => Status == AxBatchStatus.Executing ? StartedUtc ?? FirstSeenExecutingUtc : null; + + public TimeSpan? Running(DateTimeOffset now) => RunningSince is { } since ? now.UtcDateTime - since : null; + + /// How late a waiting job is: now minus its planned start, when positive. + public TimeSpan? Late(DateTimeOffset now) => + Status == AxBatchStatus.Waiting && PlannedStartUtc is { } planned && now.UtcDateTime > planned ? now.UtcDateTime - planned : null; +} + +/// Batch jobs in one AX database, or why they could not be read. +public sealed record AxBatchDatabase +{ + public required string Database { get; init; } + public AxVersion Version { get; init; } + + /// Null when measured; else "Ikke målbar: …" with the exact GRANT text when it is the read access. + public string? NotMeasurable { get; init; } + + public IReadOnlyList Jobs { get; init; } = []; + + /// Runs completed since the previous collection (stored in insight.AxBatchRuns). + public IReadOnlyList NewRuns { get; init; } = []; + + /// + /// How many jobs had each raw STATUS value: shows whether the version's status mapping fits the customer's data + /// (the AX 3.0 mapping is not verified, see docs/dynamics-ax.md). + /// + public IReadOnlyDictionary StatusCounts { get; init; } = new Dictionary(); + + /// Optional columns missing and what that leaves out. + public IReadOnlyList Notes { get; init; } = []; + + /// The highest BATCHJOBHISTORY RECID read: the next read starts there (2009 and later). + public long? HistoryWatermark { get; init; } +} + +/// The batch jobs of every monitored AX database on an instance (collector "dynamics-ax-batch", every 5 minutes). +public sealed record AxBatchSnapshot +{ + public required DateTimeOffset CapturedAt { get; init; } + public required AxThresholds Thresholds { get; init; } + public required IReadOnlyList Databases { get; init; } + + /// + /// Durations of recent runs per database and job, for comparing a job with itself. Kept in memory and rebuilt from + /// insight.AxBatchRuns after a restart; not part of the stored snapshot. + /// + [JsonIgnore] + public AxBatchHistory History { get; init; } = new(); + + /// When each running job was first seen executing, per database and job. Kept in memory. + [JsonIgnore] + public IReadOnlyDictionary FirstSeenExecuting { get; init; } = new Dictionary(); + + /// Run keys stored lately, so a run read again is not stored twice. Kept in memory. + [JsonIgnore] + public IReadOnlySet RecentRunKeys { get; init; } = new HashSet(); +} + +/// Recent run durations per database and job; bounded to runs per job. +public sealed class AxBatchHistory +{ + public const int RunsKept = 20; + + private readonly Dictionary> _runs = new(StringComparer.OrdinalIgnoreCase); + + public static string Key(string database, string jobKey) => database + "\u001f" + jobKey; + + public void Add(string database, AxBatchRun run) + { + ArgumentNullException.ThrowIfNull(run); + var key = Key(database, run.JobKey); + if (!_runs.TryGetValue(key, out var list)) + { + _runs[key] = list = []; + } + + if (list.Any(r => r.RunKey == run.RunKey)) + { + return; + } + + list.Add(run); + list.Sort((a, b) => a.EndedUtc.CompareTo(b.EndedUtc)); + if (list.Count > RunsKept) + { + list.RemoveRange(0, list.Count - RunsKept); + } + } + + /// The job's runs, oldest first. + public IReadOnlyList Runs(string database, string jobKey) => + _runs.TryGetValue(Key(database, jobKey), out var list) ? list : []; + + public int Count => _runs.Count; + + /// A copy, so the previous snapshot is never changed. + public AxBatchHistory Clone() + { + var copy = new AxBatchHistory(); + foreach (var (key, list) in _runs) + { + copy._runs[key] = [.. list]; + } + + return copy; + } + + /// Median and 90th percentile of the successful runs' durations, in seconds. + public static (double? Median, double? P90) Statistics(IEnumerable runs) + { + var seconds = runs.Where(r => r.Status == AxBatchStatus.Finished).Select(r => r.Duration.TotalSeconds).Where(s => s >= 0).Order().ToList(); + if (seconds.Count == 0) + { + return (null, null); + } + + return (Percentile(seconds, 0.5), Percentile(seconds, 0.9)); + } + + private static double Percentile(List sorted, double p) + { + var position = p * (sorted.Count - 1); + var lower = (int)Math.Floor(position); + var upper = Math.Min(lower + 1, sorted.Count - 1); + return sorted[lower] + (sorted[upper] - sorted[lower]) * (position - lower); + } +} diff --git a/src/SQLInsight.Core/DynamicsAx/AxDatabase.cs b/src/SQLInsight.Core/DynamicsAx/AxDatabase.cs new file mode 100644 index 0000000..67306e1 --- /dev/null +++ b/src/SQLInsight.Core/DynamicsAx/AxDatabase.cs @@ -0,0 +1,99 @@ +namespace SQLInsight.Core.DynamicsAx; + +/// A column of an AX table as sys.columns describes it. +public sealed record AxColumn(string Name, string Type, int Precision, int MaxLength); + +/// The agent has SELECT on the table (the optional AX read access, grant-dynamics-ax-read.sql). +public sealed record AxTable(string Name, bool Readable, IReadOnlyList Columns) +{ + public bool HasColumn(string column) => Columns.Any(c => string.Equals(c.Name, column, StringComparison.OrdinalIgnoreCase)); + + public AxColumn? Column(string column) => Columns.FirstOrDefault(c => string.Equals(c.Name, column, StringComparison.OrdinalIgnoreCase)); +} + +/// A company (DATAAREA.ID) in an AX database. +public sealed record AxCompany(string Id, string? Name, bool IsVirtual); + +/// +/// A database recognised as Dynamics AX by the central detection (): which version, which +/// schema owns the AX tables (bmssa in AX 3.0, dbo later), the AX tables SQLInsight knows with their columns and +/// whether the agent may read them, and the companies when DATAAREA is readable. +/// +public sealed record AxDatabase +{ + public required string Database { get; init; } + + /// The schema of the AX tables: bmssa (AX 3.0, usually), dbo (later). + public required string Schema { get; init; } + + public required AxVersion Version { get; init; } + + /// The AX tables SQLInsight looks at () found in . + public IReadOnlyList Tables { get; init; } = []; + + /// The separate model store database (<database>_model) of AX 2012 R2/R3, when it is on the instance. + public string? ModelDatabase { get; init; } + + /// The companies (DATAAREA), or null when the agent may not read DATAAREA. + public IReadOnlyList? Companies { get; init; } + + /// Why is null. + public string? CompaniesReason { get; init; } + + /// The database user the agent's login is in this database, or null when it has none (CONNECT ANY DATABASE). + public string? DatabaseUser { get; init; } + + /// The agent's login (SUSER_SNAME()), so the GRANT text names it exactly. + public string? AgentLogin { get; init; } + + /// Reads use snapshot isolation (ALLOW_SNAPSHOT_ISOLATION) or read committed snapshot; else read committed with a short lock timeout. + public AxReadIsolation Isolation { get; init; } = AxReadIsolation.ReadCommitted; + + /// How the version was decided, and anything uncertain about it (Danish). + public IReadOnlyList Notes { get; init; } = []; + + public string VersionText => AxVersions.Text(Version); + + public bool Verified => AxVersions.IsVerified(Version); + + public AxTable? Table(string name) => Tables.FirstOrDefault(t => string.Equals(t.Name, name, StringComparison.OrdinalIgnoreCase)); + + public bool HasTable(string name) => Table(name) is not null; + + public bool HasColumn(string table, string column) => Table(table)?.HasColumn(column) == true; + + public bool CanRead(string table) => Table(table)?.Readable == true; + + /// [schema].[TABLE], quoted. + public string Qualified(string table) => AxSql.Identifier(Schema) + "." + AxSql.Identifier(table); +} + +public enum AxReadIsolation +{ + /// Read committed with SET LOCK_TIMEOUT: a locked row makes the read give up after a few seconds, never wait. + ReadCommitted, + + /// The database has READ_COMMITTED_SNAPSHOT on: read committed reads versions and never waits for locks. + ReadCommittedSnapshot, + + /// ALLOW_SNAPSHOT_ISOLATION is on: reads use SET TRANSACTION ISOLATION LEVEL SNAPSHOT. + Snapshot, +} + +/// The AX databases on an instance, as the detector last saw them (collector "dynamics-ax", hourly). +public sealed record AxDatabaseSnapshot +{ + public required DateTimeOffset CapturedAt { get; init; } + + /// Recognised AX databases that are monitored (chosen in Servere og databaser). + public required IReadOnlyList Databases { get; init; } + + /// The agent's login (SUSER_SNAME()), for the GRANT text. + public string? AgentLogin { get; init; } + + /// Databases the detection could not look into, with why (Danish). + public IReadOnlyDictionary Problems { get; init; } = new Dictionary(); + + public AxDatabase? Find(string? database) => + database is null ? null : Databases.FirstOrDefault(d => string.Equals(d.Database, database, StringComparison.OrdinalIgnoreCase)); +} diff --git a/src/SQLInsight.Core/DynamicsAx/AxDetection.cs b/src/SQLInsight.Core/DynamicsAx/AxDetection.cs new file mode 100644 index 0000000..b04d263 --- /dev/null +++ b/src/SQLInsight.Core/DynamicsAx/AxDetection.cs @@ -0,0 +1,154 @@ +namespace SQLInsight.Core.DynamicsAx; + +/// One row of the detection query: a column of a known AX table. +public sealed record AxDetectionRow(string Schema, string Table, string Column, string Type, int Precision, int MaxLength, bool Readable, string? DatabaseUser = null); + +/// +/// The one place Dynamics AX is recognised: discovery (Servere og databaser), the AX collectors, the AX community +/// rules and the web UI all use it. Reads metadata only (sys.tables, sys.columns, HAS_PERMS_BY_NAME), never AX data. +/// +/// A database is AX when SQLDICTIONARY and DATAAREA are in the same schema. The version follows from the schema: +/// RECID int (precision 10) → Axapta 2.5/3.0; bigint without BATCHJOB → AX 4.0; BATCHJOB without NUMBERSEQUENCESCOPE → +/// AX 2009; NUMBERSEQUENCESCOPE without a PARTITION column → AX 2012 RTM/FP; PARTITION with the model store database +/// <db>_model on the instance → AX 2012 R2/R3; PARTITION without any model store → Dynamics 365 F&O (formodet). +/// Only the AX 3.0 rules are verified against a real customer schema; the rest follow Microsoft's documentation. +/// +/// +public static class AxDetection +{ + /// The AX tables SQLInsight knows: the markers and every table an AX check reads. + public static readonly IReadOnlyList KnownTables = + [ + "SQLDICTIONARY", "DATAAREA", "SYSCONFIG", + "BATCH", "BATCHJOB", "BATCHJOBHISTORY", "BATCHGROUP", + "NUMBERSEQUENCETABLE", "NUMBERSEQUENCESCOPE", "NUMBERSEQUENCEREFERENCE", + "SYSTEMSEQUENCES", "MODELELEMENT", + ]; + + private static string TableList => string.Join(", ", KnownTables.Select(t => "N'" + t + "'")); + + /// + /// Run inside the database (), SQL Server 2005 and later: every column of the known AX + /// tables, whether the agent may SELECT from the table, and the agent's database user. Names are upper-cased, so a + /// case-sensitive collation does not matter. + /// + public static string MetadataSql => $""" + SELECT s.name, UPPER(t.name), UPPER(c.name), TYPE_NAME(c.system_type_id), c.precision, c.max_length, + ISNULL(HAS_PERMS_BY_NAME(QUOTENAME(s.name) + N'.' + QUOTENAME(t.name), N'OBJECT', N'SELECT'), 0), + (SELECT TOP 1 p.name FROM sys.database_principals p WHERE p.sid = SUSER_SID()) + FROM sys.tables t + JOIN sys.schemas s ON s.schema_id = t.schema_id + JOIN sys.columns c ON c.object_id = t.object_id + WHERE UPPER(t.name) IN ({TableList}) + ORDER BY s.name, t.name, c.column_id + """; + + /// + /// SQL Server 2000 (legacy ODBC), with a three-part name so the session never changes database. No permission + /// information: the AX data checks need SQL Server 2005 or later. + /// + public static string MetadataSql2000(string database) => $""" + SELECT u.name, UPPER(o.name), UPPER(c.name), TYPE_NAME(c.xtype), c.xprec, c.length, 0, NULL + FROM {AxSql.Identifier(database)}.dbo.sysobjects o + JOIN {AxSql.Identifier(database)}.dbo.sysusers u ON u.uid = o.uid + JOIN {AxSql.Identifier(database)}.dbo.syscolumns c ON c.id = o.id + WHERE o.xtype = 'U' AND UPPER(o.name) IN ({TableList}) + ORDER BY u.name, o.name, c.colid + """; + + /// + /// Decides from a database's detection rows whether it is AX, and which version. + /// + /// Every database name on the instance, to find the AX 2012 R2/R3 model store (<db>_model). + public static AxDatabase? Recognise(string database, IReadOnlyList rows, IEnumerable databasesOnInstance) + { + ArgumentNullException.ThrowIfNull(database); + ArgumentNullException.ThrowIfNull(rows); + ArgumentNullException.ThrowIfNull(databasesOnInstance); + + var schemas = rows + .GroupBy(r => r.Schema, StringComparer.OrdinalIgnoreCase) + .Select(g => (Schema: g.Key, Tables: g.GroupBy(r => r.Table, StringComparer.OrdinalIgnoreCase) + .Select(t => new AxTable(t.Key.ToUpperInvariant(), t.Any(r => r.Readable), + t.Select(r => new AxColumn(r.Column.ToUpperInvariant(), r.Type, r.Precision, r.MaxLength)).ToList())) + .ToList())) + .Where(s => s.Tables.Any(t => t.Name == "SQLDICTIONARY") && s.Tables.Any(t => t.Name == "DATAAREA")) + .OrderByDescending(s => s.Tables.Count) + .ThenBy(s => string.Equals(s.Schema, "dbo", StringComparison.OrdinalIgnoreCase) ? 0 : 1) + .ToList(); + if (schemas.Count == 0) + { + return null; + } + + var (schema, tables) = schemas[0]; + var notes = new List(); + bool Has(string table) => tables.Any(t => t.Name == table); + var recId = tables.Single(t => t.Name == "SQLDICTIONARY").Column("RECID"); + var partition = tables.Any(t => t.Name != "SQLDICTIONARY" && t.HasColumn("PARTITION")); + var modelDatabase = databasesOnInstance.FirstOrDefault(d => string.Equals(d, database + "_model", StringComparison.OrdinalIgnoreCase)); + + AxVersion version; + if (recId is null) + { + version = AxVersion.Unknown; + notes.Add("SQLDICTIONARY har ingen RECID-kolonne, så versionen kan ikke bestemmes. AX-datatjek køres ikke."); + } + else if (recId.Precision <= 10) + { + version = AxVersion.Axapta30; + notes.Add($"RECID er 32-bit (int) i skemaet {schema}: Axapta 2.5 eller 3.0 (de to kan ikke skelnes ud fra skemaet)."); + } + else if (!Has("BATCHJOB")) + { + version = AxVersion.Ax40; + notes.Add("RECID er 64-bit, og BATCHJOB findes ikke: AX 4.0."); + } + else if (!Has("NUMBERSEQUENCESCOPE")) + { + version = AxVersion.Ax2009; + notes.Add("BATCHJOB findes, NUMBERSEQUENCESCOPE gør ikke: AX 2009."); + } + else if (!partition) + { + version = AxVersion.Ax2012; + notes.Add(Has("MODELELEMENT") + ? "NUMBERSEQUENCESCOPE uden PARTITION-kolonne, og modellageret ligger i selve databasen: AX 2012 RTM/FP." + : "NUMBERSEQUENCESCOPE uden PARTITION-kolonne: AX 2012 RTM/FP."); + } + else if (modelDatabase is not null || Has("MODELELEMENT")) + { + version = AxVersion.Ax2012R2; + notes.Add(modelDatabase is not null + ? $"PARTITION-kolonne og modeldatabasen {modelDatabase}: AX 2012 R2/R3." + : "PARTITION-kolonne og modellager i databasen: AX 2012 R2/R3."); + } + else + { + version = AxVersion.FinanceOperations; + notes.Add($"PARTITION-kolonne, men intet modellager ({database}_model findes ikke på instansen): formodet Dynamics 365 F&O on-premises. " + + "Ligger AX 2012 R2/R3-modeldatabasen på en anden instans, er det i stedet AX 2012 R2/R3."); + } + + if (!Has("SYSCONFIG")) + { + notes.Add("SYSCONFIG findes ikke (genkendt på SQLDICTIONARY og DATAAREA)."); + } + + if (!AxVersions.IsVerified(version) && version != AxVersion.Unknown) + { + notes.Add(AxVersions.VerificationText(version)); + } + + return new AxDatabase + { + Database = database, + Schema = schema, + Version = version, + Tables = tables, + ModelDatabase = modelDatabase, + DatabaseUser = rows.FirstOrDefault(r => r.DatabaseUser is not null)?.DatabaseUser, + Notes = notes, + }; + } +} diff --git a/src/SQLInsight.Core/DynamicsAx/AxReadAccess.cs b/src/SQLInsight.Core/DynamicsAx/AxReadAccess.cs new file mode 100644 index 0000000..449b688 --- /dev/null +++ b/src/SQLInsight.Core/DynamicsAx/AxReadAccess.cs @@ -0,0 +1,85 @@ +using System.Text; + +namespace SQLInsight.Core.DynamicsAx; + +/// +/// The optional read access to AX tables. The agent normally reads only metadata (grant-least-privilege.sql); the AX +/// data checks need SELECT on a few AX system tables (batch jobs, number sequences, RecId allocation, companies), and +/// the customer decides whether to give it, per database. No customer data tables, never more than SELECT. +/// db/permissions/grant-dynamics-ax-read.sql does the same for any database; makes the exact +/// statements for one database, with its schema (bmssa in AX 3.0) and the agent's login filled in. +/// +public static class AxReadAccess +{ + public const string ScriptPath = "db/permissions/grant-dynamics-ax-read.sql"; + + public const string RevokeScriptPath = "db/permissions/revoke-dynamics-ax-read.sql"; + + /// Used in the GRANT text when the agent's login is not known yet. + public const string LoginPlaceholder = @"DOMÆNE\svc-sqlinsight$"; + + /// The tables the read access covers for a version (those that exist in the database). + public static IReadOnlyList Tables(AxDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + return AxSchemaAdapter.For(database.Version) is { } adapter + ? adapter.ReadTables.Where(database.HasTable).ToList() + : []; + } + + /// The tables the agent may not read yet. + public static IReadOnlyList Missing(AxDatabase database) => Tables(database).Where(t => !database.CanRead(t)).ToList(); + + /// + /// The exact statements that give the agent SELECT on (all the version's tables when null) + /// in this database. Run by a sysadmin or db_owner; SQLInsight never runs it. + /// + public static string GrantSql(AxDatabase database, string? login, IEnumerable? tables = null) + { + ArgumentNullException.ThrowIfNull(database); + login = string.IsNullOrWhiteSpace(login) ? LoginPlaceholder : login.Trim(); + var user = database.DatabaseUser ?? login; + var sql = new StringBuilder() + .Append("USE ").Append(AxSql.Identifier(database.Database)).Append(";\r\n"); + if (database.DatabaseUser is null) + { + sql.Append("IF NOT EXISTS (SELECT 1 FROM sys.database_principals WHERE sid = SUSER_SID(").Append(AxSql.Literal(login)).Append("))\r\n") + .Append(" CREATE USER ").Append(AxSql.Identifier(login)).Append(" FOR LOGIN ").Append(AxSql.Identifier(login)).Append(";\r\n"); + } + + foreach (var table in (tables ?? Tables(database)).Distinct(StringComparer.OrdinalIgnoreCase)) + { + sql.Append("GRANT SELECT ON ").Append(database.Qualified(table)).Append(" TO ").Append(AxSql.Identifier(user)).Append(";\r\n"); + } + + return sql.ToString(); + } + + /// db/permissions/grant-dynamics-ax-read.sql as shipped. + public static string Template() + { + using var stream = typeof(AxReadAccess).Assembly.GetManifestResourceStream("SQLInsight.Core.grant-dynamics-ax-read.sql") + ?? throw new InvalidOperationException("grant-dynamics-ax-read.sql mangler i programmet."); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } + + /// The shipped script with the database and the agent's login filled in. + public static string Script(string database, string? login) + { + ArgumentNullException.ThrowIfNull(database); + return Template() + .Replace("SET @DatabaseName = N'AXDB';", "SET @DatabaseName = " + AxSql.Literal(database) + ";", StringComparison.Ordinal) + .Replace("SET @LoginName = N'" + LoginPlaceholder + "';", "SET @LoginName = " + AxSql.Literal(string.IsNullOrWhiteSpace(login) ? LoginPlaceholder : login.Trim()) + ";", + StringComparison.Ordinal); + } + + /// "Ikke målbar" for a check that needs , with the exact GRANT text. + public static string NotMeasurable(AxDatabase database, IReadOnlyList tables, string? login = null) + { + ArgumentNullException.ThrowIfNull(database); + ArgumentNullException.ThrowIfNull(tables); + return $"Ikke målbar: agenten må ikke læse {string.Join(", ", tables.Select(t => database.Schema + "." + t))} i {database.Database} " + + $"(valgfri AX-læseadgang). Kør som sysadmin eller db_owner: {GrantSql(database, login ?? database.AgentLogin, tables).Replace("\r\n", " ", StringComparison.Ordinal).Trim()}"; + } +} diff --git a/src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs b/src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs new file mode 100644 index 0000000..8930916 --- /dev/null +++ b/src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs @@ -0,0 +1,729 @@ +using System.Globalization; + +namespace SQLInsight.Core.DynamicsAx; + +/// +/// Everything SQLInsight knows about one Dynamics AX version's tables, in one place: which tables and columns the AX +/// checks read (and so which tables the optional read access must cover), the batch STATUS enum, and whether times are +/// local or UTC. Every query is built against the columns the database actually has (sys.columns, via +/// ), so a schema that differs gives "ikke målbar" with the exact column, never an error. +/// +/// Only is verified against a real customer schema (docs/dynamics-ax/schema-ax30.tsv). AX 4.0, +/// 2009, 2012 and F&O are built from Microsoft's documented schemas and NOT verified against customer data; their +/// documented tables and columns are in docs/dynamics-ax/schema-ax*.documented.tsv. When a customer supplies a real +/// schema, drop it in as docs/dynamics-ax/schema-ax2009.tsv or schema-ax2012.tsv: the adapter tests then check this +/// adapter against it instead (AxSchemaFile). +/// +/// +public abstract class AxSchemaAdapter +{ + /// How many batch jobs are read per collection (newest RECID first). + public const int MaxBatchJobs = 1000; + + /// How many history rows are read per collection. + public const int MaxHistoryRows = 1000; + + /// RecIds are handed out in blocks per AOS, so history rows may arrive slightly out of RECID order: re-read this far back. + public const long HistoryOverlap = 2000; + + /// How many number sequences are read (the most used first). + public const int MaxNumberSequences = 5000; + + /// How many RecId sequences are read (the highest first). + public const int MaxRecIdSequences = 500; + + /// DATEDIFF from the server's local time to UTC: turns AX 3.0/4.0's local times into UTC. + protected const string UtcOffset = "DATEDIFF(minute, GETDATE(), GETUTCDATE())"; + + public static readonly AxSchemaAdapter Ax30 = new Ax30Adapter(); + public static readonly AxSchemaAdapter Ax40 = new Ax40Adapter(); + public static readonly AxSchemaAdapter Ax2009 = new Ax2009Adapter(); + public static readonly AxSchemaAdapter Ax2012 = new Ax2012Adapter(); + public static readonly AxSchemaAdapter FinanceOperations = new FinanceOperationsAdapter(); + + public static IReadOnlyList All => [Ax30, Ax40, Ax2009, Ax2012, FinanceOperations]; + + /// The adapter for a version, or null when the version is unknown (then no AX data checks run). + public static AxSchemaAdapter? For(AxVersion version) => version switch + { + AxVersion.Axapta30 => Ax30, + AxVersion.Ax40 => Ax40, + AxVersion.Ax2009 => Ax2009, + AxVersion.Ax2012 or AxVersion.Ax2012R2 => Ax2012, + AxVersion.FinanceOperations => FinanceOperations, + _ => null, + }; + + /// The versions this adapter serves. + public abstract IReadOnlyList Versions { get; } + + /// The schema file name under docs/dynamics-ax (without .tsv / .documented.tsv): schema-ax30, schema-ax2009 … + public abstract string SchemaFileName { get; } + + /// The tables the batch checks read. + public abstract IReadOnlyList BatchTables { get; } + + /// The tables the number sequence checks read. + public abstract IReadOnlyList NumberSequenceTables { get; } + + /// The tables the RecId check reads (SQLDICTIONARY only for table names). + public static IReadOnlyList RecIdTables => ["SYSTEMSEQUENCES", "SQLDICTIONARY"]; + + /// Every table the optional AX read access covers for this version: batch, number sequences, RecId, companies. + public IReadOnlyList ReadTables => [.. BatchTables.Concat(NumberSequenceTables).Concat(RecIdTables).Append("DATAAREA").Distinct(StringComparer.OrdinalIgnoreCase)]; + + /// Columns each check cannot do without, per table (the adapter tests check them against the schema files). + public abstract IReadOnlyDictionary> RequiredColumns { get; } + + /// Batch times are a local date plus seconds since midnight (3.0/4.0), not UTC datetimes. + public abstract bool LocalBatchTimes { get; } + + /// The version's batch STATUS enum. + public abstract AxBatchStatus BatchStatus(int status); + + /// Whether the batch STATUS mapping is verified against customer data. + public virtual bool BatchStatusVerified => false; + + public abstract AxQuery BatchJobs(AxDatabase database); + + /// Completed runs from AX's own history after (newest first when null), or null: no history table in this version. + public virtual AxQuery? BatchHistory(AxDatabase database, long? afterRecId) => null; + + public abstract AxQuery NumberSequences(AxDatabase database); + + /// The companies: DATAAREA (ID, NAME, ISVIRTUAL). + public static AxQuery Companies(AxDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + if (Unreadable(database, ["DATAAREA"]) is { } reason) + { + return AxQuery.Unmeasurable(reason); + } + + var select = new AxSelect(database, "DATAAREA", "d") + .Add("ID", "nvarchar(10)", required: true) + .Add("NAME", "nvarchar(60)") + .Add("ISVIRTUAL"); + return select.MissingReason() is { } missing + ? AxQuery.Unmeasurable(missing) + : AxQuery.Ok($"SELECT TOP (500) {select.SelectList} FROM {database.Qualified("DATAAREA")} d ORDER BY d.{AxSql.Identifier("ID")}"); + } + + /// RecId allocation per table (per company in AX 3.0): SYSTEMSEQUENCES rows named SEQNO, the highest first. + public virtual AxQuery RecIds(AxDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + if (Unreadable(database, ["SYSTEMSEQUENCES"]) is { } reason) + { + return AxQuery.Unmeasurable(reason); + } + + var s = new AxSelect(database, "SYSTEMSEQUENCES", "s"); + var notes = new List(); + var name = database.CanRead("SQLDICTIONARY") && database.HasColumn("SQLDICTIONARY", "NAME") && database.HasColumn("SQLDICTIONARY", "FIELDID") + && database.HasColumn("SQLDICTIONARY", "TABLEID") + ? $"(SELECT TOP 1 d.{AxSql.Identifier("NAME")} FROM {database.Qualified("SQLDICTIONARY")} d WHERE d.{AxSql.Identifier("TABLEID")} = s.{AxSql.Identifier("TABID")} AND d.{AxSql.Identifier("FIELDID")} = 0)" + : "CAST(NULL AS nvarchar(40))"; + if (name.StartsWith("CAST", StringComparison.Ordinal)) + { + notes.Add("Tabelnavne vises ikke: agenten må ikke læse SQLDICTIONARY (eller den mangler NAME/FIELDID). Tabellerne vises med deres tabel-id."); + } + + s.Add("TABID", required: true) + .Add("DATAAREAID", "nvarchar(10)") + .AddExpression("CAST(" + s.Column("NEXTVAL", "bigint", required: true) + " AS bigint)") + .AddExpression("CAST(" + s.Column("MAXVAL", "bigint") + " AS bigint)") + .AddExpression(name); + var filter = database.HasColumn("SYSTEMSEQUENCES", "NAME") ? $" WHERE s.{AxSql.Identifier("NAME")} = N'SEQNO'" : string.Empty; + return s.MissingReason() is { } missing + ? AxQuery.Unmeasurable(missing) + : AxQuery.Ok($"SELECT TOP ({MaxRecIdSequences}) {s.SelectList} FROM {database.Qualified("SYSTEMSEQUENCES")} s{filter} ORDER BY s.{AxSql.Identifier("NEXTVAL")} DESC", [.. notes, .. s.Notes]); + } + + /// + /// "Ikke målbar" with the exact GRANT, when the agent may not read one of (or it is missing). + /// + protected static string? Unreadable(AxDatabase database, IReadOnlyList tables) + { + var missing = tables.Where(t => !database.HasTable(t)).ToList(); + if (missing.Count > 0) + { + return $"Ikke målbar: {database.Database} har ikke {(missing.Count == 1 ? "tabellen" : "tabellerne")} {string.Join(", ", missing.Select(t => database.Schema + "." + t))}, " + + $"som {AxVersions.Text(database.Version)} forventes at have."; + } + + var denied = tables.Where(t => !database.CanRead(t)).ToList(); + return denied.Count == 0 ? null : AxReadAccess.NotMeasurable(database, denied); + } + + // ---------------------------------------------------------------- interpretation (no SQL; unit tested) + + /// A stable key for a job, so its runs can be compared over time. + public virtual string JobKey(AxBatchJobRow row) + { + ArgumentNullException.ThrowIfNull(row); + return row.RecId.ToString(CultureInfo.InvariantCulture); + } + + /// What the UI calls a job: its caption, or class, company and group when AX stores no caption (3.0/4.0). + public virtual string Caption(AxBatchJobRow row) + { + ArgumentNullException.ThrowIfNull(row); + return string.IsNullOrWhiteSpace(row.Caption) ? $"Batchjob {row.RecId.ToString(CultureInfo.InvariantCulture)}" : row.Caption.Trim(); + } + + /// + /// A time as stored by AX in UTC: a local date plus seconds since midnight (3.0/4.0, using the server's offset when + /// read), or a UTC datetime. AX's empty date (1900-01-01) is null. + /// + public DateTime? ToUtc(DateTime? date, int? seconds, int offsetMinutes) + { + if (date is not { } d || d.Year <= 1900) + { + return null; + } + + if (!LocalBatchTimes) + { + return DateTime.SpecifyKind(d, DateTimeKind.Utc); + } + + var local = d.Date.AddSeconds(Math.Clamp(seconds ?? 0, 0, 86_399)); + return DateTime.SpecifyKind(local.AddMinutes(offsetMinutes), DateTimeKind.Utc); + } + + public AxBatchJob Interpret(AxBatchJobRow row, AxBatchHistory history, string database, DateTime? firstSeenExecuting) + { + ArgumentNullException.ThrowIfNull(row); + ArgumentNullException.ThrowIfNull(history); + var key = JobKey(row); + var runs = history.Runs(database, key); + var (median, p90) = AxBatchHistory.Statistics(runs); + var last = runs.Count > 0 ? runs[^1] : null; + return new AxBatchJob + { + Key = key, + RecId = row.RecId, + Caption = Caption(row), + Company = Trim(row.Company), + Group = Trim(row.Group), + StatusCode = row.Status, + Status = BatchStatus(row.Status), + PlannedStartUtc = ToUtc(row.PlannedDate, row.PlannedSeconds, row.ServerUtcOffsetMinutes), + StartedUtc = ToUtc(row.StartDate, row.StartSeconds, row.ServerUtcOffsetMinutes), + EndedUtc = ToUtc(row.EndDate, row.EndSeconds, row.ServerUtcOffsetMinutes), + FirstSeenExecutingUtc = firstSeenExecuting, + HistoryRuns = runs.Count, + MedianSeconds = median, + P90Seconds = p90, + LastRunSeconds = last?.Duration.TotalSeconds, + LastRunStatus = last?.Status, + LastRunEndedUtc = last?.EndedUtc, + }; + } + + /// + /// A completed run seen in the job table itself: a job that SQLInsight saw executing and now has an end time. The + /// start is the actual start when AX stores it (2009 and later), else when SQLInsight first saw it executing. + /// + public static AxBatchRun? ObservedRun(AxBatchJob job, DateTime? seenExecutingSince) + { + ArgumentNullException.ThrowIfNull(job); + if (job.Status is not (AxBatchStatus.Finished or AxBatchStatus.Error) || job.EndedUtc is not { } ended) + { + return null; + } + + var started = job.StartedUtc ?? seenExecutingSince; + if (started is not { } start || ended < start) + { + return null; + } + + return new AxBatchRun(job.Key, $"J{job.RecId.ToString(CultureInfo.InvariantCulture)}@{start:yyyyMMddHHmmss}", job.Caption, job.Company, start, ended, job.Status); + } + + /// A run from AX's batch history (2009 and later). + public AxBatchRun? HistoryRun(AxBatchHistoryRow row) + { + ArgumentNullException.ThrowIfNull(row); + var start = ToUtc(row.StartDate, null, row.ServerUtcOffsetMinutes); + var end = ToUtc(row.EndDate, null, row.ServerUtcOffsetMinutes); + if (start is not { } s || end is not { } e || e < s || row.JobRecId is not { } job) + { + return null; + } + + return new AxBatchRun(job.ToString(CultureInfo.InvariantCulture), "H" + row.RecId.ToString(CultureInfo.InvariantCulture), + Trim(row.Caption), Trim(row.Company), s, e, BatchStatus(row.Status)); + } + + /// Stable key of a number sequence for its sampled history. + public virtual string SequenceKey(AxNumberSequenceRow row) + { + ArgumentNullException.ThrowIfNull(row); + return (Trim(row.Company) ?? "*") + "/" + row.Code.Trim(); + } + + public AxNumberSequence Interpret(AxNumberSequenceRow row, IEnumerable samples, DateTime nowUtc, int forecastDays) + { + ArgumentNullException.ThrowIfNull(row); + var digits = row.Format?.Count(c => c == '#') is > 0 and var n ? n : (int?)null; + long? capacity = digits is { } d ? d >= 19 ? long.MaxValue : (long)Math.Pow(10, d) - 1 : null; + var max = capacity is { } c && c < row.Highest ? c : row.Highest; + var span = (double)max - row.Lowest + 1; + var used = span <= 0 ? 100 : Math.Clamp((row.Next - (double)row.Lowest) / span * 100, 0, 100); + var remaining = Math.Max(0, max - row.Next + 1); + var lastClean = row.LastCleanSeconds is null && row.LastCleanDate is { } utcClean && utcClean.Year > 1900 + ? DateTime.SpecifyKind(utcClean, DateTimeKind.Utc) + : ToLocalUtc(row.LastCleanDate, row.LastCleanSeconds, row.ServerUtcOffsetMinutes); + return new AxNumberSequence + { + Key = SequenceKey(row), + Code = row.Code.Trim(), + Company = Trim(row.Company), + Text = Trim(row.Text), + Lowest = row.Lowest, + Highest = row.Highest, + Next = row.Next, + Format = row.Format, + FormatDigits = digits, + FormatCapacity = capacity, + Max = max, + UsedPercent = used, + Remaining = remaining, + Blocked = row.Blocked, + Continuous = row.Continuous, + Cyclic = row.Cyclic, + Manual = row.Manual, + Referenced = row.Referenced, + LastCleanUtc = lastClean, + CleanIntervalHours = row.CleanIntervalHours, + Forecast = row.Manual || row.Cyclic ? null : AxForecast.From(samples, nowUtc, row.Next, max, forecastDays), + }; + } + + public static string RecIdKey(AxRecIdRow row) + { + ArgumentNullException.ThrowIfNull(row); + return row.TableId.ToString(CultureInfo.InvariantCulture) + "/" + (Trim(row.Company) ?? "*"); + } + + /// SYSTEMSEQUENCES.NEXTVAL (and RECID) is int: AX 3.0's 32-bit RecId. + public static AxRecIdSequence Interpret(AxRecIdRow row, bool nextValIsInt, IEnumerable samples, DateTime nowUtc, int forecastDays) + { + ArgumentNullException.ThrowIfNull(row); + var typeMax = nextValIsInt ? AxVersions.Int32RecIdMax : long.MaxValue; + var max = row.MaxVal is > 0 and var m && m < typeMax ? m : typeMax; + var used = Math.Clamp((double)row.NextVal / max * 100, 0, 100); + return new AxRecIdSequence + { + Key = RecIdKey(row), + TableId = row.TableId, + TableName = Trim(row.TableName), + Company = Trim(row.Company), + Next = row.NextVal, + Max = max, + Bits = nextValIsInt ? 32 : 64, + UsedPercent = used, + Remaining = Math.Max(0, max - row.NextVal), + Forecast = AxForecast.From(samples, nowUtc, row.NextVal, max, forecastDays), + }; + } + + private static DateTime? ToLocalUtc(DateTime? date, int? seconds, int offsetMinutes) => + date is { } d && d.Year > 1900 ? DateTime.SpecifyKind(d.Date.AddSeconds(Math.Clamp(seconds ?? 0, 0, 86_399)).AddMinutes(offsetMinutes), DateTimeKind.Utc) : null; + + protected static string? Trim(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + // ---------------------------------------------------------------- shared SQL builders + + /// + /// NUMBERSEQUENCETABLE with the company from DATAAREAID (3.0, 4.0, 2009) or NUMBERSEQUENCESCOPE (2012), and whether a + /// NUMBERSEQUENCEREFERENCE uses it. The most used sequences first, so TOP keeps those that matter. + /// + internal static AxQuery NumberSequenceQuery(AxDatabase database, string companyExpression, string? companyJoin, string? referencedExpression, IEnumerable extraNotes) + { + ArgumentNullException.ThrowIfNull(database); + if (Unreadable(database, ["NUMBERSEQUENCETABLE"]) is { } reason) + { + return AxQuery.Unmeasurable(reason); + } + + var n = new AxSelect(database, "NUMBERSEQUENCETABLE", "n"); + string Big(string column) => "CAST(" + n.Column(column, "bigint", required: true) + " AS bigint)"; + string Flag(string column, string lost) => "ISNULL(" + n.Column(column, whatIsLost: lost) + ", 0)"; + + // AX 2009 and later store the last cleanup as one UTC datetime; 3.0 as a local date plus seconds. + string cleanDate, cleanSeconds; + if (database.HasColumn("NUMBERSEQUENCETABLE", "LATESTCLEANDATETIME")) + { + cleanDate = n.Column("LATESTCLEANDATETIME", "datetime"); + cleanSeconds = "CAST(NULL AS int)"; + } + else + { + cleanDate = n.Column("LATESTCLEANDATE", "datetime", whatIsLost: "seneste oprydning af kontinuerlige nummerserier kendes ikke."); + cleanSeconds = n.Column("LATESTCLEANTIME"); + } + + n.AddExpression(Big("RECID")) + .AddExpression(n.Column("NUMBERSEQUENCE", "nvarchar(20)", required: true)) + .AddExpression(companyExpression) + .AddExpression(n.Column("TXT", "nvarchar(60)")) + .AddExpression(Big("LOWEST")) + .AddExpression(Big("HIGHEST")) + .AddExpression(Big("NEXTREC")) + .AddExpression(n.Column("FORMAT", "nvarchar(40)", whatIsLost: "formatets bredde kan ikke sammenlignes med HIGHEST.")) + .AddExpression(Flag("BLOCKED", "spærrede nummerserier kan ikke ses.")) + .AddExpression(Flag("CONTINUOUS", "kontinuerlige nummerserier kan ikke ses.")) + .AddExpression(Flag("CYCLIC", "cykliske nummerserier kan ikke ses (de regnes som almindelige).")) + .AddExpression(Flag("MANUAL", "manuelle nummerserier kan ikke ses (de regnes som almindelige).")) + .AddExpression(referencedExpression ?? "CAST(NULL AS int)") + .AddExpression(cleanDate) + .AddExpression(cleanSeconds) + .AddExpression("CAST(" + n.Column("CLEANINTERVAL", "float") + " AS float)") + .AddExpression(UtcOffset); + if (n.MissingReason() is { } missing) + { + return AxQuery.Unmeasurable(missing); + } + + var used = $"(CAST(n.{AxSql.Identifier("NEXTREC")} AS float) - n.{AxSql.Identifier("LOWEST")}) / NULLIF(CAST(n.{AxSql.Identifier("HIGHEST")} AS float) - n.{AxSql.Identifier("LOWEST")}, 0)"; + return AxQuery.Ok( + $"SELECT TOP ({MaxNumberSequences}) {n.SelectList} FROM {database.Qualified("NUMBERSEQUENCETABLE")} n{(companyJoin is null ? string.Empty : " " + companyJoin)} ORDER BY {used} DESC", + [.. n.Notes, .. extraNotes]); + } + + /// The old batch table BATCH of AX 3.0 and 4.0: a local start date plus seconds since midnight, no caption. + protected static AxQuery LegacyBatchJobs(AxDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + if (Unreadable(database, ["BATCH"]) is { } reason) + { + return AxQuery.Unmeasurable(reason); + } + + var b = new AxSelect(database, "BATCH", "b"); + b.AddExpression("CAST(" + b.Column("RECID", "bigint", required: true) + " AS bigint)") + .Add("CLASSNUM", whatIsLost: "jobbene kan ikke skelnes på klasse.") + .AddExpression("CAST(NULL AS nvarchar(60))") + .Add("COMPANY", "nvarchar(10)") + .Add("GROUPID", "nvarchar(20)") + .Add("STATUS", required: true) + .Add("STARTDATE", "datetime", required: true) + .Add("STARTTIME", required: true) + .AddExpression("CAST(NULL AS datetime)") + .AddExpression("CAST(NULL AS int)") + .Add("ENDDATE", "datetime", whatIsLost: "kørselstider kan ikke måles.") + .Add("ENDTIME") + .AddExpression(UtcOffset); + return b.MissingReason() is { } missing + ? AxQuery.Unmeasurable(missing) + : AxQuery.Ok($"SELECT TOP ({MaxBatchJobs}) {b.SelectList} FROM {database.Qualified("BATCH")} b ORDER BY b.{AxSql.Identifier("RECID")} DESC", b.Notes); + } + + /// BATCHJOB of AX 2009 and later: one row per job, planned start ORIGSTARTDATETIME, actual STARTDATETIME, all UTC. + protected static AxQuery JobBatchJobs(AxDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + if (Unreadable(database, ["BATCHJOB"]) is { } reason) + { + return AxQuery.Unmeasurable(reason); + } + + var j = new AxSelect(database, "BATCHJOB", "j"); + j.AddExpression("CAST(" + j.Column("RECID", "bigint", required: true) + " AS bigint)") + .AddExpression("CAST(NULL AS int)") + .Add("CAPTION", "nvarchar(100)", whatIsLost: "jobbene vises med RECID i stedet for navn.") + .Add("COMPANY", "nvarchar(10)") + .AddExpression("CAST(NULL AS nvarchar(20))") + .Add("STATUS", required: true) + .Add("ORIGSTARTDATETIME", "datetime", required: true) + .AddExpression("CAST(NULL AS int)") + .Add("STARTDATETIME", "datetime", whatIsLost: "den faktiske starttid kendes ikke; SQLInsight bruger, hvornår den selv så jobbet køre.") + .AddExpression("CAST(NULL AS int)") + .Add("ENDDATETIME", "datetime", whatIsLost: "sluttider kendes ikke.") + .AddExpression("CAST(NULL AS int)") + .AddExpression(UtcOffset); + return j.MissingReason() is { } missing + ? AxQuery.Unmeasurable(missing) + : AxQuery.Ok($"SELECT TOP ({MaxBatchJobs}) {j.SelectList} FROM {database.Qualified("BATCHJOB")} j ORDER BY j.{AxSql.Identifier("RECID")} DESC", j.Notes); + } + + /// + /// BATCHJOBHISTORY of AX 2009 and later, by RECID (always indexed in AX): the newest rows the first time, then the rows + /// after the previous read, re-reading RecIds back for blocks handed out by other AOS'es. + /// + protected static AxQuery? JobBatchHistory(AxDatabase database, long? afterRecId) + { + ArgumentNullException.ThrowIfNull(database); + if (!database.HasTable("BATCHJOBHISTORY")) + { + return null; + } + + if (Unreadable(database, ["BATCHJOBHISTORY"]) is { } reason) + { + return AxQuery.Unmeasurable(reason); + } + + var h = new AxSelect(database, "BATCHJOBHISTORY", "h"); + h.AddExpression("CAST(" + h.Column("RECID", "bigint", required: true) + " AS bigint)") + .AddExpression("CAST(" + h.Column("BATCHJOBID", "bigint", required: true) + " AS bigint)") + .Add("CAPTION", "nvarchar(100)") + .Add("COMPANY", "nvarchar(10)") + .Add("STATUS", required: true) + .Add("STARTDATETIME", "datetime", required: true) + .Add("ENDDATETIME", "datetime", required: true) + .AddExpression(UtcOffset); + if (h.MissingReason() is { } missing) + { + return AxQuery.Unmeasurable(missing); + } + + var recId = "h." + AxSql.Identifier("RECID"); + return AxQuery.Ok(afterRecId is { } after + ? $"SELECT TOP ({MaxHistoryRows}) {h.SelectList} FROM {database.Qualified("BATCHJOBHISTORY")} h WHERE {recId} > {AxSql.Number(Math.Max(0, after - HistoryOverlap))} ORDER BY {recId}" + : $"SELECT TOP ({MaxHistoryRows}) {h.SelectList} FROM {database.Qualified("BATCHJOBHISTORY")} h ORDER BY {recId} DESC", h.Notes); + } + + /// The 2009+ BatchStatus enum (Microsoft's documentation of BatchStatus). + protected static AxBatchStatus JobStatus(int status) => status switch + { + 0 => AxBatchStatus.Hold, + 1 => AxBatchStatus.Waiting, + 2 => AxBatchStatus.Executing, + 3 => AxBatchStatus.Error, + 4 => AxBatchStatus.Finished, + 5 => AxBatchStatus.Ready, + 6 => AxBatchStatus.NotRun, + 7 => AxBatchStatus.Cancelling, + 8 => AxBatchStatus.Canceled, + _ => AxBatchStatus.Unknown, + }; +} + +/// +/// Axapta 2.5/3.0. Verified against the real AX 3.0 schema from a customer (docs/dynamics-ax/schema-ax30.tsv): tables in +/// the bmssa schema, 32-bit RECID, BATCH with STARTDATE (datetime, local) + STARTTIME (int, seconds since midnight, +/// local), NUMBERSEQUENCETABLE with NEXTREC/HIGHEST/FORMAT/BLOCKED/INUSE per DATAAREAID, SYSTEMSEQUENCES with +/// NEXTVAL/MAXVAL per TABID and DATAAREAID. The batch STATUS values are NOT verified against customer data (the schema +/// file has no data): the UI shows the raw values found so the mapping can be checked. +/// +public sealed class Ax30Adapter : AxSchemaAdapter +{ + public override IReadOnlyList Versions => [AxVersion.Axapta30]; + public override string SchemaFileName => "schema-ax30"; + public override IReadOnlyList BatchTables => ["BATCH"]; + public override IReadOnlyList NumberSequenceTables => ["NUMBERSEQUENCETABLE", "NUMBERSEQUENCEREFERENCE"]; + public override bool LocalBatchTimes => true; + + public override IReadOnlyDictionary> RequiredColumns => new Dictionary> + { + ["BATCH"] = ["RECID", "CLASSNUM", "COMPANY", "GROUPID", "STATUS", "STARTDATE", "STARTTIME", "ENDDATE", "ENDTIME"], + ["NUMBERSEQUENCETABLE"] = ["RECID", "NUMBERSEQUENCE", "DATAAREAID", "TXT", "LOWEST", "HIGHEST", "NEXTREC", "FORMAT", "BLOCKED", "CONTINUOUS", "CYCLIC", "MANUAL", + "LATESTCLEANDATE", "LATESTCLEANTIME", "CLEANINTERVAL"], + ["NUMBERSEQUENCEREFERENCE"] = ["NUMBERSEQUENCE", "DATAAREAID"], + ["SYSTEMSEQUENCES"] = ["TABID", "DATAAREAID", "NEXTVAL", "MAXVAL", "NAME"], + ["SQLDICTIONARY"] = ["TABLEID", "FIELDID", "NAME", "RECID"], + ["DATAAREA"] = ["ID", "NAME", "ISVIRTUAL"], + }; + + /// + /// The Axapta 3.0 BatchStatus enum as SQLInsight assumes it: 0 Venter, 1 Udfører, 2 Fejl, 3 Afsluttet, 4 Tilbageholdt. + /// NOT verified against customer data; check with SELECT STATUS, COUNT(*) FROM bmssa.BATCH GROUP BY STATUS and the + /// batch list in AX. The web UI shows the values found. + /// + public override AxBatchStatus BatchStatus(int status) => status switch + { + 0 => AxBatchStatus.Waiting, + 1 => AxBatchStatus.Executing, + 2 => AxBatchStatus.Error, + 3 => AxBatchStatus.Finished, + 4 => AxBatchStatus.Hold, + _ => AxBatchStatus.Unknown, + }; + + public override string JobKey(AxBatchJobRow row) + { + ArgumentNullException.ThrowIfNull(row); + + // AX 3.0 makes a new BATCH row for each recurrence, so a job is its class, company and batch group. + return string.Join('|', row.ClassNumber?.ToString(CultureInfo.InvariantCulture) ?? "?", Trim(row.Company) ?? string.Empty, Trim(row.Group) ?? string.Empty); + } + + public override string Caption(AxBatchJobRow row) + { + ArgumentNullException.ThrowIfNull(row); + var text = row.ClassNumber is { } c ? $"Klasse {c.ToString(CultureInfo.InvariantCulture)}" : $"Batch {row.RecId.ToString(CultureInfo.InvariantCulture)}"; + return Trim(row.Group) is { } group ? $"{text} (gruppe {group})" : text; + } + + public override AxQuery BatchJobs(AxDatabase database) => LegacyBatchJobs(database); + + public override AxQuery NumberSequences(AxDatabase database) => LegacyNumberSequences(database); + + /// NUMBERSEQUENCETABLE per DATAAREAID; references by code and company (3.0, 4.0, 2009). + internal static AxQuery LegacyNumberSequences(AxDatabase database) + { + var notes = new List(); + string? referenced = null; + if (database.CanRead("NUMBERSEQUENCEREFERENCE") && database.HasColumn("NUMBERSEQUENCEREFERENCE", "NUMBERSEQUENCE") + && database.HasColumn("NUMBERSEQUENCEREFERENCE", "DATAAREAID") && database.HasColumn("NUMBERSEQUENCETABLE", "DATAAREAID")) + { + referenced = $"CASE WHEN EXISTS (SELECT 1 FROM {database.Qualified("NUMBERSEQUENCEREFERENCE")} r WHERE r.{AxSql.Identifier("NUMBERSEQUENCE")} = n.{AxSql.Identifier("NUMBERSEQUENCE")} " + + $"AND r.{AxSql.Identifier("DATAAREAID")} = n.{AxSql.Identifier("DATAAREAID")}) THEN 1 ELSE 0 END"; + } + else + { + notes.Add("Det kan ikke ses, om en spærret nummerserie stadig bruges: agenten må ikke læse NUMBERSEQUENCEREFERENCE (eller den mangler kolonner)."); + } + + var company = database.HasColumn("NUMBERSEQUENCETABLE", "DATAAREAID") ? "n." + AxSql.Identifier("DATAAREAID") : "CAST(NULL AS nvarchar(10))"; + return NumberSequenceQuery(database, company, null, referenced, notes); + } +} + +/// +/// AX 4.0: like 3.0 (the old BATCH table with local date + seconds, NUMBERSEQUENCETABLE per DATAAREAID) but dbo and +/// 64-bit RECID. Built from Microsoft's documentation and the 3.0 schema – NOT verified against customer data; the batch +/// STATUS values are assumed to be 3.0's. +/// +public sealed class Ax40Adapter : AxSchemaAdapter +{ + public override IReadOnlyList Versions => [AxVersion.Ax40]; + public override string SchemaFileName => "schema-ax40"; + public override IReadOnlyList BatchTables => ["BATCH"]; + public override IReadOnlyList NumberSequenceTables => ["NUMBERSEQUENCETABLE", "NUMBERSEQUENCEREFERENCE"]; + public override bool LocalBatchTimes => true; + + public override IReadOnlyDictionary> RequiredColumns => Ax30.RequiredColumns; + + public override AxBatchStatus BatchStatus(int status) => Ax30.BatchStatus(status); + + public override string JobKey(AxBatchJobRow row) => Ax30.JobKey(row); + + public override string Caption(AxBatchJobRow row) => Ax30.Caption(row); + + public override AxQuery BatchJobs(AxDatabase database) => LegacyBatchJobs(database); + + public override AxQuery NumberSequences(AxDatabase database) => Ax30Adapter.LegacyNumberSequences(database); +} + +/// +/// AX 2009: batch framework with BATCHJOB (one row per job: STATUS, ORIGSTARTDATETIME planned, STARTDATETIME and +/// ENDDATETIME actual, all UTC) and BATCHJOBHISTORY; NUMBERSEQUENCETABLE per DATAAREAID with NUMBERSEQUENCE, NEXTREC, +/// HIGHEST; SYSTEMSEQUENCES per table; 64-bit RECID. Baseret på Microsofts dokumenterede skema – ikke verificeret mod +/// kundedata. +/// +public sealed class Ax2009Adapter : AxSchemaAdapter +{ + public override IReadOnlyList Versions => [AxVersion.Ax2009]; + public override string SchemaFileName => "schema-ax2009"; + public override IReadOnlyList BatchTables => ["BATCHJOB", "BATCHJOBHISTORY"]; + public override IReadOnlyList NumberSequenceTables => ["NUMBERSEQUENCETABLE", "NUMBERSEQUENCEREFERENCE"]; + public override bool LocalBatchTimes => false; + + public override IReadOnlyDictionary> RequiredColumns => new Dictionary> + { + ["BATCHJOB"] = ["RECID", "CAPTION", "COMPANY", "STATUS", "ORIGSTARTDATETIME", "STARTDATETIME", "ENDDATETIME"], + ["BATCHJOBHISTORY"] = ["RECID", "BATCHJOBID", "CAPTION", "COMPANY", "STATUS", "STARTDATETIME", "ENDDATETIME"], + ["NUMBERSEQUENCETABLE"] = ["RECID", "NUMBERSEQUENCE", "DATAAREAID", "TXT", "LOWEST", "HIGHEST", "NEXTREC", "FORMAT", "BLOCKED", "CONTINUOUS", "CYCLIC", "MANUAL", + "LATESTCLEANDATETIME", "CLEANINTERVAL"], + ["NUMBERSEQUENCEREFERENCE"] = ["NUMBERSEQUENCE", "DATAAREAID"], + ["SYSTEMSEQUENCES"] = ["TABID", "DATAAREAID", "NEXTVAL", "MAXVAL", "NAME"], + ["SQLDICTIONARY"] = ["TABLEID", "FIELDID", "NAME", "RECID"], + ["DATAAREA"] = ["ID", "NAME", "ISVIRTUAL"], + }; + + public override AxBatchStatus BatchStatus(int status) => JobStatus(status); + + public override AxQuery BatchJobs(AxDatabase database) => JobBatchJobs(database); + + public override AxQuery? BatchHistory(AxDatabase database, long? afterRecId) => JobBatchHistory(database, afterRecId); + + public override AxQuery NumberSequences(AxDatabase database) => Ax30Adapter.LegacyNumberSequences(database); +} + +/// +/// AX 2012 (RTM/FP and R2/R3): BATCHJOB/BATCHJOBHISTORY as in 2009; NUMBERSEQUENCETABLE keyed by RECID with +/// NUMBERSEQUENCESCOPE (the company is NUMBERSEQUENCESCOPE.DATAAREA, empty for shared sequences) and +/// NUMBERSEQUENCEREFERENCE.NUMBERSEQUENCEID; a PARTITION column on every table from R2, whose model store is the +/// separate <db>_model database. Baseret på Microsofts dokumenterede skema – ikke verificeret mod kundedata. +/// +public class Ax2012Adapter : AxSchemaAdapter +{ + public override IReadOnlyList Versions => [AxVersion.Ax2012, AxVersion.Ax2012R2]; + public override string SchemaFileName => "schema-ax2012"; + public override IReadOnlyList BatchTables => ["BATCHJOB", "BATCHJOBHISTORY"]; + public override IReadOnlyList NumberSequenceTables => ["NUMBERSEQUENCETABLE", "NUMBERSEQUENCESCOPE", "NUMBERSEQUENCEREFERENCE"]; + public override bool LocalBatchTimes => false; + + public override IReadOnlyDictionary> RequiredColumns => new Dictionary> + { + ["BATCHJOB"] = ["RECID", "CAPTION", "COMPANY", "STATUS", "ORIGSTARTDATETIME", "STARTDATETIME", "ENDDATETIME"], + ["BATCHJOBHISTORY"] = ["RECID", "BATCHJOBID", "CAPTION", "COMPANY", "STATUS", "STARTDATETIME", "ENDDATETIME"], + ["NUMBERSEQUENCETABLE"] = ["RECID", "NUMBERSEQUENCE", "NUMBERSEQUENCESCOPE", "TXT", "LOWEST", "HIGHEST", "NEXTREC", "FORMAT", "BLOCKED", "CONTINUOUS", "CYCLIC", "MANUAL", + "LATESTCLEANDATETIME", "CLEANINTERVAL"], + ["NUMBERSEQUENCESCOPE"] = ["RECID", "DATAAREA"], + ["NUMBERSEQUENCEREFERENCE"] = ["NUMBERSEQUENCEID"], + ["SYSTEMSEQUENCES"] = ["TABID", "NEXTVAL", "MAXVAL", "NAME"], + ["SQLDICTIONARY"] = ["TABLEID", "FIELDID", "NAME", "RECID"], + ["DATAAREA"] = ["ID", "NAME", "ISVIRTUAL"], + }; + + public override AxBatchStatus BatchStatus(int status) => JobStatus(status); + + public override AxQuery BatchJobs(AxDatabase database) => JobBatchJobs(database); + + public override AxQuery? BatchHistory(AxDatabase database, long? afterRecId) => JobBatchHistory(database, afterRecId); + + /// A 2012 sequence is identified by its RECID: the same code can exist in several scopes. + public override string SequenceKey(AxNumberSequenceRow row) + { + ArgumentNullException.ThrowIfNull(row); + return row.RecId.ToString(CultureInfo.InvariantCulture); + } + + public override AxQuery NumberSequences(AxDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + var notes = new List(); + var company = "CAST(NULL AS nvarchar(10))"; + string? join = null; + if (database.CanRead("NUMBERSEQUENCESCOPE") && database.HasColumn("NUMBERSEQUENCESCOPE", "DATAAREA") && database.HasColumn("NUMBERSEQUENCESCOPE", "RECID") + && database.HasColumn("NUMBERSEQUENCETABLE", "NUMBERSEQUENCESCOPE")) + { + company = "NULLIF(sc." + AxSql.Identifier("DATAAREA") + ", N'')"; + join = $"LEFT JOIN {database.Qualified("NUMBERSEQUENCESCOPE")} sc ON sc.{AxSql.Identifier("RECID")} = n.{AxSql.Identifier("NUMBERSEQUENCESCOPE")}"; + } + else + { + notes.Add("Firma vises ikke: agenten må ikke læse NUMBERSEQUENCESCOPE (eller den mangler DATAAREA)."); + } + + string? referenced = null; + if (database.CanRead("NUMBERSEQUENCEREFERENCE") && database.HasColumn("NUMBERSEQUENCEREFERENCE", "NUMBERSEQUENCEID")) + { + referenced = $"CASE WHEN EXISTS (SELECT 1 FROM {database.Qualified("NUMBERSEQUENCEREFERENCE")} r WHERE r.{AxSql.Identifier("NUMBERSEQUENCEID")} = n.{AxSql.Identifier("RECID")}) THEN 1 ELSE 0 END"; + } + else + { + notes.Add("Det kan ikke ses, om en spærret nummerserie stadig bruges: agenten må ikke læse NUMBERSEQUENCEREFERENCE (eller den mangler NUMBERSEQUENCEID)."); + } + + return NumberSequenceQuery(database, company, join, referenced, notes); + } +} + +/// +/// Dynamics 365 Finance & Operations on-premises: recognised, and read with the AX 2012 layout (BATCHJOB, +/// NUMBERSEQUENCETABLE with NUMBERSEQUENCESCOPE), which F&O kept. Only the general and column-checked checks run; +/// NOT verified against customer data. +/// +public sealed class FinanceOperationsAdapter : Ax2012Adapter +{ + public override IReadOnlyList Versions => [AxVersion.FinanceOperations]; + public override string SchemaFileName => "schema-fo"; +} diff --git a/src/SQLInsight.Core/DynamicsAx/AxSchemaFile.cs b/src/SQLInsight.Core/DynamicsAx/AxSchemaFile.cs new file mode 100644 index 0000000..e54dc14 --- /dev/null +++ b/src/SQLInsight.Core/DynamicsAx/AxSchemaFile.cs @@ -0,0 +1,90 @@ +using System.Globalization; + +namespace SQLInsight.Core.DynamicsAx; + +/// +/// An AX schema as a tab-separated file (docs/dynamics-ax/schema-ax*.tsv: skema, tabel, kolonne, type, max_length, +/// column_id), as exported from a customer's database with sys.columns. Used to check the version adapters against real +/// schemas and to build realistic test databases. A real export (schema-ax2009.tsv) wins over the one SQLInsight wrote +/// from Microsoft's documentation (schema-ax2009.documented.tsv). +/// +public sealed record AxSchemaFile(string Path, bool Documented, IReadOnlyList Columns) +{ + public sealed record Column(string Schema, string Table, string Name, string Type, int MaxLength, int ColumnId) + { + /// sys.columns.precision for the types AX uses. + public int Precision => Type.ToLowerInvariant() switch + { + "int" => 10, + "bigint" => 19, + "smallint" => 5, + "tinyint" => 3, + "datetime" => 23, + "numeric" or "decimal" => 28, + "float" => 53, + _ => 0, + }; + + /// The column as T-SQL DDL, e.g. [NEXTREC] int NOT NULL. + public string Definition => "[" + Name + "] " + Type.ToLowerInvariant() switch + { + "varchar" or "nvarchar" or "char" or "nchar" or "varbinary" or "binary" => $"{Type}({(MaxLength < 0 ? "max" : MaxLength.ToString(CultureInfo.InvariantCulture))})", + "numeric" or "decimal" => $"{Type}(28, 12)", + _ => Type, + } + " NULL"; + } + + public IEnumerable Tables => Columns.Select(c => c.Table).Distinct(StringComparer.OrdinalIgnoreCase); + + public IEnumerable Of(string table) => Columns.Where(c => string.Equals(c.Table, table, StringComparison.OrdinalIgnoreCase)).OrderBy(c => c.ColumnId); + + public bool Has(string table, string column) => Of(table).Any(c => string.Equals(c.Name, column, StringComparison.OrdinalIgnoreCase)); + + /// The file for an adapter in : the real export when there is one, else the documented one, else null. + public static AxSchemaFile? Find(string directory, string fileName) + { + var real = System.IO.Path.Combine(directory, fileName + ".tsv"); + if (File.Exists(real)) + { + return Load(real, documented: false); + } + + var documented = System.IO.Path.Combine(directory, fileName + ".documented.tsv"); + return File.Exists(documented) ? Load(documented, documented: true) : null; + } + + public static AxSchemaFile Load(string path, bool documented) + { + var columns = new List(); + foreach (var line in File.ReadLines(path).Skip(1)) + { + var parts = line.TrimEnd('\r').Split('\t'); + if (parts.Length < 6 || parts[0].StartsWith('#')) + { + continue; + } + + columns.Add(new Column(parts[0].Trim(), parts[1].Trim().ToUpperInvariant(), parts[2].Trim().ToUpperInvariant(), parts[3].Trim(), + int.Parse(parts[4].Trim(), CultureInfo.InvariantCulture), int.Parse(parts[5].Trim(), CultureInfo.InvariantCulture))); + } + + return new AxSchemaFile(path, documented, columns); + } + + /// The detection rows the schema would give, every table readable or none. + public IReadOnlyList DetectionRows(bool readable = true, string? databaseUser = null) => + Columns.Where(c => AxDetection.KnownTables.Contains(c.Table, StringComparer.OrdinalIgnoreCase)) + .Select(c => new AxDetectionRow(c.Schema, c.Table, c.Name, c.Type, c.Precision, c.MaxLength, readable, databaseUser)) + .ToList(); + + /// CREATE TABLE statements for the known AX tables in the file (for test databases). + public IEnumerable CreateTables(IEnumerable? onlyTables = null) + { + var wanted = onlyTables?.ToHashSet(StringComparer.OrdinalIgnoreCase); + foreach (var table in Tables.Where(t => wanted?.Contains(t) ?? true)) + { + var columns = Of(table).ToList(); + yield return $"CREATE TABLE [{columns[0].Schema}].[{table}] ({string.Join(", ", columns.Select(c => c.Definition))});"; + } + } +} diff --git a/src/SQLInsight.Core/DynamicsAx/AxSequences.cs b/src/SQLInsight.Core/DynamicsAx/AxSequences.cs new file mode 100644 index 0000000..eda3695 --- /dev/null +++ b/src/SQLInsight.Core/DynamicsAx/AxSequences.cs @@ -0,0 +1,153 @@ +using System.Text.Json.Serialization; + +namespace SQLInsight.Core.DynamicsAx; + +/// A NUMBERSEQUENCETABLE row as read, before interpretation. +/// Used by at least one NUMBERSEQUENCEREFERENCE; null when the reference table cannot be read. +/// How often a continuous sequence's unused numbers are cleaned up (CLEANINTERVAL, hours). +public sealed record AxNumberSequenceRow( + long RecId, + string Code, + string? Company, + string? Text, + long Lowest, + long Highest, + long Next, + string? Format, + bool Blocked, + bool Continuous, + bool Cyclic, + bool Manual, + bool? Referenced, + DateTime? LastCleanDate, + int? LastCleanSeconds, + double? CleanIntervalHours, + int ServerUtcOffsetMinutes); + +/// A number sequence now, with its forecast from SQLInsight's own sampled history of NEXTREC. +public sealed record AxNumberSequence +{ + /// Company/code, or the RECID for a shared sequence (2012). + public required string Key { get; init; } + + public required string Code { get; init; } + public string? Company { get; init; } + public string? Text { get; init; } + public long Lowest { get; init; } + public long Highest { get; init; } + public long Next { get; init; } + public string? Format { get; init; } + + /// The number of # in the format: the digits the number gets. + public int? FormatDigits { get; init; } + + /// The largest number the format can show (10^digits − 1). + public long? FormatCapacity { get; init; } + + /// The last number the sequence can hand out: HIGHEST, or the format's capacity when that is lower. + public long Max { get; init; } + + public double UsedPercent { get; init; } + public long Remaining { get; init; } + public bool Blocked { get; init; } + public bool Continuous { get; init; } + public bool Cyclic { get; init; } + public bool Manual { get; init; } + public bool? Referenced { get; init; } + public DateTime? LastCleanUtc { get; init; } + public double? CleanIntervalHours { get; init; } + public AxForecast? Forecast { get; init; } + + /// HIGHEST is larger than the format can show: numbers above the format's capacity cannot be made. + [JsonIgnore] + public bool FormatTooShort => FormatCapacity is { } capacity && Highest > capacity; +} + +/// A SYSTEMSEQUENCES row (RecId allocation) as read. +public sealed record AxRecIdRow(int TableId, string? Company, long NextVal, long? MaxVal, string? TableName); + +/// A RecId sequence now: one per table (per company in AX 3.0), with its forecast. +public sealed record AxRecIdSequence +{ + public required string Key { get; init; } + public int TableId { get; init; } + public string? TableName { get; init; } + public string? Company { get; init; } + public long Next { get; init; } + + /// The last RecId that can be handed out: the column type's limit (int in AX 3.0), or MAXVAL when lower. + public long Max { get; init; } + + public int Bits { get; init; } + public double UsedPercent { get; init; } + public long Remaining { get; init; } + public AxForecast? Forecast { get; init; } +} + +/// The consumption rate from the oldest sample in the window to now, and when that runs out. +/// Numbers used per day. +/// Days until exhaustion at that rate; null when nothing is being used. +/// How many days of history the rate is based on. +public sealed record AxForecast(double PerDay, double? DaysLeft, double BasisDays) +{ + /// + /// The forecast from daily samples of the value, or null with less than a day of history. Uses the oldest sample + /// at most old (a recent rate) and at least one day old. + /// + public static AxForecast? From(IEnumerable samples, DateTime nowUtc, long current, long max, int windowDays = 30) + { + ArgumentNullException.ThrowIfNull(samples); + var basis = samples + .Where(s => s.SampledUtc <= nowUtc.AddDays(-1) && s.SampledUtc >= nowUtc.AddDays(-windowDays)) + .OrderBy(s => s.SampledUtc) + .FirstOrDefault(); + if (basis is null) + { + return null; + } + + var days = (nowUtc - basis.SampledUtc).TotalDays; + var perDay = (current - basis.Value) / days; + if (perDay <= 0) + { + return new AxForecast(0, null, days); + } + + return new AxForecast(perDay, Math.Max(0, (max - current) / perDay), days); + } +} + +/// One stored daily sample of a sequence's value (insight.AxSequenceSamples). +public sealed record AxSample(DateTime SampledUtc, long Value); + +/// Number sequences and RecIds of one AX database, or why they could not be read. +public sealed record AxSequenceDatabase +{ + public required string Database { get; init; } + public AxVersion Version { get; init; } + public string? NumberSequencesNotMeasurable { get; init; } + public IReadOnlyList NumberSequences { get; init; } = []; + public string? RecIdsNotMeasurable { get; init; } + public IReadOnlyList RecIds { get; init; } = []; + public IReadOnlyList Notes { get; init; } = []; +} + +/// Number sequences and RecIds of every monitored AX database on an instance (collector "dynamics-ax-sequences", hourly). +public sealed record AxSequenceSnapshot +{ + public const char NumberSequenceKind = 'N'; + public const char RecIdKind = 'R'; + + public required DateTimeOffset CapturedAt { get; init; } + public required AxThresholds Thresholds { get; init; } + public required IReadOnlyList Databases { get; init; } + + /// + /// Daily samples per database, kind (/) and key, for the + /// forecasts. Kept in memory (bounded to the forecast window) and rebuilt from insight.AxSequenceSamples after a restart. + /// + [JsonIgnore] + public IReadOnlyDictionary> History { get; init; } = new Dictionary>(); + + public static string HistoryKey(string database, char kind, string key) => database + "\u001f" + kind + "\u001f" + key; +} diff --git a/src/SQLInsight.Core/DynamicsAx/AxSessionMapper.cs b/src/SQLInsight.Core/DynamicsAx/AxSessionMapper.cs new file mode 100644 index 0000000..3b16ff5 --- /dev/null +++ b/src/SQLInsight.Core/DynamicsAx/AxSessionMapper.cs @@ -0,0 +1,98 @@ +using System.Globalization; +using System.Text; +using SQLInsight.Core.Data; + +namespace SQLInsight.Core.DynamicsAx; + +/// Who in AX is behind a SQL Server session, as far as SQL Server can tell. +/// The AX user id, when the AOS put it in CONTEXT_INFO. +/// The AX session id, when the AOS put it in CONTEXT_INFO. +/// "context_info", or "ikke tilgængelig". +/// How it was found, or why it cannot be (Danish). +/// CONTEXT_INFO as text, when it is printable. +public sealed record AxSessionIdentity(string? AxUser, int? AxSessionId, string Source, string Explanation, string? RawContext = null) +{ + public bool Known => AxUser is not null; +} + +/// +/// Maps SQL Server sessions to Dynamics AX users. All AX users reach SQL Server through the AOS's own login, so the +/// login says nothing about the AX user. AX 2009 and 2012 (and F&O) AOS can write the AX user id and AX session id to +/// CONTEXT_INFO on every connection when "connectioncontext" is turned on in the AOS configuration (registry); Microsoft +/// documents this for AX 2012 – the exact text layout is not verified against customer data, so the user is taken as the +/// first token with a letter and the session as the first number. AX 3.0 and 4.0 never set CONTEXT_INFO: the AX user +/// cannot be found from SQL Server, only the host (the AOS in a 3-tier installation, the user's own PC for a 2-tier +/// client) and the program. +/// +public static class AxSessionMapper +{ + public static AxSessionIdentity Map(SessionInfo session, AxVersion version) + { + ArgumentNullException.ThrowIfNull(session); + var text = Decode(session.ContextInfo); + if (version is AxVersion.Axapta30 or AxVersion.Ax40) + { + return new AxSessionIdentity(null, null, "ikke tilgængelig", + $"{AxVersions.Text(version)} sætter ikke CONTEXT_INFO, så AX-brugeren kan ikke ses fra SQL Server. " + + $"Vært: {session.HostName ?? "ukendt"} (AOS'en ved 3-lags-installation, brugerens egen pc ved 2-lags-klient). " + + $"Program: {session.ProgramName ?? "ukendt"}. Brugeren kan findes i AX under Online brugere ud fra tidspunktet.", text); + } + + if (text is not null) + { + var tokens = text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + var user = tokens.FirstOrDefault(t => t.Any(char.IsLetter)); + int? axSession = tokens.FirstOrDefault(t => t.All(char.IsAsciiDigit)) is { } digits + && int.TryParse(digits, NumberStyles.None, CultureInfo.InvariantCulture, out var id) ? id : null; + if (user is not null) + { + return new AxSessionIdentity(user, axSession, "context_info", + "Fra CONTEXT_INFO, som AOS'en sætter, når connectioncontext er slået til.", text); + } + } + + return new AxSessionIdentity(null, null, "ikke tilgængelig", + "AOS'en har ikke sat CONTEXT_INFO på sessionen. Slå connectioncontext til i AOS-konfigurationen (registreringsdatabasen på AOS-serveren, " + + "kræver genstart af AOS) for at se AX-brugeren bag SQL-sessioner.", text); + } + + /// CONTEXT_INFO hex as printable text (ASCII, or UTF-16 when every second byte is zero), or null. + public static string? Decode(string? hex) + { + if (string.IsNullOrWhiteSpace(hex)) + { + return null; + } + + byte[] bytes; + try + { + bytes = Convert.FromHexString(hex.StartsWith("0x", StringComparison.OrdinalIgnoreCase) ? hex[2..] : hex); + } + catch (FormatException) + { + return null; + } + + var length = bytes.Length; + while (length > 0 && bytes[length - 1] == 0) + { + length--; + } + + if (length == 0) + { + return null; + } + + var unicode = length >= 4 && Enumerable.Range(0, (length + 1) / 2).All(i => 2 * i + 1 >= length || bytes[2 * i + 1] == 0); + var text = unicode + ? Encoding.Unicode.GetString(bytes, 0, Math.Min(bytes.Length, length + (length % 2))) + : Encoding.Latin1.GetString(bytes, 0, length); + var printable = new string(text.Select(c => char.IsControl(c) ? ' ' : c).ToArray()).Trim(); + return printable.Length == 0 || printable.Count(c => char.IsAsciiLetterOrDigit(c) || c is ' ' or '\\' or '-' or '_' or '.' or '@' or 'æ' or 'ø' or 'å' or 'Æ' or 'Ø' or 'Å') + < printable.Length * 0.8 + ? null + : printable; + } +} diff --git a/src/SQLInsight.Core/DynamicsAx/AxSql.cs b/src/SQLInsight.Core/DynamicsAx/AxSql.cs new file mode 100644 index 0000000..0f8f384 --- /dev/null +++ b/src/SQLInsight.Core/DynamicsAx/AxSql.cs @@ -0,0 +1,136 @@ +using System.Globalization; +using System.Text; + +namespace SQLInsight.Core.DynamicsAx; + +/// +/// How SQLInsight reads AX tables: only SELECT, bounded with TOP, in the AX database's own context through +/// [db].sys.sp_executesql (the session never changes database), with a short lock timeout under read committed +/// so a read gives up instead of waiting behind AX, or snapshot isolation when the database allows it. Never NOLOCK. +/// +public static class AxSql +{ + /// A read of an AX table gives up after this long waiting for a lock (SET LOCK_TIMEOUT). + public const int LockTimeoutMilliseconds = 2000; + + /// SQL Server's error number for "Lock request time out period exceeded". + public const int LockTimeoutError = 1222; + + /// Command timeout for AX reads: short; they are small and indexed. + public const int CommandTimeoutSeconds = 15; + + public static string Identifier(string name) + { + ArgumentNullException.ThrowIfNull(name); + return "[" + name.Replace("]", "]]", StringComparison.Ordinal) + "]"; + } + + public static string Literal(string value) + { + ArgumentNullException.ThrowIfNull(value); + return "N'" + value.Replace("'", "''", StringComparison.Ordinal) + "'"; + } + + public static string Number(long value) => value.ToString(CultureInfo.InvariantCulture); + + /// + /// Runs in without changing the session's database, with + /// the isolation and lock timeout for AX reads. The SET options only apply inside sp_executesql; they are reset + /// afterwards anyway. + /// + public static string InDatabase(string database, string innerSql, AxReadIsolation isolation = AxReadIsolation.ReadCommitted) + { + ArgumentNullException.ThrowIfNull(innerSql); + var inner = new StringBuilder() + .Append("SET NOCOUNT ON; SET LOCK_TIMEOUT ").Append(LockTimeoutMilliseconds.ToString(CultureInfo.InvariantCulture)).Append("; ") + .Append(isolation == AxReadIsolation.Snapshot ? "SET TRANSACTION ISOLATION LEVEL SNAPSHOT; " : "SET TRANSACTION ISOLATION LEVEL READ COMMITTED; ") + .Append(innerSql) + .ToString(); + return "EXEC " + Identifier(database) + ".sys.sp_executesql " + Literal(inner) + ";\nSET LOCK_TIMEOUT -1; SET TRANSACTION ISOLATION LEVEL READ COMMITTED;"; + } +} + +/// +/// A query an AX check would run, or why it cannot (Danish, "Ikke målbar: …"). Built by the version adapters, which +/// check every table and column in sys.columns first, so a query never fails on a schema it does not fit. +/// +public sealed record AxQuery +{ + private AxQuery(string? sql, string? notMeasurable, IReadOnlyList notes) + { + Sql = sql; + NotMeasurable = notMeasurable; + Notes = notes; + } + + /// The inner SQL (run with ), when measurable. + public string? Sql { get; } + + public string? NotMeasurable { get; } + + /// Optional columns or tables that were missing or unreadable, and what that leaves out. + public IReadOnlyList Notes { get; } + + public bool IsMeasurable => Sql is not null; + + public static AxQuery Ok(string sql, IReadOnlyList? notes = null) => new(sql, null, notes ?? []); + + public static AxQuery Unmeasurable(string reason) => new(null, reason, []); +} + +/// +/// Builds a SELECT list where every column is checked against the database's actual columns: a missing required column +/// makes the query not measurable with the exact name, a missing optional column becomes NULL with a note. +/// +public sealed class AxSelect(AxDatabase database, string table, string alias) +{ + private readonly List _missingRequired = []; + private readonly List _notes = []; + private readonly List _columns = []; + + public IReadOnlyList MissingRequired => _missingRequired; + public IReadOnlyList Notes => _notes; + public string Table => table; + + /// The expression for a column, or NULL of when optional and missing. + public string Column(string column, string nullType = "int", bool required = false, string? whatIsLost = null) + { + if (database.HasColumn(table, column)) + { + return alias + "." + AxSql.Identifier(column); + } + + if (required) + { + _missingRequired.Add($"{database.Schema}.{table}.{column}"); + } + else if (whatIsLost is not null) + { + _notes.Add($"Kolonnen {database.Schema}.{table}.{column} findes ikke: {whatIsLost}"); + } + + return "CAST(NULL AS " + nullType + ")"; + } + + /// Adds a column to the SELECT list (in order). + public AxSelect Add(string column, string nullType = "int", bool required = false, string? whatIsLost = null) + { + _columns.Add(Column(column, nullType, required, whatIsLost)); + return this; + } + + /// Adds an expression to the SELECT list (in order). + public AxSelect AddExpression(string expression) + { + _columns.Add(expression); + return this; + } + + public string SelectList => string.Join(", ", _columns); + + /// "Ikke målbar" with the exact missing columns, or null when all required columns exist. + public string? MissingReason() => _missingRequired.Count == 0 + ? null + : $"Ikke målbar: {database.Database} mangler {(_missingRequired.Count == 1 ? "kolonnen" : "kolonnerne")} {string.Join(", ", _missingRequired)}, " + + $"som {AxVersions.Text(database.Version)}-skemaet forventes at have. Skemaet afviger fra det kendte, så tjekket springes over."; +} diff --git a/src/SQLInsight.Core/DynamicsAx/AxThresholds.cs b/src/SQLInsight.Core/DynamicsAx/AxThresholds.cs new file mode 100644 index 0000000..279662f --- /dev/null +++ b/src/SQLInsight.Core/DynamicsAx/AxThresholds.cs @@ -0,0 +1,57 @@ +namespace SQLInsight.Core.DynamicsAx; + +/// +/// When the AX checks raise a finding (SQLInsight:DynamicsAx in the agent's configuration). Carried in the snapshots, +/// so the rules and the web UI use the same values as the collection. +/// +public sealed record AxThresholds +{ + /// A waiting batch job whose planned start is more than this many minutes ago is late (SQLI-AX-003). + public int BatchLateMinutes { get; init; } = 30; + + /// Late by more than this many hours: it has in practice never started (High). + public int BatchNeverStartedHours { get; init; } = 24; + + /// A batch job executing longer than this many hours is stuck (SQLI-AX-004). + public int BatchStuckHours { get; init; } = 6; + + /// Running longer than this many times its own 90th percentile is unusually long (SQLI-AX-002). + public double BatchLongRunningFactor { get; init; } = 2.0; + + /// …but never for runs shorter than this many minutes. + public int BatchLongRunningMinimumMinutes { get; init; } = 10; + + /// Earlier runs needed before a job's own history is used. + public int BatchHistoryMinimumRuns { get; init; } = 5; + + /// A run that failed within this many hours is reported (SQLI-AX-001). + public int BatchFailedLookbackHours { get; init; } = 24; + + /// A number sequence more used than this (percent) is a finding (SQLI-AX-005); is High. + public double NumberSequenceWarnPercent { get; init; } = 80; + + public double NumberSequenceCriticalPercent { get; init; } = 95; + + /// Forecast to run out within this many days: a finding; within : High. + public int ForecastWarnDays { get; init; } = 90; + + public int ForecastCriticalDays { get; init; } = 30; + + /// AX 3.0 (32-bit RecId): a finding from this share of the int range, earlier than for 64-bit. + public double RecId32WarnPercent { get; init; } = 50; + + /// AX 3.0: Critical from this share. + public double RecId32CriticalPercent { get; init; } = 75; + + /// AX 3.0: forecast to run out within this many days is a finding (Critical within half). + public int RecId32ForecastDays { get; init; } = 730; + + /// AX 4.0 and later (64-bit RecId, or a lowered MAXVAL): a finding from this share. + public double RecId64WarnPercent { get; init; } = 80; + + /// An AX session blocking others for at least this many seconds is reported with its AX user (SQLI-AX-010). + public int BlockingSeconds { get; init; } = 30; + + /// How many days back the forecasts look for their oldest sample. + public int ForecastBasisDays { get; init; } = 30; +} diff --git a/src/SQLInsight.Core/DynamicsAx/AxVersion.cs b/src/SQLInsight.Core/DynamicsAx/AxVersion.cs new file mode 100644 index 0000000..ea48503 --- /dev/null +++ b/src/SQLInsight.Core/DynamicsAx/AxVersion.cs @@ -0,0 +1,90 @@ +namespace SQLInsight.Core.DynamicsAx; + +/// +/// The Dynamics AX generation of a database, as far as its schema tells (). Stored as a number +/// in snapshots: only add at the end. +/// +public enum AxVersion +{ + Unknown = 0, + + /// Axapta 2.5 or 3.0: 32-bit RECID, tables usually owned by bmssa. The two cannot be told apart from the schema. + Axapta30 = 1, + + /// AX 4.0: 64-bit RECID, the old batch table (BATCH with STARTDATE/STARTTIME), no BATCHJOB. + Ax40 = 2, + + /// AX 2009: BATCHJOB/BATCHJOBHISTORY with UTC times, no NUMBERSEQUENCESCOPE. + Ax2009 = 3, + + /// AX 2012 RTM/FP: NUMBERSEQUENCESCOPE, model store inside the business database, no PARTITION column. + Ax2012 = 4, + + /// AX 2012 R2/R3: PARTITION column, model store in the separate <database>_model database. + Ax2012R2 = 5, + + /// + /// Dynamics 365 Finance & Operations on-premises (formodet): PARTITION column but no model store anywhere on the + /// instance. Only recognised, not verified. + /// + FinanceOperations = 6, +} + +public static class AxVersions +{ + /// The largest RECID AX 3.0 can hand out: RECID is int (32-bit). + public const long Int32RecIdMax = int.MaxValue; + + /// Short text, as stored in insight.ConfiguredDatabases.DynamicsAxVersion (varchar(16)). + public static string ShortText(AxVersion version) => version switch + { + AxVersion.Axapta30 => "3.0", + AxVersion.Ax40 => "4.0", + AxVersion.Ax2009 => "2009", + AxVersion.Ax2012 => "2012", + AxVersion.Ax2012R2 => "2012 R2/R3", + AxVersion.FinanceOperations => "F&O", + _ => "ukendt", + }; + + /// The name shown in the web UI. + public static string Text(AxVersion version) => version switch + { + AxVersion.Axapta30 => "Axapta 3.0 (eller 2.5)", + AxVersion.Ax40 => "Dynamics AX 4.0", + AxVersion.Ax2009 => "Dynamics AX 2009", + AxVersion.Ax2012 => "Dynamics AX 2012 (RTM/FP)", + AxVersion.Ax2012R2 => "Dynamics AX 2012 R2/R3", + AxVersion.FinanceOperations => "Dynamics 365 Finance & Operations (on-premises, formodet)", + _ => "Ukendt AX-version", + }; + + /// + /// Whether SQLInsight's knowledge of the version's tables is checked against a real customer schema + /// (docs/dynamics-ax/schema-ax30.tsv). Everything else is built from Microsoft's documentation. + /// + public static bool IsVerified(AxVersion version) => version == AxVersion.Axapta30; + + /// What the UI and the docs say about how sure SQLInsight is of the version's schema. + public static string VerificationText(AxVersion version) => IsVerified(version) + ? "Verificeret mod en rigtig AX 3.0-database (kundeskema, docs/dynamics-ax/schema-ax30.tsv)." + : "Baseret på Microsofts dokumenterede skema – ikke verificeret mod kundedata."; + + /// RECID is 32-bit (int) in Axapta 2.5/3.0 and 64-bit (bigint) from AX 4.0. + public static bool Has32BitRecId(AxVersion version) => version == AxVersion.Axapta30; + + /// Batch times: local date + seconds since midnight (3.0/4.0) or UTC datetime (2009 and later). + public static bool HasUtcBatchTimes(AxVersion version) => version is AxVersion.Ax2009 or AxVersion.Ax2012 or AxVersion.Ax2012R2 or AxVersion.FinanceOperations; + + /// Parses back; also the 1.3 values "3.0" and "4.0+". + public static AxVersion Parse(string? text) => text switch + { + "3.0" => AxVersion.Axapta30, + "4.0" => AxVersion.Ax40, + "2009" => AxVersion.Ax2009, + "2012" => AxVersion.Ax2012, + "2012 R2/R3" => AxVersion.Ax2012R2, + "F&O" => AxVersion.FinanceOperations, + _ => AxVersion.Unknown, + }; +} diff --git a/src/SQLInsight.Core/SQLInsight.Core.csproj b/src/SQLInsight.Core/SQLInsight.Core.csproj index d26d127..6133836 100644 --- a/src/SQLInsight.Core/SQLInsight.Core.csproj +++ b/src/SQLInsight.Core/SQLInsight.Core.csproj @@ -5,5 +5,7 @@ + + diff --git a/tests/SQLInsight.Core.Tests/DynamicsAxTests.cs b/tests/SQLInsight.Core.Tests/DynamicsAxTests.cs new file mode 100644 index 0000000..f09e44d --- /dev/null +++ b/tests/SQLInsight.Core.Tests/DynamicsAxTests.cs @@ -0,0 +1,373 @@ +using System.Text.RegularExpressions; +using SQLInsight.Core.Configuration; +using SQLInsight.Core.Data; +using SQLInsight.Core.DynamicsAx; + +namespace SQLInsight.Core.Tests; + +/// Builds AX databases as the detector would see them, from the schema files in docs/dynamics-ax. +internal static class AxFixtures +{ + public static string Directory => Path.Combine(AppContext.BaseDirectory, "DynamicsAx"); + + public static AxSchemaFile Schema(string fileName) => AxSchemaFile.Find(Directory, fileName) + ?? throw new InvalidOperationException($"{fileName}.tsv mangler i docs/dynamics-ax."); + + /// The real AX 3.0 schema from a customer (bmssa, 32-bit RECID). + public static AxSchemaFile Ax30 => Schema("schema-ax30"); + + public static AxDatabase Recognise(AxSchemaFile schema, string name = "AXDB", bool readable = true, string? user = null, IEnumerable? databases = null, + Func? keep = null) => + AxDetection.Recognise(name, schema.DetectionRows(readable, user).Where(keep ?? (_ => true)).ToList(), databases ?? [name]) + ?? throw new InvalidOperationException("Ikke genkendt som AX."); +} + +public class AxDetectionTests +{ + [Fact] + public void The_real_ax30_schema_is_axapta_30_in_bmssa_and_verified() + { + var ax = AxFixtures.Recognise(AxFixtures.Ax30); + + Assert.Equal((AxVersion.Axapta30, "bmssa", true), (ax.Version, ax.Schema, ax.Verified)); + Assert.Equal("int", ax.Table("BATCH")!.Column("STARTTIME")!.Type); + Assert.Equal(10, ax.Table("SQLDICTIONARY")!.Column("RECID")!.Precision); + Assert.False(AxFixtures.Ax30.Documented); + } + + [Fact] + public void Ax2009_and_2012_are_told_apart_by_batchjob_scope_partition_and_the_model_database() + { + var ax2009 = AxFixtures.Schema("schema-ax2009"); + var ax2012 = AxFixtures.Schema("schema-ax2012"); + + Assert.Equal(AxVersion.Ax2009, AxFixtures.Recognise(ax2009).Version); + Assert.Equal(AxVersion.Ax2012R2, AxFixtures.Recognise(ax2012, "AX2012", databases: ["AX2012", "AX2012_model"]).Version); + Assert.Equal("AX2012_model", AxFixtures.Recognise(ax2012, "AX2012", databases: ["AX2012", "ax2012_MODEL"]).ModelDatabase, StringComparer.OrdinalIgnoreCase); + Assert.Equal(AxVersion.FinanceOperations, AxFixtures.Recognise(ax2012, "AxDB").Version); + Assert.Equal(AxVersion.Ax2012, AxFixtures.Recognise(ax2012, keep: r => r.Column != "PARTITION").Version); + Assert.Equal(AxVersion.Ax40, AxFixtures.Recognise(ax2009, keep: r => r.Table is not ("BATCHJOB" or "BATCHJOBHISTORY")).Version); + Assert.All(new[] { AxVersion.Ax40, AxVersion.Ax2009, AxVersion.Ax2012, AxVersion.Ax2012R2, AxVersion.FinanceOperations }, v => Assert.False(AxVersions.IsVerified(v))); + Assert.Contains(AxFixtures.Recognise(ax2009).Notes, n => n.Contains("ikke verificeret mod kundedata", StringComparison.Ordinal)); + } + + [Fact] + public void A_database_without_sqldictionary_and_dataarea_in_one_schema_is_not_ax() + { + var rows = AxFixtures.Ax30.DetectionRows().Where(r => r.Table != "DATAAREA").ToList(); + + Assert.Null(AxDetection.Recognise("NotAx", rows, ["NotAx"])); + Assert.Null(AxDetection.Recognise("Empty", [], ["Empty"])); + Assert.Equal(AxVersion.Unknown, AxFixtures.Recognise(AxFixtures.Ax30, keep: r => !(r.Table == "SQLDICTIONARY" && r.Column == "RECID")).Version); + } + + [Theory] + [InlineData("3.0", AxVersion.Axapta30)] + [InlineData("2012 R2/R3", AxVersion.Ax2012R2)] + [InlineData("F&O", AxVersion.FinanceOperations)] + [InlineData("4.0+", AxVersion.Unknown)] + public void Short_version_texts_round_trip(string text, AxVersion version) + { + Assert.Equal(version, AxVersions.Parse(text)); + if (version != AxVersion.Unknown) + { + Assert.Equal(text, AxVersions.ShortText(version)); + Assert.True(text.Length <= 16); // insight.ConfiguredDatabases.DynamicsAxVersion varchar(16) + } + } +} + +public class AxSchemaAdapterTests +{ + public static TheoryData Adapters() => [.. AxSchemaAdapter.All.Select(a => a.GetType().Name)]; + + private static AxSchemaAdapter Adapter(string name) => AxSchemaAdapter.All.Single(a => a.GetType().Name == name); + + [Theory] + [MemberData(nameof(Adapters))] + public void Every_column_an_adapter_needs_is_in_its_schema_file(string name) + { + var adapter = Adapter(name); + var schema = AxSchemaFile.Find(AxFixtures.Directory, adapter.SchemaFileName); + if (schema is null) + { + // AX 4.0 and F&O: no schema file yet (neither from a customer nor written from the documentation). + Assert.Contains(adapter.Versions, v => v is AxVersion.Ax40 or AxVersion.FinanceOperations); + return; + } + + Assert.Equal(adapter is not Ax30Adapter, schema.Documented); + foreach (var (table, columns) in adapter.RequiredColumns) + { + Assert.All(columns, c => Assert.True(schema.Has(table, c), $"{schema.Path}: {table}.{c} mangler")); + } + + Assert.All(adapter.ReadTables, t => Assert.Contains(t, schema.Tables, StringComparer.OrdinalIgnoreCase)); + } + + [Fact] + public void Ax30_queries_use_bmssa_top_and_no_nolock() + { + var ax = AxFixtures.Recognise(AxFixtures.Ax30); + var adapter = AxSchemaAdapter.For(ax.Version)!; + + var queries = new[] { adapter.BatchJobs(ax), adapter.NumberSequences(ax), adapter.RecIds(ax), AxSchemaAdapter.Companies(ax) }; + + Assert.All(queries, q => Assert.True(q.IsMeasurable, q.NotMeasurable)); + Assert.All(queries, q => Assert.Contains("TOP (", q.Sql!, StringComparison.Ordinal)); + Assert.All(queries, q => Assert.DoesNotContain("NOLOCK", q.Sql!, StringComparison.OrdinalIgnoreCase)); + Assert.Contains("[bmssa].[BATCH]", queries[0].Sql!, StringComparison.Ordinal); + Assert.Contains("[bmssa].[NUMBERSEQUENCEREFERENCE]", queries[1].Sql!, StringComparison.Ordinal); + Assert.Contains("N'SEQNO'", queries[2].Sql!, StringComparison.Ordinal); + Assert.Null(adapter.BatchHistory(ax, null)); + Assert.Empty(queries[0].Notes); + } + + [Fact] + public void Ax2012_reads_the_company_from_the_scope_and_history_by_recid() + { + var ax = AxFixtures.Recognise(AxFixtures.Schema("schema-ax2012"), "AX2012", databases: ["AX2012", "AX2012_model"]); + var adapter = AxSchemaAdapter.For(ax.Version)!; + + var sequences = adapter.NumberSequences(ax); + var first = adapter.BatchHistory(ax, null)!; + var next = adapter.BatchHistory(ax, 10_000)!; + + Assert.Contains("LEFT JOIN [dbo].[NUMBERSEQUENCESCOPE]", sequences.Sql!, StringComparison.Ordinal); + Assert.Contains("[NUMBERSEQUENCEID]", sequences.Sql!, StringComparison.Ordinal); + Assert.Contains("ORDER BY h.[RECID] DESC", first.Sql!, StringComparison.Ordinal); + Assert.Contains($"h.[RECID] > {10_000 - AxSchemaAdapter.HistoryOverlap}", next.Sql!, StringComparison.Ordinal); + Assert.Contains("[ORIGSTARTDATETIME]", adapter.BatchJobs(ax).Sql!, StringComparison.Ordinal); + } + + [Fact] + public void Without_read_access_a_check_is_not_measurable_with_the_exact_grant() + { + var ax = AxFixtures.Recognise(AxFixtures.Ax30, "AX30", readable: false) with { AgentLogin = @"KUNDE\svc-sqlinsight$" }; + + var batch = AxSchemaAdapter.Ax30.BatchJobs(ax); + + Assert.False(batch.IsMeasurable); + Assert.StartsWith("Ikke målbar: agenten må ikke læse bmssa.BATCH i AX30", batch.NotMeasurable, StringComparison.Ordinal); + Assert.Contains(@"CREATE USER [KUNDE\svc-sqlinsight$] FOR LOGIN [KUNDE\svc-sqlinsight$];", batch.NotMeasurable, StringComparison.Ordinal); + Assert.Contains(@"GRANT SELECT ON [bmssa].[BATCH] TO [KUNDE\svc-sqlinsight$];", batch.NotMeasurable, StringComparison.Ordinal); + } + + [Fact] + public void A_missing_required_column_names_it_and_a_missing_optional_one_becomes_null_with_a_note() + { + var withoutStartTime = AxFixtures.Recognise(AxFixtures.Ax30, keep: r => !(r.Table == "BATCH" && r.Column == "STARTTIME")); + var withoutFormat = AxFixtures.Recognise(AxFixtures.Ax30, keep: r => !(r.Table == "NUMBERSEQUENCETABLE" && r.Column == "FORMAT")); + + var batch = AxSchemaAdapter.Ax30.BatchJobs(withoutStartTime); + var sequences = AxSchemaAdapter.Ax30.NumberSequences(withoutFormat); + + Assert.Contains("bmssa.BATCH.STARTTIME", batch.NotMeasurable, StringComparison.Ordinal); + Assert.True(sequences.IsMeasurable); + Assert.Contains("CAST(NULL AS nvarchar(40))", sequences.Sql!, StringComparison.Ordinal); + Assert.Contains(sequences.Notes, n => n.Contains("FORMAT", StringComparison.Ordinal)); + } + + [Fact] + public void Grant_script_covers_exactly_the_adapters_tables_per_version() + { + var lists = Regex.Matches(AxReadAccess.Template(), @"N''\|([A-Z|]+)\|''").Select(m => m.Groups[1].Value.Split('|')).ToList(); + + Assert.Equal(3, lists.Count); + Assert.Equal(AxSchemaAdapter.Ax30.ReadTables.Order(), lists[0].Order()); + Assert.Equal(AxSchemaAdapter.Ax40.ReadTables.Order(), lists[0].Order()); + Assert.Equal(AxSchemaAdapter.Ax2009.ReadTables.Order(), lists[1].Order()); + Assert.Equal(AxSchemaAdapter.Ax2012.ReadTables.Order(), lists[2].Order()); + Assert.Equal(AxSchemaAdapter.FinanceOperations.ReadTables.Order(), lists[2].Order()); + Assert.Contains("SET @DatabaseName = N'AX30';", AxReadAccess.Script("AX30", "svc"), StringComparison.Ordinal); + } + + [Fact] + public void Grant_text_uses_the_existing_database_user_and_only_existing_tables() + { + var ax = AxFixtures.Recognise(AxFixtures.Ax30, "AX30", user: "sqlinsight") with { AgentLogin = "svc" }; + + var grant = AxReadAccess.GrantSql(ax, ax.AgentLogin); + + Assert.DoesNotContain("CREATE USER", grant, StringComparison.Ordinal); + Assert.Contains("GRANT SELECT ON [bmssa].[SYSTEMSEQUENCES] TO [sqlinsight];", grant, StringComparison.Ordinal); + Assert.DoesNotContain("BATCHJOB", grant, StringComparison.Ordinal); + Assert.Empty(AxReadAccess.Missing(ax)); + } +} + +public class AxInterpretationTests +{ + private static readonly DateTime Now = new(2026, 10, 2, 10, 0, 0, DateTimeKind.Utc); + + [Fact] + public void Ax30_times_are_a_local_date_plus_seconds_since_midnight() + { + // 02:30:00 local on a server two hours ahead of UTC (offset -120 minutes) is 00:30 UTC. + Assert.Equal(new DateTime(2026, 10, 2, 0, 30, 0, DateTimeKind.Utc), AxSchemaAdapter.Ax30.ToUtc(new DateTime(2026, 10, 2), 9000, -120)); + Assert.Null(AxSchemaAdapter.Ax30.ToUtc(new DateTime(1900, 1, 1), 0, 0)); + Assert.Equal(new DateTime(2026, 10, 2, 8, 0, 0, DateTimeKind.Utc), AxSchemaAdapter.Ax2009.ToUtc(new DateTime(2026, 10, 2, 8, 0, 0), null, -120)); + } + + [Fact] + public void Batch_status_enums_per_version() + { + Assert.Equal(AxBatchStatus.Executing, AxSchemaAdapter.Ax30.BatchStatus(1)); + Assert.Equal(AxBatchStatus.Error, AxSchemaAdapter.Ax30.BatchStatus(2)); + Assert.Equal(AxBatchStatus.Waiting, AxSchemaAdapter.Ax2009.BatchStatus(1)); + Assert.Equal(AxBatchStatus.Executing, AxSchemaAdapter.Ax2012.BatchStatus(2)); + Assert.Equal(AxBatchStatus.Canceled, AxSchemaAdapter.Ax2012.BatchStatus(8)); + Assert.Equal(AxBatchStatus.Unknown, AxSchemaAdapter.Ax2012.BatchStatus(42)); + Assert.All(AxSchemaAdapter.All, a => Assert.False(a.BatchStatusVerified)); + } + + [Fact] + public void Ax30_jobs_are_keyed_by_class_company_and_group_and_late_jobs_are_measured_from_the_planned_start() + { + var row = new AxBatchJobRow(17, 1234, null, "dat", "NAT", 0, new DateTime(2026, 10, 2), 9 * 3600, null, null, null, null, -120); + + var job = AxSchemaAdapter.Ax30.Interpret(row, new AxBatchHistory(), "AX30", null); + + Assert.Equal("1234|dat|NAT", job.Key); + Assert.Equal("Klasse 1234 (gruppe NAT)", job.Caption); + Assert.Equal(AxBatchStatus.Waiting, job.Status); + Assert.Equal(new DateTime(2026, 10, 2, 7, 0, 0, DateTimeKind.Utc), job.PlannedStartUtc); + Assert.Equal(TimeSpan.FromHours(3), job.Late(new DateTimeOffset(Now))); + } + + [Fact] + public void Job_history_gives_median_and_90th_percentile_of_successful_runs_and_is_bounded() + { + var history = new AxBatchHistory(); + for (var i = 1; i <= 30; i++) + { + history.Add("AX", new AxBatchRun("7", "H" + i, "Lager", "dat", Now.AddHours(-i), Now.AddHours(-i).AddMinutes(i <= 25 ? 10 : 200), i == 1 ? AxBatchStatus.Error : AxBatchStatus.Finished)); + } + + history.Add("AX", new AxBatchRun("7", "H3", "Lager", "dat", Now, Now.AddMinutes(500), AxBatchStatus.Finished)); // seen before: ignored + var runs = history.Runs("AX", "7"); + var (median, p90) = AxBatchHistory.Statistics(runs); + + Assert.Equal(AxBatchHistory.RunsKept, runs.Count); + Assert.Equal(600, median); + Assert.Equal(600, p90); + Assert.Empty(history.Runs("AX", "8")); + } + + [Fact] + public void Observed_run_needs_an_end_and_a_start_from_ax_or_from_sqlinsight() + { + var finished = new AxBatchJob { Key = "k", RecId = 5, Caption = "c", Status = AxBatchStatus.Finished, EndedUtc = Now }; + + Assert.Null(AxSchemaAdapter.ObservedRun(finished, null)); + Assert.Equal(TimeSpan.FromMinutes(30), AxSchemaAdapter.ObservedRun(finished, Now.AddMinutes(-30))!.Duration); + Assert.Equal(TimeSpan.FromMinutes(5), AxSchemaAdapter.ObservedRun(finished with { StartedUtc = Now.AddMinutes(-5) }, Now.AddMinutes(-30))!.Duration); + Assert.Null(AxSchemaAdapter.ObservedRun(finished with { Status = AxBatchStatus.Executing }, Now.AddMinutes(-30))); + } + + [Fact] + public void Number_sequence_usage_format_width_and_forecast() + { + var row = new AxNumberSequenceRow(1, "Debitor", "dat", "Debitorer", 1, 9_999_999, 95_001, "DEB-####", false, false, false, false, true, null, null, null, -120); + var samples = new[] { new AxSample(Now.AddDays(-20), 85_001), new AxSample(Now.AddDays(-10), 90_001) }; + + var sequence = AxSchemaAdapter.Ax30.Interpret(row, samples, Now, 30); + + Assert.Equal(4, sequence.FormatDigits); + Assert.Equal(9_999, sequence.FormatCapacity); + Assert.True(sequence.FormatTooShort); + Assert.Equal(9_999, sequence.Max); + Assert.Equal(100, sequence.UsedPercent); + Assert.Equal(0, sequence.Remaining); + Assert.Equal("dat/Debitor", sequence.Key); + Assert.Equal(500, sequence.Forecast!.PerDay, 3); + Assert.Equal(0, sequence.Forecast.DaysLeft); + } + + [Fact] + public void Number_sequence_forecast_needs_a_day_of_history_and_ignores_manual_and_cyclic() + { + var row = new AxNumberSequenceRow(1, "Ordre", "dat", null, 1, 1_000_000, 500_001, "######", false, false, false, false, null, null, null, null, 0); + + Assert.Null(AxSchemaAdapter.Ax2009.Interpret(row, [new AxSample(Now.AddHours(-2), 1)], Now, 30).Forecast); + Assert.Null(AxSchemaAdapter.Ax2009.Interpret(row with { Manual = true }, [new AxSample(Now.AddDays(-5), 1)], Now, 30).Forecast); + var sequence = AxSchemaAdapter.Ax2009.Interpret(row, [new AxSample(Now.AddDays(-5), 450_001), new AxSample(Now.AddDays(-60), 1)], Now, 30); + Assert.Equal(10_000, sequence.Forecast!.PerDay, 3); + Assert.Equal(50, sequence.Forecast.DaysLeft!.Value, 3); + Assert.Equal(50, sequence.UsedPercent, 3); + Assert.Equal("1", AxSchemaAdapter.Ax2012.SequenceKey(row)); + } + + [Fact] + public void Ax30_recid_is_limited_by_the_int_range() + { + var recId = AxSchemaAdapter.Interpret(new AxRecIdRow(-1, "dat", 1_610_612_736, 0, null), nextValIsInt: true, + [new AxSample(Now.AddDays(-10), 1_600_612_736)], Now, 30); + var bigint = AxSchemaAdapter.Interpret(new AxRecIdRow(77, null, 5_000_000_000, null, "CUSTTABLE"), nextValIsInt: false, [], Now, 30); + + Assert.Equal((32, int.MaxValue, 75.0), (recId.Bits, recId.Max, Math.Round(recId.UsedPercent, 1))); + Assert.Equal(1_000_000, recId.Forecast!.PerDay, 3); + Assert.Equal(536.87, recId.Forecast.DaysLeft!.Value, 2); + Assert.Equal("-1/dat", recId.Key); + Assert.Equal((64, long.MaxValue), (bigint.Bits, bigint.Max)); + Assert.True(bigint.UsedPercent < 0.001); + } +} + +public class AxSessionMapperTests +{ + private static string Hex(string text, bool unicode = false) => + Convert.ToHexString(unicode ? System.Text.Encoding.Unicode.GetBytes(text) : System.Text.Encoding.ASCII.GetBytes(text)) + "000000"; + + private static SessionInfo Session(string? context) => new() { SessionId = 61, HostName = "AOS01", ProgramName = "Microsoft Dynamics AX", ContextInfo = context }; + + [Fact] + public void Ax2012_context_info_gives_the_ax_user_and_session() + { + var identity = AxSessionMapper.Map(Session(Hex("anne 23")), AxVersion.Ax2012R2); + + Assert.Equal(("anne", 23, "context_info"), (identity.AxUser, identity.AxSessionId, identity.Source)); + Assert.Equal("anne", AxSessionMapper.Map(Session(Hex("23 anne", unicode: true)), AxVersion.Ax2009).AxUser); + } + + [Fact] + public void Ax30_is_honest_that_the_user_cannot_be_seen() + { + var identity = AxSessionMapper.Map(Session(null), AxVersion.Axapta30); + + Assert.False(identity.Known); + Assert.Contains("sætter ikke CONTEXT_INFO", identity.Explanation, StringComparison.Ordinal); + Assert.Contains("AOS01", identity.Explanation, StringComparison.Ordinal); + } + + [Fact] + public void Without_context_info_the_explanation_says_how_to_turn_it_on() + { + var identity = AxSessionMapper.Map(Session(null), AxVersion.Ax2012); + + Assert.False(identity.Known); + Assert.Contains("connectioncontext", identity.Explanation, StringComparison.Ordinal); + Assert.Null(AxSessionMapper.Decode("00000000")); + Assert.Null(AxSessionMapper.Decode("zz")); + Assert.Null(AxSessionMapper.Decode("01FF02FE")); + } +} + +public class DynamicsAxHubSubjectTests +{ + [Theory] + [InlineData("database: AX30", "AX30")] + [InlineData("database: AX30 · batch: 17", "AX30")] + [InlineData("database: AX 30 · nummerserie: dat/Debitor", "AX 30")] + [InlineData("job: x", null)] + public void Database_of_an_ax_subject(string subject, string? database) => Assert.Equal(database, DynamicsAxFindings.DatabaseOf(subject)); + + [Theory] + [InlineData("SQLI-AX-005", true)] + [InlineData("COM-AX-001", true)] + [InlineData("SQLI-AG-001", false)] + public void Ax_rules(string ruleId, bool ax) => Assert.Equal(ax, DynamicsAxFindings.IsDynamicsAxRule(ruleId)); + + [Fact] + public void Subjects_round_trip() => Assert.Equal("AX30", DynamicsAxFindings.DatabaseOf(DynamicsAxFindings.Subject("AX30", "batch: 1"))); +} diff --git a/tests/SQLInsight.Core.Tests/SQLInsight.Core.Tests.csproj b/tests/SQLInsight.Core.Tests/SQLInsight.Core.Tests.csproj index 923a479..673fc63 100644 --- a/tests/SQLInsight.Core.Tests/SQLInsight.Core.Tests.csproj +++ b/tests/SQLInsight.Core.Tests/SQLInsight.Core.Tests.csproj @@ -8,6 +8,10 @@ + + + +