diff --git a/db/permissions/revoke-dynamics-ax-read.sql b/db/permissions/revoke-dynamics-ax-read.sql index 0e3011d..52b0b67 100644 --- a/db/permissions/revoke-dynamics-ax-read.sql +++ b/db/permissions/revoke-dynamics-ax-read.sql @@ -36,13 +36,31 @@ BEGIN RETURN; END; +/* Only the AX schema (where SQLDICTIONARY and DATAAREA are, as grant-dynamics-ax-read.sql finds it), and only a SELECT + that was granted: a DENY, or a grant on a same-named table in another schema set up for another reason, is left alone. */ +DECLARE @schema sysname; +SELECT TOP 1 @schema = s.name +FROM sys.tables t +JOIN sys.schemas s ON s.schema_id = t.schema_id +WHERE UPPER(t.name) = N''SQLDICTIONARY'' AND OBJECT_ID(QUOTENAME(s.name) + N''.DATAAREA'', N''U'') IS NOT NULL +ORDER BY CASE WHEN s.name = N''dbo'' THEN 0 ELSE 1 END; +IF @schema IS NULL +BEGIN + PRINT DB_NAME() + N'' er ikke en Dynamics AX-database (SQLDICTIONARY og DATAAREA findes ikke i samme skema). Intet er ændret.''; + RETURN; +END; + SET @revoke = N''''; SET @count = 0; SELECT @revoke = @revoke + N''REVOKE SELECT ON '' + QUOTENAME(s.name) + N''.'' + QUOTENAME(t.name) + N'' FROM '' + QUOTENAME(@user) + N''; '', @count = @count + 1 FROM sys.tables t JOIN sys.schemas s ON s.schema_id = t.schema_id -WHERE UPPER(t.name) IN (N''BATCH'', N''BATCHJOB'', N''BATCHJOBHISTORY'', N''NUMBERSEQUENCETABLE'', N''NUMBERSEQUENCESCOPE'', - N''NUMBERSEQUENCEREFERENCE'', N''SYSTEMSEQUENCES'', N''SQLDICTIONARY'', N''DATAAREA''); +WHERE s.name = @schema + AND UPPER(t.name) IN (N''BATCH'', N''BATCHJOB'', N''BATCHJOBHISTORY'', N''NUMBERSEQUENCETABLE'', N''NUMBERSEQUENCESCOPE'', + N''NUMBERSEQUENCEREFERENCE'', N''SYSTEMSEQUENCES'', N''SQLDICTIONARY'', N''DATAAREA'') + AND EXISTS (SELECT 1 FROM sys.database_permissions p + WHERE p.class = 1 AND p.major_id = t.object_id AND p.minor_id = 0 AND p.grantee_principal_id = USER_ID(@user) + AND p.permission_name = N''SELECT'' AND p.state IN (''G'', ''W'')); IF @count > 0 EXEC (@revoke); PRINT N''SELECT fjernet på '' + CAST(@count AS nvarchar(10)) + N'' AX-tabeller i '' + DB_NAME() + N'' for '' + @user + N''.'';'; EXEC @Procedure @Sql, N'@login sysname', @login = @LoginName; diff --git a/docs/dynamics-ax.md b/docs/dynamics-ax.md index 82cfc63..bcda43c 100644 --- a/docs/dynamics-ax.md +++ b/docs/dynamics-ax.md @@ -36,8 +36,8 @@ Uden adgangen er tjekket **ikke målbart**, og årsagen er den nøjagtige `GRANT ## Sådan læser SQLInsight AX - **Kun SELECT**, kun på standardtabellerne ovenfor, og aldrig i kundens database ud over det: al historik (batchkørsler, målinger af nummerserier) gemmes i SQLInsights eget repository. -- **Begrænset:** `TOP` på hver forespørgsel (1000 batchjob, 1000 historikrækker, de 5000 mest brugte nummerserier, de 500 højeste RecId-serier), og historikken læses efter RECID, som AX altid har et unikt indeks på. -- **Uden at vente på AX:** `READ COMMITTED` med `SET LOCK_TIMEOUT 2000` – er en række låst, opgiver læsningen efter 2 sekunder og prøver igen næste gang (det forrige billede beholdes, så fund ikke blinker). Har databasen `ALLOW_SNAPSHOT_ISOLATION`, bruges `SNAPSHOT`; med `READ_COMMITTED_SNAPSHOT` læses versioner. **Aldrig `NOLOCK`.** +- **Begrænset:** `TOP` på hver forespørgsel (1000 batchjob; 1000 historikrækker første gang og derefter 3000 fra seneste læste RECID minus et overlap på 2000, så mindst 1000 nye kørsler læses hver gang; de 5000 mest brugte nummerserier; de 500 højeste RecId-serier), og historikken læses efter RECID, som AX altid har et unikt indeks på. +- **Uden at vente på AX:** `READ COMMITTED` med `SET LOCK_TIMEOUT 2000` – er en række låst, opgiver læsningen efter 2 sekunder og prøver igen næste gang (det forrige billede beholdes i op til 15 minutter, så fund ikke blinker). Bagefter sættes agentens egne sessionsindstillinger igen (`LOCK_TIMEOUT 5000`, `READ UNCOMMITTED` til DMV-forespørgslerne), så de næste collectors ikke påvirkes. Har databasen `ALLOW_SNAPSHOT_ISOLATION`, bruges `SNAPSHOT`; med `READ_COMMITTED_SNAPSHOT` læses versioner. **Aldrig `NOLOCK`.** - **Uden at skifte database:** forespørgslerne køres i AX-databasen med `[db].sys.sp_executesql`. - **Kolonnetjek først:** hver forespørgsel bygges ud fra de kolonner, databasen faktisk har. Mangler en nødvendig kolonne, er tjekket "ikke målbart" med den præcise kolonne; mangler en valgfri, udelades den med en note. - **SQL Server 2005 og nyere.** På SQL Server 2000 (legacy ODBC) genkendes AX under *Servere og databaser*, men datatjekkene kører ikke. @@ -96,15 +96,16 @@ AX-fund (`SQLI-AX-*` og `COM-AX-*`) sendes kun til SQLInsight Hub for databaser, | Endpoint | Indhold | |---|---| | `GET /api/v1/dynamics-ax/databases` | Genkendte AX-databaser: version, verificeret, skema, modeldatabase, firmaer, læseadgang pr. tabel, manglende tabeller og GRANT | +| `GET /api/v1/dynamics-ax/detection-problems` | Overvågede databaser, AX-genkendelsen ikke kunne læse (fx ingen databasebruger før SQL Server 2014) | | `GET /api/v1/dynamics-ax/grant-script?instance=&database=` | GRANT-sætningerne og hele `grant-dynamics-ax-read.sql` udfyldt for databasen | -| `GET /api/v1/dynamics-ax/batch-jobs` | Batchjob nu: status, planlagt/faktisk start, forsinkelse, kørselstid, median og 90 %-fraktil, rå STATUS-værdier | +| `GET /api/v1/dynamics-ax/batch-jobs` | Batchjob nu: status, planlagt/faktisk start, forsinkelse, kørselstid, median og 90 %-fraktil, rå STATUS-værdier og hvad de betyder i versionen | | `GET /api/v1/dynamics-ax/batch-runs?database=&job=&days=7` | Afsluttede kørsler fra repository'et | | `GET /api/v1/dynamics-ax/number-sequences` | Nummerserier med forbrug, format, prognose og flag | | `GET /api/v1/dynamics-ax/recids` | RecId pr. tabel med grænse og prognose | -| `GET /api/v1/dynamics-ax/sequence-samples?instance=&database=&kind=N\|R&key=&days=30` | De daglige målinger bag prognoserne | +| `GET /api/v1/dynamics-ax/sequence-samples?instance=&database=&kind=N\|R&key=&days=30` | De daglige målinger bag prognoserne (højst 10.000; `X-Truncated: true`, når der er flere) | | `GET /api/v1/dynamics-ax/blocking?days=7` | AX-brugere bag blokering | -Alle tager `?instance=` for én instans. AX-bruger, vært, program, sætning og agentens login er `[udeladt]`, medmindre `SQLInsight:Web:ApiIncludesSensitiveEvidence` er slået til. +Alle tager `?instance=` for én instans; fjernede servere er ikke med. AX-bruger, vært, program, sætning og rå `CONTEXT_INFO` er `[udeladt]`, og agentens login og databasebruger erstattes af pladsholderen `DOMÆNE\svc-sqlinsight$` (også i GRANT-teksterne i "ikke målbar" og i `/api/v1/instances/{name}/rule-states`), medmindre `SQLInsight:Web:ApiIncludesSensitiveEvidence` er slået til. ## Nyt kundeskema diff --git a/src/SQLInsight.Collectors/DynamicsAx/AxBatchCollector.cs b/src/SQLInsight.Collectors/DynamicsAx/AxBatchCollector.cs index 5dbcbef..e023318 100644 --- a/src/SQLInsight.Collectors/DynamicsAx/AxBatchCollector.cs +++ b/src/SQLInsight.Collectors/DynamicsAx/AxBatchCollector.cs @@ -20,6 +20,12 @@ public sealed class AxBatchCollector(AxDatabaseDetector detector, AxThresholds? private const int JobsShown = 500; + /// + /// A lock timeout keeps the earlier picture (so findings do not flap), but no longer than this: running and late times + /// keep growing against an old picture, and a job that has started meanwhile would still be reported late. + /// + internal static readonly TimeSpan KeptThroughLockTimeouts = TimeSpan.FromMinutes(15); + private readonly AxThresholds _thresholds = thresholds ?? new AxThresholds(); public override string Name => "dynamics-ax-batch"; @@ -63,8 +69,9 @@ public sealed class AxBatchCollector(AxDatabaseDetector detector, AxThresholds? var (rows, problem) = await AxReader.ReadAsync(context, database, query, Map).ConfigureAwait(false); if (rows is null) { - // A lock timeout keeps the previous picture (marked as such), so findings do not flap. + // A lock timeout keeps the previous picture for a while (marked as such), so findings do not flap. return AxReader.IsTransient(problem) && earlier is { NotMeasurable: null } + && context.Now - (earlier.MeasuredAt ?? previous!.CapturedAt) <= KeptThroughLockTimeouts ? earlier with { NewRuns = [], Notes = [.. earlier.Notes.Where(n => n != AxReader.LockTimeoutText), AxReader.LockTimeoutText] } : new AxBatchDatabase { Database = name, Version = database.Version, NotMeasurable = problem }; } @@ -147,6 +154,7 @@ public sealed class AxBatchCollector(AxDatabaseDetector detector, AxThresholds? StatusCounts = rows.GroupBy(r => r.Status).ToDictionary(g => g.Key, g => g.Count()), Notes = notes, HistoryWatermark = watermark, + MeasuredAt = context.Now, }; } diff --git a/src/SQLInsight.Collectors/DynamicsAx/AxDatabaseDetector.cs b/src/SQLInsight.Collectors/DynamicsAx/AxDatabaseDetector.cs index 8d5847e..42617ae 100644 --- a/src/SQLInsight.Collectors/DynamicsAx/AxDatabaseDetector.cs +++ b/src/SQLInsight.Collectors/DynamicsAx/AxDatabaseDetector.cs @@ -57,7 +57,7 @@ public sealed class AxDatabaseDetector if (force || !_cache.TryGetValue(name, out var cached) || context.Now - cached.At >= RefreshFor(cached.Database)) { - cached = await DetectAsync(context, name, names).ConfigureAwait(false); + cached = await DetectAsync(context, name, names, _cache.TryGetValue(name, out var earlier) ? earlier.Database : null).ConfigureAwait(false); _cache[name] = cached; } @@ -84,7 +84,9 @@ public sealed class AxDatabaseDetector return new AxDatabaseSnapshot { CapturedAt = context.Now, Databases = found, AgentLogin = _login, Problems = problems }; } - private async Task<(DateTimeOffset, AxDatabase?, string?)> DetectAsync(CollectorContext context, string database, IReadOnlyList names) + /// What the previous detection found: kept through a passing error (a lock timeout during an AX + /// table synchronisation, a deadlock), so the database does not drop out of AX monitoring for an hour. + private async Task<(DateTimeOffset, AxDatabase?, string?)> DetectAsync(CollectorContext context, string database, IReadOnlyList names, AxDatabase? earlier) { try { @@ -106,8 +108,14 @@ public sealed class AxDatabaseDetector return (context.Now, ax, null); } - catch (DbException ex) + catch (DbException ex) when (!context.CancellationToken.IsCancellationRequested) { + if (earlier is not null && Db.ErrorNumber(ex) is AxSql.LockTimeoutError or 1205 or null) + { + // A passing error: keep what was found and try again within minutes. + return (context.Now - Refresh + RefreshWithoutReadAccess, earlier, null); + } + // No access to its metadata (e.g. no database user before 2014): it cannot be told apart, and nothing AX-specific runs there. return (context.Now, null, "AX-genkendelsen kunne ikke læse databasens metadata: " + ex.Message); } @@ -136,8 +144,9 @@ internal static class AxReader return (await Db.QueryAsync(context.Connection, AxSql.InDatabase(database.Database, sql, database.Isolation), map, AxSql.CommandTimeoutSeconds, context.CancellationToken).ConfigureAwait(false), null); } - catch (DbException ex) + catch (DbException ex) when (!context.CancellationToken.IsCancellationRequested) { + // (A cancelled collection is not an AX problem: SqlClient reports it as a DbException, which is passed on.) return (null, Db.ErrorNumber(ex) switch { AxSql.LockTimeoutError => LockTimeoutText, diff --git a/src/SQLInsight.Core/DynamicsAx/AxBatch.cs b/src/SQLInsight.Core/DynamicsAx/AxBatch.cs index d78ead0..3283e2b 100644 --- a/src/SQLInsight.Core/DynamicsAx/AxBatch.cs +++ b/src/SQLInsight.Core/DynamicsAx/AxBatch.cs @@ -120,6 +120,9 @@ public sealed record AxBatchDatabase /// The highest BATCHJOBHISTORY RECID read: the next read starts there (2009 and later). public long? HistoryWatermark { get; init; } + + /// When the batch tables were last read (a lock timeout keeps the earlier picture for a while, see AxBatchCollector). + public DateTimeOffset? MeasuredAt { get; init; } } /// The batch jobs of every monitored AX database on an instance (collector "dynamics-ax-batch", every 5 minutes). diff --git a/src/SQLInsight.Rules/DynamicsAx/DynamicsAxRules.cs b/src/SQLInsight.Rules/DynamicsAx/DynamicsAxRules.cs index 4c295f7..107dfc3 100644 --- a/src/SQLInsight.Rules/DynamicsAx/DynamicsAxRules.cs +++ b/src/SQLInsight.Rules/DynamicsAx/DynamicsAxRules.cs @@ -45,6 +45,9 @@ public abstract class DynamicsAxRule : Rule protected static string Seconds(double? seconds) => seconds is { } s ? Duration(TimeSpan.FromSeconds(s)) : "–"; + /// When a forecast runs out: "om ca. 12 dage", or "inden for et døgn" (never "om ca. 0 dage"). + protected static string DaysText(double days) => days < 1 ? "inden for et døgn" : $"om ca. {Format.Number(days)} dage"; + protected static Evidence Version(AxVersion version) => new("AX-version", AxVersions.Text(version) + (AxVersions.IsVerified(version) ? string.Empty : " – " + AxVersions.VerificationText(version))); } @@ -295,7 +298,7 @@ public sealed class AxNumberSequenceExhaustionRule : DynamicsAxNumberSequenceRul s.Remaining == 0 ? $"Nummerserien {Name(s)} i {database.Database} er opbrugt" : s.Forecast?.DaysLeft is { } days && days <= thresholds.ForecastWarnDays - ? $"Nummerserien {Name(s)} i {database.Database} løber tør om ca. {Format.Number(days)} dage" + ? $"Nummerserien {Name(s)} i {database.Database} løber tør {DaysText(days)}" : $"Nummerserien {Name(s)} i {database.Database} er {Format.Number(s.UsedPercent, 1)} % brugt", "Når en nummerserie er brugt op, kan AX ikke tildele flere numre: oprettelse eller bogføring af fx ordrer, fakturaer eller posteringer, " + "der bruger serien, fejler for alle brugere. Prognosen bygger på SQLInsights egne daglige målinger af NEXTREC.", @@ -305,7 +308,7 @@ public sealed class AxNumberSequenceExhaustionRule : DynamicsAxNumberSequenceRul new Evidence("Brugt", $"{Format.Number(s.UsedPercent, 1)} %"), new Evidence("Numre tilbage", Format.Number(s.Remaining)), new Evidence("Forbrug pr. dag", s.Forecast is { PerDay: > 0 } f ? $"{Format.Number(f.PerDay, 1)} (målt over {Format.Number(f.BasisDays, 1)} dage)" : "ukendt endnu"), - new Evidence("Løber tør om", s.Forecast?.DaysLeft is { } left ? $"{Format.Number(left)} dage" : "–"), + new Evidence("Løber tør", s.Forecast?.DaysLeft is { } left ? DaysText(left) : "–"), ]); } } @@ -319,7 +322,7 @@ public sealed class AxNumberSequenceFormatRule : DynamicsAxNumberSequenceRule protected override IEnumerable Check(AxSequenceDatabase database, AxThresholds thresholds, DateTimeOffset now) { - foreach (var s in database.NumberSequences.Where(s => s.FormatTooShort && !s.Manual).OrderByDescending(s => s.Next)) + foreach (var s in database.NumberSequences.Where(s => s.FormatTooShort && !s.Manual && !s.Blocked).OrderByDescending(s => s.Next)) { var overflowing = s.Next > s.FormatCapacity; yield return SequenceFinding(database, s, overflowing ? Severity.High : Severity.Medium, @@ -446,7 +449,7 @@ public sealed class AxRecIdExhaustionRule : DynamicsAxRule new Evidence("Næste RecId (NEXTVAL)", Format.Number(r.Next)), new Evidence("Grænse", Format.Number(r.Max) + (r.Bits == 32 ? " (32-bit)" : string.Empty)), new Evidence("Forbrug pr. dag", r.Forecast is { PerDay: > 0 } f ? Format.Number(f.PerDay) : "ukendt endnu"), - new Evidence("Løber tør om", r.Forecast?.DaysLeft is { } d ? $"{Format.Number(d)} dage" : "–"), + new Evidence("Løber tør", r.Forecast?.DaysLeft is { } d ? DaysText(d) : "–"), Version(database.Version), ], DynamicsAxFindings.Subject(database.Database, "recid: " + r.Key)); diff --git a/tests/SQLInsight.IntegrationTests/DynamicsAxTests.cs b/tests/SQLInsight.IntegrationTests/DynamicsAxTests.cs index 737e150..6b07b61 100644 --- a/tests/SQLInsight.IntegrationTests/DynamicsAxTests.cs +++ b/tests/SQLInsight.IntegrationTests/DynamicsAxTests.cs @@ -252,6 +252,13 @@ public class DynamicsAxTests(SqlServerFixture sql) : IAsyncLifetime var order = sequences.NumberSequences.Single(); Assert.Equal(("dk01", true, "2001"), (order.Company, order.Referenced, order.Key)); Assert.Equal(64, sequences.RecIds.Single().Bits); + + // The collectors of an instance share the connection: after the AX reads it has the agent's own settings again. + await using var check = session.Connection.CreateCommand(); + check.CommandText = "SELECT @@LOCK_TIMEOUT, (SELECT transaction_isolation_level FROM sys.dm_exec_sessions WHERE session_id = @@SPID)"; + await using var reader = await check.ExecuteReaderAsync(); + Assert.True(await reader.ReadAsync()); + Assert.Equal((5000, 1), (reader.GetInt32(0), (int)reader.GetInt16(1))); } [Fact] @@ -294,7 +301,8 @@ public class DynamicsAxTests(SqlServerFixture sql) : IAsyncLifetime .Where(f => Path.GetFileName(f).Contains("AX", StringComparison.OrdinalIgnoreCase))) { var text = await File.ReadAllTextAsync(file); - foreach (var batch in Regex.Split(text[text.IndexOf("-- >>>", StringComparison.Ordinal)..], @"^\s*GO\s*$", RegexOptions.Multiline)) + // Every block with SQL from the code ("-- Kilde:"), as a DBA would run it in SSMS. + foreach (var batch in Regex.Split(text, @"^\s*GO\s*$", RegexOptions.Multiline).Where(b => b.Contains("-- Kilde: ", StringComparison.Ordinal))) { var variant = Regex.Match(batch, @"-- Variant: (?[^\r\n]*)").Groups["v"].Value; var target = variant.StartsWith("Axapta 3.0", StringComparison.Ordinal) ? _ax30