Review 1.4: the AX batch history watermark no longer gets stuck

After the first read, BATCHJOBHISTORY was read from the watermark minus an
overlap of 2000 RecIds with TOP (1000), oldest first. From AX 4.0 RecIds are
handed out per table and are dense, so once 1000 rows existed in the overlap
every read returned the same old rows: the watermark never moved and new runs
were never read again (every AX 2009/2012 installation with more than ~1000
history rows). The read after the watermark now takes the overlap plus 1000
rows (TOP (3000)), so at least 1000 new runs are read each time. Test with
20,000 dense RecIds.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

SQL Scripts/SQLI-AX-001.sql

@@ -131,7 +131,7 @@ GO
 -- Kilde: AxSchemaAdapter.BatchHistory(2009, næste)
 -- Kode:  src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs : AxSchemaAdapter.BatchHistory(database, afterRecId) via AxSql.InDatabase
 -- Agenten indsætter den højeste RECID, den har læst, minus overlappet, hvor der her står 0.
-EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (1000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
+EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (3000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
 SET LOCK_TIMEOUT 5000; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
 GO
 
@@ -153,7 +153,7 @@ GO
 -- Kilde: AxSchemaAdapter.BatchHistory(2012, næste)
 -- Kode:  src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs : AxSchemaAdapter.BatchHistory(database, afterRecId) via AxSql.InDatabase
 -- Agenten indsætter den højeste RECID, den har læst, minus overlappet, hvor der her står 0.
-EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (1000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
+EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (3000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
 SET LOCK_TIMEOUT 5000; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
 GO
 

SQL Scripts/SQLI-AX-002.sql

@@ -135,7 +135,7 @@ GO
 -- Kilde: AxSchemaAdapter.BatchHistory(2009, næste)
 -- Kode:  src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs : AxSchemaAdapter.BatchHistory(database, afterRecId) via AxSql.InDatabase
 -- Agenten indsætter den højeste RECID, den har læst, minus overlappet, hvor der her står 0.
-EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (1000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
+EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (3000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
 SET LOCK_TIMEOUT 5000; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
 GO
 
@@ -157,7 +157,7 @@ GO
 -- Kilde: AxSchemaAdapter.BatchHistory(2012, næste)
 -- Kode:  src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs : AxSchemaAdapter.BatchHistory(database, afterRecId) via AxSql.InDatabase
 -- Agenten indsætter den højeste RECID, den har læst, minus overlappet, hvor der her står 0.
-EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (1000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
+EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (3000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
 SET LOCK_TIMEOUT 5000; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
 GO
 

SQL Scripts/SQLI-AX-003.sql

@@ -134,7 +134,7 @@ GO
 -- Kilde: AxSchemaAdapter.BatchHistory(2009, næste)
 -- Kode:  src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs : AxSchemaAdapter.BatchHistory(database, afterRecId) via AxSql.InDatabase
 -- Agenten indsætter den højeste RECID, den har læst, minus overlappet, hvor der her står 0.
-EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (1000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
+EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (3000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
 SET LOCK_TIMEOUT 5000; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
 GO
 
@@ -156,7 +156,7 @@ GO
 -- Kilde: AxSchemaAdapter.BatchHistory(2012, næste)
 -- Kode:  src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs : AxSchemaAdapter.BatchHistory(database, afterRecId) via AxSql.InDatabase
 -- Agenten indsætter den højeste RECID, den har læst, minus overlappet, hvor der her står 0.
-EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (1000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
+EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (3000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
 SET LOCK_TIMEOUT 5000; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
 GO
 

SQL Scripts/SQLI-AX-004.sql

@@ -135,7 +135,7 @@ GO
 -- Kilde: AxSchemaAdapter.BatchHistory(2009, næste)
 -- Kode:  src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs : AxSchemaAdapter.BatchHistory(database, afterRecId) via AxSql.InDatabase
 -- Agenten indsætter den højeste RECID, den har læst, minus overlappet, hvor der her står 0.
-EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (1000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
+EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (3000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
 SET LOCK_TIMEOUT 5000; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
 GO
 
@@ -157,7 +157,7 @@ GO
 -- Kilde: AxSchemaAdapter.BatchHistory(2012, næste)
 -- Kode:  src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs : AxSchemaAdapter.BatchHistory(database, afterRecId) via AxSql.InDatabase
 -- Agenten indsætter den højeste RECID, den har læst, minus overlappet, hvor der her står 0.
-EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (1000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
+EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (3000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
 SET LOCK_TIMEOUT 5000; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
 GO
 

SQL Scripts/_collectors/AxBatchCollector.sql

@@ -70,7 +70,7 @@ GO
 -- Kilde: AxSchemaAdapter.BatchHistory(2009, næste)
 -- Kode:  src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs : AxSchemaAdapter.BatchHistory(database, afterRecId) via AxSql.InDatabase
 -- Agenten indsætter den højeste RECID, den har læst, minus overlappet, hvor der her står 0.
-EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (1000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
+EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (3000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
 SET LOCK_TIMEOUT 5000; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
 GO
 
@@ -92,7 +92,7 @@ GO
 -- Kilde: AxSchemaAdapter.BatchHistory(2012, næste)
 -- Kode:  src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs : AxSchemaAdapter.BatchHistory(database, afterRecId) via AxSql.InDatabase
 -- Agenten indsætter den højeste RECID, den har læst, minus overlappet, hvor der her står 0.
-EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (1000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
+EXEC [AXDB].sys.sp_executesql N'SET NOCOUNT ON; SET LOCK_TIMEOUT 2000; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT TOP (3000) CAST(h.[RECID] AS bigint), CAST(h.[BATCHJOBID] AS bigint), h.[CAPTION], h.[COMPANY], h.[STATUS], h.[STARTDATETIME], h.[ENDDATETIME], DATEDIFF(minute, GETDATE(), GETUTCDATE()) FROM [dbo].[BATCHJOBHISTORY] h WHERE h.[RECID] > 0 ORDER BY h.[RECID]';
 SET LOCK_TIMEOUT 5000; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
 GO
 

src/SQLInsight.Core/DynamicsAx/AxSchemaAdapter.cs

@@ -26,6 +26,13 @@ public abstract class AxSchemaAdapter
     /// <summary>RecIds are handed out in blocks per AOS, so history rows may arrive slightly out of RECID order: re-read this far back.</summary>
     public const long HistoryOverlap = 2000;
 
+    /// <summary>
+    /// Rows read after the watermark: the overlap plus <see cref="MaxHistoryRows"/> new rows. AX 4.0 and later hand out
+    /// RecIds per table, so BATCHJOBHISTORY's RecIds are dense; with no more rows than the overlap the read would return the
+    /// same old rows every time and the watermark would never move again.
+    /// </summary>
+    public const int MaxHistoryRowsAfterWatermark = (int)HistoryOverlap + MaxHistoryRows;
+
     /// <summary>How many number sequences are read (the most used first).</summary>
     public const int MaxNumberSequences = 5000;
 
@@ -485,7 +492,7 @@ public abstract class AxSchemaAdapter
 
         var recId = "h." + AxSql.Identifier("RECID");
         return AxQuery.Ok(afterRecId is { } after
-            ? $"SELECT TOP ({MaxHistoryRows}) {h.SelectList} FROM {database.Qualified("BATCHJOBHISTORY")} h WHERE {recId} > {AxSql.Number(Math.Max(0, after - HistoryOverlap))} ORDER BY {recId}"
+            ? $"SELECT TOP ({MaxHistoryRowsAfterWatermark}) {h.SelectList} FROM {database.Qualified("BATCHJOBHISTORY")} h WHERE {recId} > {AxSql.Number(Math.Max(0, after - HistoryOverlap))} ORDER BY {recId}"
             : $"SELECT TOP ({MaxHistoryRows}) {h.SelectList} FROM {database.Qualified("BATCHJOBHISTORY")} h ORDER BY {recId} DESC", h.Notes);
     }
 

tests/SQLInsight.Core.Tests/DynamicsAxTests.cs

@@ -371,3 +371,28 @@ public class DynamicsAxHubSubjectTests
     [Fact]
     public void Subjects_round_trip() => Assert.Equal("AX30", DynamicsAxFindings.DatabaseOf(DynamicsAxFindings.Subject("AX30", "batch: 1")));
 }
+
+public class AxBatchHistoryWatermarkTests
+{
+    [Fact]
+    public void The_history_watermark_keeps_moving_through_dense_recids()
+    {
+        // AX 4.0 and later: BATCHJOBHISTORY's RecIds are dense. A read returns the rows after watermark - overlap, in RECID
+        // order, at most MaxHistoryRowsAfterWatermark of them.
+        Assert.True(AxSchemaAdapter.MaxHistoryRowsAfterWatermark > AxSchemaAdapter.HistoryOverlap);
+        var history = Enumerable.Range(1, 20_000).Select(i => (long)i).ToList();
+        long watermark = history[AxSchemaAdapter.MaxHistoryRows - 1];
+        for (var read = 0; read < 25 && watermark < history[^1]; read++)
+        {
+            var after = Math.Max(0, watermark - AxSchemaAdapter.HistoryOverlap);
+            var rows = history.Where(r => r > after).Take(AxSchemaAdapter.MaxHistoryRowsAfterWatermark).ToList();
+            Assert.True(rows[^1] > watermark, $"Læsning {read} kom ikke forbi RECID {watermark}.");
+            watermark = rows[^1];
+        }
+
+        Assert.Equal(history[^1], watermark);
+
+        var database = AxFixtures.Recognise(AxFixtures.Schema("schema-ax2012"), databases: ["AXDB", "AXDB_model"]);
+        Assert.Contains($"SELECT TOP ({AxSchemaAdapter.MaxHistoryRowsAfterWatermark}) ", AxSchemaAdapter.Ax2012.BatchHistory(database, 50_000)!.Sql, StringComparison.Ordinal);
+    }
+}