Review 1.5: disk forecasts, 24-hour event counts, performance gaps and per-user tasks in Core

- DiskForecast.EnoughHistory: false for "ingen målinger endnu" and "for lidt historik",
  so a rule can say "ikke målbar" instead of "fyldes ikke op". Days are counted from
  now (a last point older than its hour is subtracted); FullAt takes the time the
  forecast was made; Describe says "fuld nu" at 0 days.
- ServerEventGroup.Buckets: counts per clock hour, and Recent sums only the hours with
  events in the last 24 hours (FirstAt within the window), so a recurring error no
  longer adds up from when it was first seen. Older groups without buckets still merge.
- ServerPerformanceMath.Sample takes the interval: after a gap of more than three
  intervals CPU and pages/s are null rather than the gap's average stamped now.
  Commit percent is computed in floating point.
- TaskResults.IsUserOrUpdaterTask (a user's SID in the name, OneDrive/Edge/Google
  updaters, User_Feed_Synchronization), HasUserSid and Display (SID replaced by a
  short hash for titles and subjects).
- Tests: forecast from real history, irregular sampling, identical timestamps,
  a shrunk volume, days from now; an error every ten hours over three days; a
  sample after a gap.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/SQLInsight.Core/Servers/DiskForecast.cs

@@ -6,18 +6,28 @@ namespace SQLInsight.Core.Servers;
 public sealed record DiskHistoryPoint(DateTimeOffset At, double SizeGb, double FreeGb);
 
 /// <summary>When a volume will be full if it keeps filling at the rate of the last weeks.</summary>
-/// <param name="DaysUntilFull">Null when the volume is not filling up (or the trend is too unclear to say).</param>
+/// <param name="DaysUntilFull">
+/// Counted from when the forecast was made (the disk snapshot's CapturedAt, which <see cref="FullAt"/> takes). Null when
+/// the volume is not filling up, the trend is too unclear to say, or there is too little history.
+/// </param>
 /// <param name="GrowthGbPerDay">How fast the used space grows (positive: filling up).</param>
 /// <param name="Fit">R² of the trend line: 1 is a straight line, near 0 no trend.</param>
 /// <param name="Note">Why there is no forecast (Danish), when there is none.</param>
-public sealed record DiskForecast(string Volume, double? DaysUntilFull, double GrowthGbPerDay, int Samples, double HistoryDays, double Fit, string? Note = null)
+/// <param name="EnoughHistory">
+/// Whether there was enough history to judge the trend at all. False: the volume could not be judged yet, so a rule
+/// must say "ikke målbar" rather than "fyldes ikke op".
+/// </param>
+public sealed record DiskForecast(string Volume, double? DaysUntilFull, double GrowthGbPerDay, int Samples, double HistoryDays, double Fit, string? Note = null,
+    bool EnoughHistory = true)
 {
-    public DateTimeOffset? FullAt(DateTimeOffset now) => DaysUntilFull is { } days ? now.AddDays(days) : null;
+    /// <param name="madeAt">When the forecast was made (the disk snapshot's CapturedAt).</param>
+    public DateTimeOffset? FullAt(DateTimeOffset madeAt) => DaysUntilFull is { } days ? madeAt.AddDays(days) : null;
 
     /// <summary>"fuld om ca. 12 dage" (Danish), or the note.</summary>
     public string Describe() => DaysUntilFull switch
     {
         null => Note ?? "fyldes ikke op",
+        <= 0 => "fuld nu",
         < 1 => "fuld inden for et døgn",
         { } d => string.Create(CultureInfo.InvariantCulture, $"fuld om ca. {Math.Round(d):0} dage"),
     };
@@ -41,6 +51,9 @@ public sealed record DiskForecastSettings
 
     /// <summary>Growth slower than this (GB per day) is noise.</summary>
     public double MinimumGrowthGbPerDay { get; init; } = 0.01;
+
+    /// <summary>What one history point stands for: the repository averages the samples per clock hour, stamped at the hour's start.</summary>
+    public TimeSpan Resolution { get; init; } = TimeSpan.FromHours(1);
 }
 
 /// <summary>
@@ -59,7 +72,7 @@ public static class DiskForecaster
             .OrderBy(p => p.At).ToList();
         if (points.Count == 0)
         {
-            return new DiskForecast(volume, null, 0, 0, 0, 0, "ingen målinger endnu");
+            return new DiskForecast(volume, null, 0, 0, 0, 0, "ingen målinger endnu", EnoughHistory: false);
         }
 
         // A volume that was extended (or shrunk) starts a new trend.
@@ -70,12 +83,17 @@ public static class DiskForecaster
         if (points.Count < settings.MinimumSamples || span < settings.MinimumHistoryDays)
         {
             return new DiskForecast(volume, null, 0, points.Count, span, 0,
-                string.Create(CultureInfo.InvariantCulture, $"for lidt historik til en prognose ({span:0.#} af {settings.MinimumHistoryDays:0.#} dage)"));
+                string.Create(CultureInfo.InvariantCulture, $"for lidt historik til en prognose ({span:0.#} af {settings.MinimumHistoryDays:0.#} dage)"),
+                EnoughHistory: false);
         }
 
         var (slope, fit) = Regression(points);
         var growth = -slope;
         var free = points[^1].FreeGb;
+
+        // Counted from now, not from the last point, when that is older than its own resolution (the agent could not reach
+        // the server for a while). A point stands for the period that starts at its time (the repository's hourly averages).
+        var sinceLastSample = Math.Max(0, (now - points[^1].At - settings.Resolution).TotalDays);
         if (free <= 0)
         {
             return new DiskForecast(volume, 0, growth, points.Count, span, fit);
@@ -91,7 +109,7 @@ public static class DiskForecaster
             return new DiskForecast(volume, null, growth, points.Count, span, fit, "ingen tydelig tendens (pladsen svinger)");
         }
 
-        var days = free / growth;
+        var days = Math.Max(0, (free / growth) - sinceLastSample);
         return days > settings.MaximumDays
             ? new DiskForecast(volume, null, growth, points.Count, span, fit, "fuld om mere end et år")
             : new DiskForecast(volume, days, growth, points.Count, span, fit);

src/SQLInsight.Core/Servers/ServerEvents.cs

@@ -32,11 +32,18 @@ public enum ServerEventCategory
 /// <param name="Message">Can name users, computers and paths: sensitive.</param>
 public sealed record ServerEventEntry(string Log, string Source, long EventId, int Level, DateTimeOffset At, string? Message = null, long? RecordId = null);
 
+/// <summary>The events of one group within one clock hour: what the count of the last 24 hours is summed from.</summary>
+public sealed record ServerEventBucket(DateTimeOffset FirstAt, DateTimeOffset LastAt, int Count);
+
 /// <summary>The same event repeated, summed: one line per log, source and event id.</summary>
+/// <param name="Count">In <see cref="ServerEventSnapshot.Recent"/>: the events of the last 24 hours (to the hour), not since first seen.</param>
 public sealed record ServerEventGroup(string Log, string Source, long EventId, int Level, ServerEventCategory Category, int Count, DateTimeOffset FirstAt,
     DateTimeOffset LastAt, string? Sample)
 {
     public string Key => Log + "/" + Source + "/" + EventId.ToString(System.Globalization.CultureInfo.InvariantCulture);
+
+    /// <summary>The counts per clock hour (at most 25 in <see cref="ServerEventSnapshot.Recent"/>), so events that fall out of the window stop counting.</summary>
+    public IReadOnlyList<ServerEventBucket> Buckets { get; init; } = [];
 }
 
 public sealed record ServerEventSnapshot
@@ -178,36 +185,67 @@ public static class ServerEventAnalysis
             {
                 var newest = g.OrderByDescending(e => e.At).First();
                 return new ServerEventGroup(newest.Log, newest.Source, newest.EventId, g.Min(e => e.Level), Categorize(newest), g.Count(),
-                    g.Min(e => e.At), g.Max(e => e.At), Cut(g.OrderByDescending(e => e.At).Select(e => e.Message).FirstOrDefault(m => !string.IsNullOrWhiteSpace(m))));
+                    g.Min(e => e.At), g.Max(e => e.At), Cut(g.OrderByDescending(e => e.At).Select(e => e.Message).FirstOrDefault(m => !string.IsNullOrWhiteSpace(m))))
+                {
+                    Buckets = g.GroupBy(e => Hour(e.At)).Select(h => new ServerEventBucket(h.Min(e => e.At), h.Max(e => e.At), h.Count())).OrderBy(b => b.FirstAt).ToList(),
+                };
             })
             .OrderByDescending(g => g.Category).ThenByDescending(g => g.Count)
             .ToList();
         return (groups, ignored);
     }
 
-    /// <summary>The groups of the last day: this collection's plus the earlier ones still in the window, merged.</summary>
+    /// <summary>
+    /// The groups of the last day: this collection's plus the earlier ones, merged. Counts are kept per clock hour, and
+    /// only the hours with events in the last <see cref="RecentWindow"/> are summed, so an error that recurs every ten
+    /// hours never adds up to more than the few times it was actually seen in a day.
+    /// </summary>
     public static IReadOnlyList<ServerEventGroup> Recent(IReadOnlyList<ServerEventGroup> previous, IReadOnlyList<ServerEventGroup> current, DateTimeOffset now)
     {
         ArgumentNullException.ThrowIfNull(previous);
         ArgumentNullException.ThrowIfNull(current);
-        return previous.Where(g => now - g.LastAt < RecentWindow).Concat(current)
+        var since = now - RecentWindow;
+        return previous.Concat(current)
             .GroupBy(g => g.Key, StringComparer.OrdinalIgnoreCase)
-            .Select(g =>
+            .Select(ServerEventGroup? (g) =>
             {
+                var buckets = g.SelectMany(BucketsOf)
+                    .Where(b => b.LastAt > since)
+                    .GroupBy(b => Hour(b.FirstAt))
+                    .Select(h => new ServerEventBucket(h.Min(b => b.FirstAt), h.Max(b => b.LastAt), h.Sum(b => b.Count)))
+                    .OrderBy(b => b.FirstAt)
+                    .ToList();
+                if (buckets.Count == 0)
+                {
+                    return null;
+                }
+
                 var newest = g.OrderByDescending(x => x.LastAt).First();
                 return newest with
                 {
-                    Level = g.Min(x => x.Level),
-                    Count = g.Sum(x => x.Count),
-                    FirstAt = g.Min(x => x.FirstAt),
-                    LastAt = g.Max(x => x.LastAt),
+                    Level = g.Where(x => x.LastAt > since).Min(x => x.Level),
+                    Count = buckets.Sum(b => b.Count),
+                    FirstAt = buckets[0].FirstAt,
+                    LastAt = buckets.Max(b => b.LastAt),
                     Sample = g.OrderByDescending(x => x.LastAt).Select(x => x.Sample).FirstOrDefault(s => s is not null),
+                    Buckets = buckets,
                 };
             })
+            .OfType<ServerEventGroup>()
             .OrderByDescending(g => g.Category).ThenByDescending(g => g.Count)
             .ToList();
     }
 
+    /// <summary>A group saved before the counts were kept per hour has one bucket for all of it.</summary>
+    private static IReadOnlyList<ServerEventBucket> BucketsOf(ServerEventGroup group) =>
+        group.Buckets is { Count: > 0 } buckets ? buckets : [new ServerEventBucket(group.FirstAt, group.LastAt, group.Count)];
+
+    private static DateTimeOffset Hour(DateTimeOffset at)
+    {
+        var ticks = at.UtcTicks;
+        return new DateTimeOffset(ticks - (ticks % TimeSpan.TicksPerHour), TimeSpan.Zero);
+    }
+
     public static string Display(ServerEventCategory category) => category switch
     {
         ServerEventCategory.UnexpectedShutdown => "Uventet nedlukning",

src/SQLInsight.Core/Servers/ServerPerformance.cs

@@ -6,18 +6,27 @@ public static class ServerPerformanceMath
     /// <summary>How long the per-minute samples are kept in memory for the rules.</summary>
     public static readonly TimeSpan Window = TimeSpan.FromHours(1);
 
-    /// <summary>The interval's sample. CPU and pages/s need the previous raw sample; without it (or after a reboot) they are null.</summary>
-    public static ServerPerformanceSample Sample(ServerPerformanceRaw? previous, ServerPerformanceRaw current, ServerOsInfo os)
+    /// <summary>
+    /// The interval's sample. CPU and pages/s need the previous raw sample; without it (or after a reboot) they are null.
+    /// With <paramref name="interval"/> they are also null when the previous sample is more than three intervals old (the
+    /// agent could not reach the server for a while): the rate would be the average of the whole gap, stamped now.
+    /// </summary>
+    public static ServerPerformanceSample Sample(ServerPerformanceRaw? previous, ServerPerformanceRaw current, ServerOsInfo os, TimeSpan? interval = null)
     {
         ArgumentNullException.ThrowIfNull(current);
         ArgumentNullException.ThrowIfNull(os);
+        if (previous is not null && interval is { } expected && expected > TimeSpan.Zero && current.CapturedAt - previous.CapturedAt > 3 * expected)
+        {
+            previous = null;
+        }
+
         return new ServerPerformanceSample(
             current.CapturedAt,
             Cpu(previous, current),
             current.AvailableMb,
             os.TotalMemoryMb,
             PagesPerSecond(previous, current),
-            current.CommittedBytes is { } committed && current.CommitLimit is > 0 ? Math.Clamp(100 * committed / current.CommitLimit.Value, 0, 100) : null);
+            current.CommittedBytes is { } committed && current.CommitLimit is { } limit and > 0 ? Math.Clamp(100.0 * committed / limit, 0, 100) : null);
     }
 
     /// <summary>

src/SQLInsight.Core/Servers/ServerTasks.cs

@@ -1,9 +1,12 @@
 using System.Globalization;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.RegularExpressions;
 
 namespace SQLInsight.Core.Servers;
 
 /// <summary>Task Scheduler's result codes (LastTaskResult) in Danish, and which of them mean the task failed.</summary>
-public static class TaskResults
+public static partial class TaskResults
 {
     public const uint Success = 0;
     public const uint Ready = 0x41300;
@@ -48,4 +51,58 @@ public static class TaskResults
         ArgumentNullException.ThrowIfNull(path);
         return path.StartsWith(@"\Microsoft\", StringComparison.OrdinalIgnoreCase);
     }
+
+    /// <summary>
+    /// Tasks that programs create per user or to update themselves, in the root folder rather than under \Microsoft\:
+    /// they are not the server's own work, and their failures are rarely anyone's business. A name, or a prefix ending in *.
+    /// </summary>
+    public static readonly IReadOnlyList<string> DefaultIgnoredNames =
+    [
+        "OneDrive Standalone Update Task*",
+        "OneDrive Per-Machine Standalone Update Task*",
+        "OneDrive Reporting Task*",
+        "OneDrive Startup Task*",
+        "MicrosoftEdgeUpdateTask*",
+        "GoogleUpdateTask*",
+        "GoogleUpdaterTask*",
+        "User_Feed_Synchronization-*",
+    ];
+
+    /// <summary>
+    /// Whether a task belongs to one user (its name carries the user's SID, S-1-5-21-…) or is a program's own updater
+    /// (<see cref="DefaultIgnoredNames"/>). Such tasks are not watched unless asked for.
+    /// </summary>
+    public static bool IsUserOrUpdaterTask(string path)
+    {
+        ArgumentNullException.ThrowIfNull(path);
+        if (path.Contains("S-1-5-21-", StringComparison.OrdinalIgnoreCase))
+        {
+            return true;
+        }
+
+        var name = path[(path.LastIndexOf('\\') + 1)..];
+        return DefaultIgnoredNames.Any(p => p.EndsWith('*')
+            ? name.StartsWith(p[..^1], StringComparison.OrdinalIgnoreCase)
+            : name.Equals(p, StringComparison.OrdinalIgnoreCase));
+    }
+
+    /// <summary>Whether the task's path names a user by SID.</summary>
+    public static bool HasUserSid(string path)
+    {
+        ArgumentNullException.ThrowIfNull(path);
+        return Sid().IsMatch(path);
+    }
+
+    /// <summary>
+    /// The task's path for titles and subjects: a user's SID is replaced by "bruger-" and a short hash of it, which tells
+    /// two users' tasks apart without naming the user.
+    /// </summary>
+    public static string Display(string path)
+    {
+        ArgumentNullException.ThrowIfNull(path);
+        return Sid().Replace(path, m => "bruger-" + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(m.Value.ToUpperInvariant())))[..8].ToLowerInvariant());
+    }
+
+    [GeneratedRegex(@"S-1-5-21(-[0-9]+)+", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)]
+    private static partial Regex Sid();
 }

tests/SQLInsight.Core.Tests/WindowsServerTests.cs

@@ -90,6 +90,69 @@ public class DiskForecastTests
 
         Assert.Null(DiskForecaster.Forecast(@"D:\", history, Now).DaysUntilFull);
     }
+
+    [Fact]
+    public void Only_too_little_history_is_not_enough_history()
+    {
+        Assert.False(DiskForecaster.Forecast(@"D:\", [], Now).EnoughHistory);
+        Assert.False(DiskForecaster.Forecast(@"D:\", History(1, age => 20 + age), Now).EnoughHistory);
+        Assert.True(DiskForecaster.Forecast(@"D:\", History(14, _ => 40), Now).EnoughHistory); // flat: judged, and not filling up
+        Assert.True(DiskForecaster.Forecast(@"D:\", History(14, age => 20 + age), Now).EnoughHistory);
+    }
+
+    [Fact]
+    public void Days_are_counted_from_now_not_from_the_last_sample()
+    {
+        // 1 GB per day used; the last sample is two days old and had 20 GB free: full in 18 days from now.
+        var history = History(14, age => 20 + age).Select(p => p with { At = p.At.AddDays(-2) }).ToList();
+
+        var forecast = DiskForecaster.Forecast(@"D:\", history, Now);
+
+        Assert.Equal(18, forecast.DaysUntilFull!.Value, 1);
+        Assert.Equal(Now.AddDays(18).Date, forecast.FullAt(Now)!.Value.Date);
+    }
+
+    [Fact]
+    public void Irregular_sampling_gives_the_same_trend()
+    {
+        // Samples at uneven times (the agent was down now and then), 2 GB per day used, 30 GB free now.
+        var random = new Random(3);
+        var at = Now.AddDays(-14);
+        var history = new List<DiskHistoryPoint>();
+        while (at <= Now)
+        {
+            history.Add(new DiskHistoryPoint(at, 100, 30 + (2 * (Now - at).TotalDays)));
+            at = at.AddMinutes(15 + random.Next(0, 600));
+        }
+
+        history.Add(new DiskHistoryPoint(Now, 100, 30));
+        var forecast = DiskForecaster.Forecast(@"D:\", history, Now);
+
+        Assert.Equal(2, forecast.GrowthGbPerDay, 6);
+        Assert.Equal(15, forecast.DaysUntilFull!.Value, 3);
+    }
+
+    [Fact]
+    public void Identical_timestamps_have_no_slope()
+    {
+        var points = Enumerable.Range(0, 20).Select(i => new DiskHistoryPoint(Now, 100, 50 - i)).ToList();
+
+        Assert.Equal((0, 0), DiskForecaster.Regression(points));
+        Assert.Null(DiskForecaster.Forecast(@"D:\", points, Now, new DiskForecastSettings { MinimumHistoryDays = 0, MinimumSamples = 2 }).DaysUntilFull);
+    }
+
+    [Fact]
+    public void A_shrunk_volume_starts_a_new_trend()
+    {
+        // Was 200 GB and flat; shrunk to 100 GB a day ago and filling fast since: too little history for the new size.
+        var history = History(14, age => age > 1 ? 150 : 20 + (10 * age)).Select(p => (Now - p.At).TotalDays > 1 ? p with { SizeGb = 200 } : p).ToList();
+
+        var forecast = DiskForecaster.Forecast(@"D:\", history, Now);
+
+        Assert.Null(forecast.DaysUntilFull);
+        Assert.False(forecast.EnoughHistory);
+        Assert.InRange(forecast.HistoryDays, 0.9, 1.1);
+    }
 }
 
 public class ServerPerformanceTests
@@ -126,6 +189,29 @@ public class ServerPerformanceTests
         Assert.Equal(12.5, sample.AvailablePercent!.Value, 6);
         Assert.Equal(90, sample.CommitPercent!.Value, 6);
         Assert.Null(sample.CpuPercent);
+
+        // Not rounded down to whole percent: 89.9 % stays under a 90 % limit only because it is under it.
+        var big = ServerPerformanceMath.Sample(null, raw with { CommittedBytes = 8_999_000_000, CommitLimit = 10_000_000_000 }, new ServerOsInfo());
+        Assert.Equal(89.99, big.CommitPercent!.Value, 6);
+    }
+
+    [Fact]
+    public void A_sample_after_a_long_gap_has_no_rates()
+    {
+        var first = new ServerPerformanceRaw
+        {
+            CapturedAt = T0, ProcessorIdle100Ns = 1_000_000_000, ProcessorTimestamp100Ns = 5_000_000_000, Pages = 1000, MemoryPerfTime = 10_000_000,
+            MemoryPerfFrequency = 10_000_000,
+        };
+        var minute = first with { CapturedAt = T0.AddMinutes(1), ProcessorIdle100Ns = 1_150_000_000, ProcessorTimestamp100Ns = 5_600_000_000, Pages = 61_000, MemoryPerfTime = 610_000_000 };
+        var hourLater = minute with { CapturedAt = T0.AddMinutes(61) };
+        var os = new ServerOsInfo();
+
+        Assert.Equal(75, ServerPerformanceMath.Sample(first, minute, os, TimeSpan.FromMinutes(1)).CpuPercent!.Value, 6);
+        var afterGap = ServerPerformanceMath.Sample(first, hourLater, os, TimeSpan.FromMinutes(1));
+        Assert.Null(afterGap.CpuPercent);
+        Assert.Null(afterGap.PagesPerSecond);
+        Assert.NotNull(ServerPerformanceMath.Sample(first, hourLater, os, TimeSpan.FromMinutes(30)).CpuPercent); // within three intervals
     }
 
     [Fact]
@@ -235,6 +321,38 @@ public class ServerEventTests
         Assert.Equal(Now.AddMinutes(-2), disk.LastAt);
     }
 
+    [Fact]
+    public void The_24_hour_count_only_counts_the_last_24_hours()
+    {
+        // The same error every ten hours for three days, collected every five minutes as the agent does.
+        IReadOnlyList<ServerEventGroup> recent = [];
+        var start = Now.AddDays(-3);
+        for (var at = start; at <= Now; at = at.AddMinutes(5))
+        {
+            var entries = (at - start).TotalMinutes % 600 == 0 ? new[] { new ServerEventEntry("Application", "Some-App", 9, 2, at, "tekst") } : [];
+            var (groups, _) = ServerEventAnalysis.Group(entries, []);
+            recent = ServerEventAnalysis.Recent(recent, groups, at);
+        }
+
+        var group = Assert.Single(recent);
+        Assert.Equal(3, group.Count); // not the 8 seen since the first one
+        Assert.True(Now - group.FirstAt <= ServerEventAnalysis.RecentWindow);
+        Assert.InRange(group.Buckets.Count, 1, 25);
+    }
+
+    [Fact]
+    public void Groups_saved_without_hourly_counts_are_still_merged()
+    {
+        var old = new ServerEventGroup("System", "disk", 153, 2, ServerEventCategory.Disk, 4, Now.AddHours(-5), Now.AddHours(-3), "tekst");
+        var (current, _) = ServerEventAnalysis.Group([Entry("disk", 153, minutesAgo: 2)], []);
+
+        var disk = Assert.Single(ServerEventAnalysis.Recent([old], current, Now));
+
+        Assert.Equal(5, disk.Count);
+        Assert.Equal(Now.AddHours(-5), disk.FirstAt);
+        Assert.Empty(ServerEventAnalysis.Recent([old], [], Now.AddHours(22)));
+    }
+
     [Fact]
     public void Samples_are_cut()
     {