diff --git a/src/SQLInsight.Core/Servers/DiskForecast.cs b/src/SQLInsight.Core/Servers/DiskForecast.cs index f6fdc27..cd9c7f1 100644 --- a/src/SQLInsight.Core/Servers/DiskForecast.cs +++ b/src/SQLInsight.Core/Servers/DiskForecast.cs @@ -6,18 +6,28 @@ namespace SQLInsight.Core.Servers; public sealed record DiskHistoryPoint(DateTimeOffset At, double SizeGb, double FreeGb); /// When a volume will be full if it keeps filling at the rate of the last weeks. -/// Null when the volume is not filling up (or the trend is too unclear to say). +/// +/// Counted from when the forecast was made (the disk snapshot's CapturedAt, which takes). Null when +/// the volume is not filling up, the trend is too unclear to say, or there is too little history. +/// /// How fast the used space grows (positive: filling up). /// R² of the trend line: 1 is a straight line, near 0 no trend. /// Why there is no forecast (Danish), when there is none. -public sealed record DiskForecast(string Volume, double? DaysUntilFull, double GrowthGbPerDay, int Samples, double HistoryDays, double Fit, string? Note = null) +/// +/// Whether there was enough history to judge the trend at all. False: the volume could not be judged yet, so a rule +/// must say "ikke målbar" rather than "fyldes ikke op". +/// +public sealed record DiskForecast(string Volume, double? DaysUntilFull, double GrowthGbPerDay, int Samples, double HistoryDays, double Fit, string? Note = null, + bool EnoughHistory = true) { - public DateTimeOffset? FullAt(DateTimeOffset now) => DaysUntilFull is { } days ? now.AddDays(days) : null; + /// When the forecast was made (the disk snapshot's CapturedAt). + public DateTimeOffset? FullAt(DateTimeOffset madeAt) => DaysUntilFull is { } days ? madeAt.AddDays(days) : null; /// "fuld om ca. 12 dage" (Danish), or the note. public string Describe() => DaysUntilFull switch { null => Note ?? "fyldes ikke op", + <= 0 => "fuld nu", < 1 => "fuld inden for et døgn", { } d => string.Create(CultureInfo.InvariantCulture, $"fuld om ca. {Math.Round(d):0} dage"), }; @@ -41,6 +51,9 @@ public sealed record DiskForecastSettings /// Growth slower than this (GB per day) is noise. public double MinimumGrowthGbPerDay { get; init; } = 0.01; + + /// What one history point stands for: the repository averages the samples per clock hour, stamped at the hour's start. + public TimeSpan Resolution { get; init; } = TimeSpan.FromHours(1); } /// @@ -59,7 +72,7 @@ public static class DiskForecaster .OrderBy(p => p.At).ToList(); if (points.Count == 0) { - return new DiskForecast(volume, null, 0, 0, 0, 0, "ingen målinger endnu"); + return new DiskForecast(volume, null, 0, 0, 0, 0, "ingen målinger endnu", EnoughHistory: false); } // A volume that was extended (or shrunk) starts a new trend. @@ -70,12 +83,17 @@ public static class DiskForecaster if (points.Count < settings.MinimumSamples || span < settings.MinimumHistoryDays) { return new DiskForecast(volume, null, 0, points.Count, span, 0, - string.Create(CultureInfo.InvariantCulture, $"for lidt historik til en prognose ({span:0.#} af {settings.MinimumHistoryDays:0.#} dage)")); + string.Create(CultureInfo.InvariantCulture, $"for lidt historik til en prognose ({span:0.#} af {settings.MinimumHistoryDays:0.#} dage)"), + EnoughHistory: false); } var (slope, fit) = Regression(points); var growth = -slope; var free = points[^1].FreeGb; + + // Counted from now, not from the last point, when that is older than its own resolution (the agent could not reach + // the server for a while). A point stands for the period that starts at its time (the repository's hourly averages). + var sinceLastSample = Math.Max(0, (now - points[^1].At - settings.Resolution).TotalDays); if (free <= 0) { return new DiskForecast(volume, 0, growth, points.Count, span, fit); @@ -91,7 +109,7 @@ public static class DiskForecaster return new DiskForecast(volume, null, growth, points.Count, span, fit, "ingen tydelig tendens (pladsen svinger)"); } - var days = free / growth; + var days = Math.Max(0, (free / growth) - sinceLastSample); return days > settings.MaximumDays ? new DiskForecast(volume, null, growth, points.Count, span, fit, "fuld om mere end et år") : new DiskForecast(volume, days, growth, points.Count, span, fit); diff --git a/src/SQLInsight.Core/Servers/ServerEvents.cs b/src/SQLInsight.Core/Servers/ServerEvents.cs index 26aabab..be31c36 100644 --- a/src/SQLInsight.Core/Servers/ServerEvents.cs +++ b/src/SQLInsight.Core/Servers/ServerEvents.cs @@ -32,11 +32,18 @@ public enum ServerEventCategory /// Can name users, computers and paths: sensitive. public sealed record ServerEventEntry(string Log, string Source, long EventId, int Level, DateTimeOffset At, string? Message = null, long? RecordId = null); +/// The events of one group within one clock hour: what the count of the last 24 hours is summed from. +public sealed record ServerEventBucket(DateTimeOffset FirstAt, DateTimeOffset LastAt, int Count); + /// The same event repeated, summed: one line per log, source and event id. +/// In : the events of the last 24 hours (to the hour), not since first seen. public sealed record ServerEventGroup(string Log, string Source, long EventId, int Level, ServerEventCategory Category, int Count, DateTimeOffset FirstAt, DateTimeOffset LastAt, string? Sample) { public string Key => Log + "/" + Source + "/" + EventId.ToString(System.Globalization.CultureInfo.InvariantCulture); + + /// The counts per clock hour (at most 25 in ), so events that fall out of the window stop counting. + public IReadOnlyList Buckets { get; init; } = []; } public sealed record ServerEventSnapshot @@ -178,36 +185,67 @@ public static class ServerEventAnalysis { var newest = g.OrderByDescending(e => e.At).First(); return new ServerEventGroup(newest.Log, newest.Source, newest.EventId, g.Min(e => e.Level), Categorize(newest), g.Count(), - g.Min(e => e.At), g.Max(e => e.At), Cut(g.OrderByDescending(e => e.At).Select(e => e.Message).FirstOrDefault(m => !string.IsNullOrWhiteSpace(m)))); + g.Min(e => e.At), g.Max(e => e.At), Cut(g.OrderByDescending(e => e.At).Select(e => e.Message).FirstOrDefault(m => !string.IsNullOrWhiteSpace(m)))) + { + Buckets = g.GroupBy(e => Hour(e.At)).Select(h => new ServerEventBucket(h.Min(e => e.At), h.Max(e => e.At), h.Count())).OrderBy(b => b.FirstAt).ToList(), + }; }) .OrderByDescending(g => g.Category).ThenByDescending(g => g.Count) .ToList(); return (groups, ignored); } - /// The groups of the last day: this collection's plus the earlier ones still in the window, merged. + /// + /// The groups of the last day: this collection's plus the earlier ones, merged. Counts are kept per clock hour, and + /// only the hours with events in the last are summed, so an error that recurs every ten + /// hours never adds up to more than the few times it was actually seen in a day. + /// public static IReadOnlyList Recent(IReadOnlyList previous, IReadOnlyList current, DateTimeOffset now) { ArgumentNullException.ThrowIfNull(previous); ArgumentNullException.ThrowIfNull(current); - return previous.Where(g => now - g.LastAt < RecentWindow).Concat(current) + var since = now - RecentWindow; + return previous.Concat(current) .GroupBy(g => g.Key, StringComparer.OrdinalIgnoreCase) - .Select(g => + .Select(ServerEventGroup? (g) => { + var buckets = g.SelectMany(BucketsOf) + .Where(b => b.LastAt > since) + .GroupBy(b => Hour(b.FirstAt)) + .Select(h => new ServerEventBucket(h.Min(b => b.FirstAt), h.Max(b => b.LastAt), h.Sum(b => b.Count))) + .OrderBy(b => b.FirstAt) + .ToList(); + if (buckets.Count == 0) + { + return null; + } + var newest = g.OrderByDescending(x => x.LastAt).First(); return newest with { - Level = g.Min(x => x.Level), - Count = g.Sum(x => x.Count), - FirstAt = g.Min(x => x.FirstAt), - LastAt = g.Max(x => x.LastAt), + Level = g.Where(x => x.LastAt > since).Min(x => x.Level), + Count = buckets.Sum(b => b.Count), + FirstAt = buckets[0].FirstAt, + LastAt = buckets.Max(b => b.LastAt), Sample = g.OrderByDescending(x => x.LastAt).Select(x => x.Sample).FirstOrDefault(s => s is not null), + Buckets = buckets, }; }) + .OfType() .OrderByDescending(g => g.Category).ThenByDescending(g => g.Count) .ToList(); } + /// A group saved before the counts were kept per hour has one bucket for all of it. + private static IReadOnlyList BucketsOf(ServerEventGroup group) => + group.Buckets is { Count: > 0 } buckets ? buckets : [new ServerEventBucket(group.FirstAt, group.LastAt, group.Count)]; + + private static DateTimeOffset Hour(DateTimeOffset at) + { + var ticks = at.UtcTicks; + return new DateTimeOffset(ticks - (ticks % TimeSpan.TicksPerHour), TimeSpan.Zero); + } + public static string Display(ServerEventCategory category) => category switch { ServerEventCategory.UnexpectedShutdown => "Uventet nedlukning", diff --git a/src/SQLInsight.Core/Servers/ServerPerformance.cs b/src/SQLInsight.Core/Servers/ServerPerformance.cs index 6e8fc51..2d76881 100644 --- a/src/SQLInsight.Core/Servers/ServerPerformance.cs +++ b/src/SQLInsight.Core/Servers/ServerPerformance.cs @@ -6,18 +6,27 @@ public static class ServerPerformanceMath /// How long the per-minute samples are kept in memory for the rules. public static readonly TimeSpan Window = TimeSpan.FromHours(1); - /// The interval's sample. CPU and pages/s need the previous raw sample; without it (or after a reboot) they are null. - public static ServerPerformanceSample Sample(ServerPerformanceRaw? previous, ServerPerformanceRaw current, ServerOsInfo os) + /// + /// The interval's sample. CPU and pages/s need the previous raw sample; without it (or after a reboot) they are null. + /// With they are also null when the previous sample is more than three intervals old (the + /// agent could not reach the server for a while): the rate would be the average of the whole gap, stamped now. + /// + public static ServerPerformanceSample Sample(ServerPerformanceRaw? previous, ServerPerformanceRaw current, ServerOsInfo os, TimeSpan? interval = null) { ArgumentNullException.ThrowIfNull(current); ArgumentNullException.ThrowIfNull(os); + if (previous is not null && interval is { } expected && expected > TimeSpan.Zero && current.CapturedAt - previous.CapturedAt > 3 * expected) + { + previous = null; + } + return new ServerPerformanceSample( current.CapturedAt, Cpu(previous, current), current.AvailableMb, os.TotalMemoryMb, PagesPerSecond(previous, current), - current.CommittedBytes is { } committed && current.CommitLimit is > 0 ? Math.Clamp(100 * committed / current.CommitLimit.Value, 0, 100) : null); + current.CommittedBytes is { } committed && current.CommitLimit is { } limit and > 0 ? Math.Clamp(100.0 * committed / limit, 0, 100) : null); } /// diff --git a/src/SQLInsight.Core/Servers/ServerTasks.cs b/src/SQLInsight.Core/Servers/ServerTasks.cs index 933c45a..298dc62 100644 --- a/src/SQLInsight.Core/Servers/ServerTasks.cs +++ b/src/SQLInsight.Core/Servers/ServerTasks.cs @@ -1,9 +1,12 @@ using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; namespace SQLInsight.Core.Servers; /// Task Scheduler's result codes (LastTaskResult) in Danish, and which of them mean the task failed. -public static class TaskResults +public static partial class TaskResults { public const uint Success = 0; public const uint Ready = 0x41300; @@ -48,4 +51,58 @@ public static class TaskResults ArgumentNullException.ThrowIfNull(path); return path.StartsWith(@"\Microsoft\", StringComparison.OrdinalIgnoreCase); } + + /// + /// Tasks that programs create per user or to update themselves, in the root folder rather than under \Microsoft\: + /// they are not the server's own work, and their failures are rarely anyone's business. A name, or a prefix ending in *. + /// + public static readonly IReadOnlyList DefaultIgnoredNames = + [ + "OneDrive Standalone Update Task*", + "OneDrive Per-Machine Standalone Update Task*", + "OneDrive Reporting Task*", + "OneDrive Startup Task*", + "MicrosoftEdgeUpdateTask*", + "GoogleUpdateTask*", + "GoogleUpdaterTask*", + "User_Feed_Synchronization-*", + ]; + + /// + /// Whether a task belongs to one user (its name carries the user's SID, S-1-5-21-…) or is a program's own updater + /// (). Such tasks are not watched unless asked for. + /// + public static bool IsUserOrUpdaterTask(string path) + { + ArgumentNullException.ThrowIfNull(path); + if (path.Contains("S-1-5-21-", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + var name = path[(path.LastIndexOf('\\') + 1)..]; + return DefaultIgnoredNames.Any(p => p.EndsWith('*') + ? name.StartsWith(p[..^1], StringComparison.OrdinalIgnoreCase) + : name.Equals(p, StringComparison.OrdinalIgnoreCase)); + } + + /// Whether the task's path names a user by SID. + public static bool HasUserSid(string path) + { + ArgumentNullException.ThrowIfNull(path); + return Sid().IsMatch(path); + } + + /// + /// The task's path for titles and subjects: a user's SID is replaced by "bruger-" and a short hash of it, which tells + /// two users' tasks apart without naming the user. + /// + public static string Display(string path) + { + ArgumentNullException.ThrowIfNull(path); + return Sid().Replace(path, m => "bruger-" + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(m.Value.ToUpperInvariant())))[..8].ToLowerInvariant()); + } + + [GeneratedRegex(@"S-1-5-21(-[0-9]+)+", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex Sid(); } diff --git a/tests/SQLInsight.Core.Tests/WindowsServerTests.cs b/tests/SQLInsight.Core.Tests/WindowsServerTests.cs index 4ea98d9..25e2b8d 100644 --- a/tests/SQLInsight.Core.Tests/WindowsServerTests.cs +++ b/tests/SQLInsight.Core.Tests/WindowsServerTests.cs @@ -90,6 +90,69 @@ public class DiskForecastTests Assert.Null(DiskForecaster.Forecast(@"D:\", history, Now).DaysUntilFull); } + + [Fact] + public void Only_too_little_history_is_not_enough_history() + { + Assert.False(DiskForecaster.Forecast(@"D:\", [], Now).EnoughHistory); + Assert.False(DiskForecaster.Forecast(@"D:\", History(1, age => 20 + age), Now).EnoughHistory); + Assert.True(DiskForecaster.Forecast(@"D:\", History(14, _ => 40), Now).EnoughHistory); // flat: judged, and not filling up + Assert.True(DiskForecaster.Forecast(@"D:\", History(14, age => 20 + age), Now).EnoughHistory); + } + + [Fact] + public void Days_are_counted_from_now_not_from_the_last_sample() + { + // 1 GB per day used; the last sample is two days old and had 20 GB free: full in 18 days from now. + var history = History(14, age => 20 + age).Select(p => p with { At = p.At.AddDays(-2) }).ToList(); + + var forecast = DiskForecaster.Forecast(@"D:\", history, Now); + + Assert.Equal(18, forecast.DaysUntilFull!.Value, 1); + Assert.Equal(Now.AddDays(18).Date, forecast.FullAt(Now)!.Value.Date); + } + + [Fact] + public void Irregular_sampling_gives_the_same_trend() + { + // Samples at uneven times (the agent was down now and then), 2 GB per day used, 30 GB free now. + var random = new Random(3); + var at = Now.AddDays(-14); + var history = new List(); + while (at <= Now) + { + history.Add(new DiskHistoryPoint(at, 100, 30 + (2 * (Now - at).TotalDays))); + at = at.AddMinutes(15 + random.Next(0, 600)); + } + + history.Add(new DiskHistoryPoint(Now, 100, 30)); + var forecast = DiskForecaster.Forecast(@"D:\", history, Now); + + Assert.Equal(2, forecast.GrowthGbPerDay, 6); + Assert.Equal(15, forecast.DaysUntilFull!.Value, 3); + } + + [Fact] + public void Identical_timestamps_have_no_slope() + { + var points = Enumerable.Range(0, 20).Select(i => new DiskHistoryPoint(Now, 100, 50 - i)).ToList(); + + Assert.Equal((0, 0), DiskForecaster.Regression(points)); + Assert.Null(DiskForecaster.Forecast(@"D:\", points, Now, new DiskForecastSettings { MinimumHistoryDays = 0, MinimumSamples = 2 }).DaysUntilFull); + } + + [Fact] + public void A_shrunk_volume_starts_a_new_trend() + { + // Was 200 GB and flat; shrunk to 100 GB a day ago and filling fast since: too little history for the new size. + var history = History(14, age => age > 1 ? 150 : 20 + (10 * age)).Select(p => (Now - p.At).TotalDays > 1 ? p with { SizeGb = 200 } : p).ToList(); + + var forecast = DiskForecaster.Forecast(@"D:\", history, Now); + + Assert.Null(forecast.DaysUntilFull); + Assert.False(forecast.EnoughHistory); + Assert.InRange(forecast.HistoryDays, 0.9, 1.1); + } } public class ServerPerformanceTests @@ -126,6 +189,29 @@ public class ServerPerformanceTests Assert.Equal(12.5, sample.AvailablePercent!.Value, 6); Assert.Equal(90, sample.CommitPercent!.Value, 6); Assert.Null(sample.CpuPercent); + + // Not rounded down to whole percent: 89.9 % stays under a 90 % limit only because it is under it. + var big = ServerPerformanceMath.Sample(null, raw with { CommittedBytes = 8_999_000_000, CommitLimit = 10_000_000_000 }, new ServerOsInfo()); + Assert.Equal(89.99, big.CommitPercent!.Value, 6); + } + + [Fact] + public void A_sample_after_a_long_gap_has_no_rates() + { + var first = new ServerPerformanceRaw + { + CapturedAt = T0, ProcessorIdle100Ns = 1_000_000_000, ProcessorTimestamp100Ns = 5_000_000_000, Pages = 1000, MemoryPerfTime = 10_000_000, + MemoryPerfFrequency = 10_000_000, + }; + var minute = first with { CapturedAt = T0.AddMinutes(1), ProcessorIdle100Ns = 1_150_000_000, ProcessorTimestamp100Ns = 5_600_000_000, Pages = 61_000, MemoryPerfTime = 610_000_000 }; + var hourLater = minute with { CapturedAt = T0.AddMinutes(61) }; + var os = new ServerOsInfo(); + + Assert.Equal(75, ServerPerformanceMath.Sample(first, minute, os, TimeSpan.FromMinutes(1)).CpuPercent!.Value, 6); + var afterGap = ServerPerformanceMath.Sample(first, hourLater, os, TimeSpan.FromMinutes(1)); + Assert.Null(afterGap.CpuPercent); + Assert.Null(afterGap.PagesPerSecond); + Assert.NotNull(ServerPerformanceMath.Sample(first, hourLater, os, TimeSpan.FromMinutes(30)).CpuPercent); // within three intervals } [Fact] @@ -235,6 +321,38 @@ public class ServerEventTests Assert.Equal(Now.AddMinutes(-2), disk.LastAt); } + [Fact] + public void The_24_hour_count_only_counts_the_last_24_hours() + { + // The same error every ten hours for three days, collected every five minutes as the agent does. + IReadOnlyList recent = []; + var start = Now.AddDays(-3); + for (var at = start; at <= Now; at = at.AddMinutes(5)) + { + var entries = (at - start).TotalMinutes % 600 == 0 ? new[] { new ServerEventEntry("Application", "Some-App", 9, 2, at, "tekst") } : []; + var (groups, _) = ServerEventAnalysis.Group(entries, []); + recent = ServerEventAnalysis.Recent(recent, groups, at); + } + + var group = Assert.Single(recent); + Assert.Equal(3, group.Count); // not the 8 seen since the first one + Assert.True(Now - group.FirstAt <= ServerEventAnalysis.RecentWindow); + Assert.InRange(group.Buckets.Count, 1, 25); + } + + [Fact] + public void Groups_saved_without_hourly_counts_are_still_merged() + { + var old = new ServerEventGroup("System", "disk", 153, 2, ServerEventCategory.Disk, 4, Now.AddHours(-5), Now.AddHours(-3), "tekst"); + var (current, _) = ServerEventAnalysis.Group([Entry("disk", 153, minutesAgo: 2)], []); + + var disk = Assert.Single(ServerEventAnalysis.Recent([old], current, Now)); + + Assert.Equal(5, disk.Count); + Assert.Equal(Now.AddHours(-5), disk.FirstAt); + Assert.Empty(ServerEventAnalysis.Recent([old], [], Now.AddHours(22))); + } + [Fact] public void Samples_are_cut() {