sqlinsight-pro/tests/SQLInsight.Collectors.Tests/SqlScripts/SqlScriptTests.cs

using System.Reflection;
using System.Text.RegularExpressions;
using SQLInsight.Rules;
using SQLInsight.Rules.Lifecycle;

namespace SQLInsight.Collectors.Tests.SqlScripts;

/// <summary>
/// Keeps the folder "SQL Scripts" (one reviewable script per rule, for DBAs) in step with the code: a script for every
/// rule and no other, the collectors' SQL in them word for word, and nothing in them that writes.
/// </summary>
public sealed partial class SqlScriptTests
{
    private static readonly string Folder = Path.Combine(RepositoryRoot(), "SQL Scripts");

    [Fact]
    public void Every_registered_rule_has_a_script_and_every_script_a_rule()
    {
        var ids = BuiltInRules.Create().Select(r => r.Id)
            .Concat(RulePack.Embedded().CommunityRules.Select(r => r.Id))
            .ToHashSet(StringComparer.Ordinal);
        var files = Directory.EnumerateFiles(Folder, "*.sql", SearchOption.TopDirectoryOnly)
            .Select(Path.GetFileNameWithoutExtension)
            .OfType<string>()
            .ToHashSet(StringComparer.Ordinal);

        var missing = ids.Except(files).Order(StringComparer.Ordinal).ToList();
        var unknown = files.Except(ids).Order(StringComparer.Ordinal).ToList();
        Assert.True(missing.Count == 0, "Regler uden script i \"SQL Scripts\" (tilføj <id>.sql, se CONTRIBUTING.md): " + string.Join(", ", missing));
        Assert.True(unknown.Count == 0, "Scripts i \"SQL Scripts\" for regel-id'er, der ikke findes: " + string.Join(", ", unknown));
    }

    [Fact]
    public void Rule_scripts_have_the_review_header()
    {
        string[] labels =
        [
            "-- Regel-id:", "-- Titel:", "-- Alvor:", "-- Versioner:", "-- Hvad tjekkes, og hvorfor:", "-- Vurdering", "-- Grænseværdier",
            "-- Rettigheder", "-- Data kommer fra", "-- Regelkode:", "-- Read-only:",
        ];
        var problems = new List<string>();
        foreach (var file in RuleScripts())
        {
            var text = Read(file);
            var id = Path.GetFileNameWithoutExtension(file);
            problems.AddRange(labels.Where(l => !text.Contains(l, StringComparison.Ordinal)).Select(l => $"{id}: mangler '{l}'"));
            if (!Regex.IsMatch(text, @"^-- Regel-id:\s+" + Regex.Escape(id) + @"\s*$", RegexOptions.Multiline))
            {
                problems.Add($"{id}: Regel-id i headeren er ikke filens navn");
            }
        }

        Assert.True(problems.Count == 0, string.Join("; ", problems));
    }

    [Fact]
    public void Scripts_contain_the_current_sql_from_the_code()
    {
        var problems = new List<string>();
        foreach (var file in AllScripts())
        {
            var name = Path.GetRelativePath(Folder, file);
            if (Environment.GetEnvironmentVariable("SQLINSIGHT_UPDATE_SQL_SCRIPTS") == "1")
            {
                Update(file);
            }

            var blocks = Blocks(Read(file)).ToList();
            if (blocks.Count == 0)
            {
                problems.Add($"{name}: ingen '-- Kilde:'-blokke");
            }

            foreach (var (key, body) in blocks)
            {
                if (!SqlScriptSources.ByKey.TryGetValue(key, out var source))
                {
                    problems.Add($"{name}: ukendt kilde '{key}'");
                }
                else if (!SqlScriptSources.Pattern(source.Sql).IsMatch(body))
                {
                    problems.Add($"{name}: '{key}' er ikke længere den SQL, koden sender ({source.File} : {source.Member})");
                }
            }
        }

        Assert.True(problems.Count == 0, "Opdatér \"SQL Scripts\": " + string.Join("; ", problems));
    }

    [Fact]
    public void Every_collector_query_is_registered_and_in_a_collector_script()
    {
        var problems = new List<string>();
        var registered = SqlScriptSources.All.Select(s => s.Sql).ToList();
        foreach (var type in typeof(ICollector).Assembly.GetTypes())
        {
            const BindingFlags Static = BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly;

            // SQL constants: found word for word in a registered text ({placeholders} are filled in by the code).
            foreach (var field in type.GetFields(Static).Where(f => f.IsLiteral && f.FieldType == typeof(string) && f.Name.Contains("Sql", StringComparison.Ordinal)))
            {
                var text = SqlScriptSources.Normalize((string)field.GetRawConstantValue()!);
                var pattern = new Regex(string.Join("(?:.+?)", Regex.Split(text, @"\{[A-Za-z]+\}").Select(Regex.Escape)), RegexOptions.Singleline);
                if (!registered.Any(pattern.IsMatch))
                {
                    problems.Add($"{type.Name}.{field.Name} er ikke registreret i SqlScriptSources");
                }
            }

            // SQL builders: at least one registered rendering.
            foreach (var method in type.GetMethods(Static).Where(m => m.ReturnType == typeof(string) && !m.Name.Contains('<', StringComparison.Ordinal)
                         && (m.Name.StartsWith("Build", StringComparison.Ordinal) || m.Name.EndsWith("Sql", StringComparison.Ordinal) || m.Name == "SqlFor")))
            {
                if (!SqlScriptSources.All.Any(s => s.Member.StartsWith(type.Name + ".", StringComparison.Ordinal) && s.Member.Contains(method.Name + "(", StringComparison.Ordinal)))
                {
                    problems.Add($"{type.Name}.{method.Name}() er ikke registreret i SqlScriptSources");
                }
            }
        }

        var inCollectorScripts = Directory.EnumerateFiles(Path.Combine(Folder, "_collectors"), "*.sql")
            .SelectMany(f => Blocks(Read(f)).Select(b => b.Key))
            .ToHashSet(StringComparer.Ordinal);
        problems.AddRange(SqlScriptSources.All.Where(s => !inCollectorScripts.Contains(s.Key)).Select(s => $"'{s.Key}' står ikke i _collectors/{s.Collector}.sql"));

        Assert.True(problems.Count == 0, string.Join("; ", problems));
    }

    [Fact]
    public void Community_rule_scripts_contain_the_rule_query_verbatim()
    {
        var problems = RulePack.Embedded().CommunityRules
            .Where(r => !File.Exists(Path.Combine(Folder, r.Id + ".sql"))
                        || !Read(Path.Combine(Folder, r.Id + ".sql")).Contains(SqlScriptSources.Normalize(r.Query), StringComparison.Ordinal))
            .Select(r => r.Id)
            .ToList();

        Assert.True(problems.Count == 0, "Community-reglens query står ikke ordret i SQL Scripts/<id>.sql: " + string.Join(", ", problems));
    }

    [Fact]
    public void Scripts_are_read_only()
    {
        var problems = new List<string>();
        foreach (var file in AllScripts())
        {
            problems.AddRange(WritesIn(Read(file)).Select(w => $"{Path.GetRelativePath(Folder, file)}: {w}").Distinct());
        }

        Assert.True(problems.Count == 0, "Skrivende sætninger i \"SQL Scripts\": " + string.Join("; ", problems));
    }

    [Theory]
    [InlineData("SELECT name INTO dbo.x FROM sys.objects", true)]
    [InlineData("SELECT name INTO #x FROM sys.objects", true)]
    [InlineData("UPDATE dbo.t SET a = 1", true)]
    [InlineData("EXEC sys.sp_configure N'x', 1", true)]
    [InlineData("SELECT is_auto_update_stats_on, create_date FROM sys.databases -- DROP TABLE x", false)]
    [InlineData("EXEC master.dbo.xp_readerrorlog 0, 1, NULL, NULL, @fra, @til", false)]
    [InlineData("DBCC TRACESTATUS(-1) WITH NO_INFOMSGS", false)]
    [InlineData("SELECT N'DELETE' AS [UPDATE]", false)]
    [InlineData("EXEC [AXDB].sys.sp_executesql N'SET LOCK_TIMEOUT 2000; SELECT TOP (10) a FROM [dbo].[BATCH] WHERE b = N''x''';", false)]
    [InlineData("EXEC [AXDB].sys.sp_executesql N'DELETE FROM [dbo].[BATCH]';", true)]
    [InlineData("EXEC [AXDB].sys.sp_executesql N'SELECT N''x'' AS a INTO #t';", true)]
    [InlineData("EXEC [AXDB].sys.sp_executesql @sql;", true)]
    public void The_read_only_check_itself_recognises_writes(string sql, bool writes) => Assert.Equal(writes, WritesIn(sql).Any());

    /// <summary>
    /// The writing keywords in <paramref name="sql"/>, outside comments and strings. The batch of an
    /// <c>EXEC [db].sys.sp_executesql N'…'</c> (how the Dynamics AX reads run in the AX database) is checked the same way:
    /// the call itself only runs that batch, so it is read-only exactly when the batch is.
    /// </summary>
    private static IEnumerable<string> WritesIn(string sql)
    {
        var batches = new List<string>();
        sql = DynamicSql().Replace(sql, m =>
        {
            batches.Add(m.Groups["sql"].Value.Replace("''", "'", StringComparison.Ordinal));
            return " ";
        });
        var code = AllowedStatements().Replace(CommentsAndStrings().Replace(sql, " "), " ");
        return Writes().Matches(code).Select(m => m.Value.ToUpperInvariant()).Concat(batches.SelectMany(WritesIn)).ToList();
    }

    /// <summary>
    /// SQLINSIGHT_UPDATE_SQL_SCRIPTS=1: a block whose SQL is no longer what the code sends gets the current SQL (its
    /// comment lines are kept). A block with run-time values (⟪tokens⟫) is only filled while it has no SQL yet.
    /// </summary>
    private static void Update(string file)
    {
        var lines = Read(file).Split('\n').ToList();
        var output = new List<string>();
        for (var i = 0; i < lines.Count; i++)
        {
            output.Add(lines[i]);
            if (!lines[i].StartsWith("-- Kilde: ", StringComparison.Ordinal)
                || !SqlScriptSources.ByKey.TryGetValue(lines[i]["-- Kilde: ".Length..].Trim(), out var source))
            {
                continue;
            }

            var end = lines.FindIndex(i + 1, l => l.Trim() == "GO");
            if (end < 0 || SqlScriptSources.Pattern(source.Sql).IsMatch(string.Join("\n", lines[(i + 1)..end])))
            {
                continue;
            }

            // A block with run-time values is only filled when it is new (no SQL yet); the ⟪tokens⟫ must then be replaced by hand.
            var comments = lines[(i + 1)..end].TakeWhile(l => l.StartsWith("--", StringComparison.Ordinal)).ToList();
            if (source.Sql.Contains('⟪', StringComparison.Ordinal) && comments.Count < end - i - 1)
            {
                continue;
            }

            output.AddRange(comments);
            output.AddRange(source.Sql.TrimEnd('\n').Split('\n'));
            i = end - 1;
        }

        File.WriteAllText(file, string.Join("\r\n", output), new System.Text.UTF8Encoding(false));
    }

    /// <summary>The <c>-- Kilde: key</c> blocks: everything after the line up to the next GO.</summary>
    private static IEnumerable<(string Key, string Body)> Blocks(string text)
    {
        string? key = null;
        var body = new List<string>();
        foreach (var line in text.Split('\n'))
        {
            if (line.StartsWith("-- Kilde: ", StringComparison.Ordinal))
            {
                key = line["-- Kilde: ".Length..].Trim();
                body.Clear();
            }
            else if (key is not null && line.Trim() == "GO")
            {
                yield return (key, string.Join("\n", body));
                key = null;
            }
            else if (key is not null)
            {
                body.Add(line);
            }
        }
    }

    private static IEnumerable<string> RuleScripts() => Directory.EnumerateFiles(Folder, "*.sql", SearchOption.TopDirectoryOnly);

    private static IEnumerable<string> AllScripts() => Directory.EnumerateFiles(Folder, "*.sql", SearchOption.AllDirectories);

    private static string Read(string file) => SqlScriptSources.Normalize(File.ReadAllText(file));

    private static string RepositoryRoot()
    {
        var root = AppContext.BaseDirectory;
        while (!File.Exists(Path.Combine(root, "SQLInsight.slnx")))
        {
            root = Path.GetDirectoryName(root) ?? throw new InvalidOperationException("SQLInsight.slnx ikke fundet.");
        }

        return root;
    }

    [GeneratedRegex(@"--[^\n]*|/\*.*?\*/|N?'(?:[^']|'')*'|\[[^\]]*\]", RegexOptions.Singleline)]
    private static partial Regex CommentsAndStrings();

    /// <summary>Read-only statements that use otherwise writing keywords.</summary>
    [GeneratedRegex(@"\bEXEC\s+master\.dbo\.xp_(readerrorlog|msver)\b|\bDBCC\s+TRACESTATUS\b|\bDBCC\s+SQLPERF\s*\(\s*WAITSTATS\s*\)", RegexOptions.IgnoreCase)]
    private static partial Regex AllowedStatements();

    [GeneratedRegex(@"\b(INSERT|UPDATE|DELETE|MERGE|DROP|ALTER|CREATE|TRUNCATE|GRANT|REVOKE|DENY|BACKUP|RESTORE|RECONFIGURE|KILL|SHUTDOWN|BULK|OPENROWSET|OPENQUERY|OPENDATASOURCE|WAITFOR|INTO|EXEC|EXECUTE|DBCC|SETUSER)\b", RegexOptions.IgnoreCase)]
    private static partial Regex Writes();

    /// <summary>A call of sp_executesql in a named database with a literal batch (not a variable, which cannot be checked).</summary>
    [GeneratedRegex(@"\bEXEC\s+\[[^\]]+\]\.sys\.sp_executesql\s+N'(?<sql>(?:[^']|'')*)'", RegexOptions.IgnoreCase)]
    private static partial Regex DynamicSql();
}