Add diagnostic severity configuration and suppression-aware assertions

README.md

@@ -192,7 +192,10 @@ Multiple source files can be added with separate `WithSource` calls.
 | `WithInterceptorNamespace(ns)` | Add an interceptors namespace feature flag to the compilation. |
 | `WithProperty(name, value)` | Add an arbitrary MSBuild-style build property (feature flag) to the compilation. The property is available through global and syntax-tree analyzer config options. |
 | `WithEditorConfigOption(name, value, filename)` | Add an EditorConfig option to syntax-tree analyzer config options. Omit `filename` to apply it to all source files; specify a filename to scope it to that source file. EditorConfig options are not available through global analyzer config options. |
-| `AssertDiagnostics(expectedDiagnosticIds)` | Run the analyzer and assert that the diagnostics match the `[|…|]`-marked locations. |
+| `WithDiagnosticSeverity(id, severity, filename)` | Configure the severity of a diagnostic, like `dotnet_diagnostic.<id>.severity` in an .editorconfig. Omit `filename` to apply it to all source files. File-specific severity overrides all-source severity, and tree-level severity overrides `WithGlobalDiagnosticSeverity`. |
+| `WithGlobalDiagnosticSeverity(id, severity)` | Configure the severity of a diagnostic globally for the compilation, like a global configuration file. Applies only when no tree-level severity is configured. |
+| `AssertDiagnostics(expectedDiagnosticIds)` | Run the analyzer and assert that the diagnostics match the `[|…|]`-marked locations and their effective severities. Diagnostics suppressed by a severity configuration are not considered reported. |
+| `AssertSuppressedDiagnostics(expectedDiagnosticIds)` | Assert that the analyzer reported the `[|…|]`-marked diagnostics but that they are suppressed, for example because a severity configuration (per-diagnostic, global, or bulk `dotnet_analyzer_diagnostic` EditorConfig severity) set their severity to none. Distinguishes "the analyzer did not report" from "reported but suppressed". Fails with an explanatory message when the analyzer is self-gating (opt-in) and does not report without a severity configuration. |
 
 ## Testing code fixes
 
@@ -256,7 +259,9 @@ To configure all code fix tests in a project, use `CodeFixTest.ConfigureAllCodeF
 | `WithInterceptorNamespace(ns)` | Add an interceptors namespace feature flag to the compilation. |
 | `WithProperty(name, value)` | Add an arbitrary build property to the compilation. The property is available through global and syntax-tree analyzer config options. |
 | `WithEditorConfigOption(name, value, filename)` | Add an EditorConfig option to syntax-tree analyzer config options. Omit `filename` to apply it to all source files; specify a filename to scope it to that source file. EditorConfig options are not available through global analyzer config options. |
-| `AssertCodeFixes()` | Apply code fixes iteratively and assert that the final source matches the expected output. |
+| `WithDiagnosticSeverity(id, severity, filename)` | Configure the severity of a diagnostic, like `dotnet_diagnostic.<id>.severity` in an .editorconfig. Omit `filename` to apply it to all source files. |
+| `WithGlobalDiagnosticSeverity(id, severity)` | Configure the severity of a diagnostic globally for the compilation, like a global configuration file. |
+| `AssertCodeFixes()` | Apply code fixes iteratively and assert that the final source matches the expected output. Suppressed diagnostics are never offered as fixes. |
 
 ## Testing source generators
 
@@ -312,6 +317,8 @@ To configure all source generator tests in a project, use `SourceGeneratorTest.C
 | `WithInterceptorNamespace(ns)` | Add an interceptors namespace feature flag to the compilation. |
 | `WithProperty(name, value)` | Add an arbitrary build property to the compilation. The property is available through global and syntax-tree analyzer config options. |
 | `WithEditorConfigOption(name, value, filename)` | Add an EditorConfig option to syntax-tree analyzer config options. Omit `filename` to apply it to all source files; specify a filename to scope it to that source file. EditorConfig options are not available through global analyzer config options. |
+| `WithDiagnosticSeverity(id, severity, filename)` | Configure the severity of a diagnostic, like `dotnet_diagnostic.<id>.severity` in an .editorconfig. Omit `filename` to apply it to all source files. Applies to generator and analyzer diagnostics. |
+| `WithGlobalDiagnosticSeverity(id, severity)` | Configure the severity of a diagnostic globally for the compilation, like a global configuration file. |
 | `Run()` | Run the source generator without running an approval test. |
 | `Approve(scrubber)` | Run the generator (if not already run) and perform an approval test on the generated output. |
 | `ShouldNotGenerateCode()` | Assert that the source generator produces no output for the given sources. |

src/FakeAnalyzers/DiagnosticDescriptors.cs

@@ -43,4 +43,12 @@ public static class DiagnosticDescriptors
         category: "Code",
         defaultSeverity: DiagnosticSeverity.Error,
         isEnabledByDefault: true);
+
+    public static readonly DiagnosticDescriptor GeneratorReported = new(
+        id: DiagnosticIds.GeneratorReported,
+        title: "Generator reported a diagnostic",
+        messageFormat: "The source generator reported a diagnostic for '{0}'",
+        category: "Code",
+        defaultSeverity: DiagnosticSeverity.Error,
+        isEnabledByDefault: true);
 }

src/FakeAnalyzers/DiagnosticEmittingSourceGenerator.cs

@@ -0,0 +1,29 @@
+namespace FakeAnalyzers;
+
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Text;
+
+[Generator(LanguageNames.CSharp)]
+public sealed class DiagnosticEmittingSourceGenerator : IIncrementalGenerator
+{
+    public void Initialize(IncrementalGeneratorInitializationContext context)
+    {
+        var syntaxTrees = context.CompilationProvider.SelectMany(static (compilation, _) => compilation.SyntaxTrees)
+            .WithTrackingName(TrackingNames.SyntaxTrees);
+
+        context.RegisterSourceOutput(syntaxTrees, static (productionContext, tree) =>
+        {
+            productionContext.ReportDiagnostic(Diagnostic.Create(
+                DiagnosticDescriptors.GeneratorReported,
+                Location.Create(tree, new TextSpan(0, 0)),
+                tree.FilePath));
+        });
+    }
+
+    internal static class TrackingNames
+    {
+        public const string SyntaxTrees = nameof(SyntaxTrees);
+
+        public static string[] All => [SyntaxTrees];
+    }
+}

src/FakeAnalyzers/DiagnosticIds.cs

@@ -11,4 +11,5 @@ public static class DiagnosticIds
     public const string IdentifierContainsFoo = "FAKE0003";
     public const string TestFlagEnabled = "FAKE0004";
     public const string EditorConfigOptionEnabled = "FAKE0005";
+    public const string GeneratorReported = "FAKE0006";
 }

src/Particular.AnalyzerTesting/AnalyzerConfigOptionsFactory.cs

@@ -1,5 +1,6 @@
 namespace Particular.AnalyzerTesting;
 
+using System;
 using System.Collections.Generic;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.Diagnostics;
@@ -10,27 +11,37 @@ public static AnalyzerConfigOptionsProvider CreateOptionsProvider(
         IReadOnlyDictionary<string, string> globalProperties,
         IReadOnlyDictionary<string, string>? sourceProperties = null,
         IReadOnlyDictionary<string, Dictionary<string, string>>? sourceFileProperties = null)
-        => new OptionsProvider(globalProperties, sourceProperties ?? new Dictionary<string, string>(), sourceFileProperties ?? new Dictionary<string, Dictionary<string, string>>());
+        => new OptionsProvider(globalProperties, sourceProperties ?? new Dictionary<string, string>(), sourceFileProperties ?? new Dictionary<string, Dictionary<string, string>>(), excludeBulkSeverityKeys: false);
 
-    public static AnalyzerOptions CreateAnalyzerOptions(
+    /// <summary>
+    /// Like <see cref="CreateOptionsProvider" />, but the syntax-tree options omit bulk analyzer
+    /// severity keys (<c>dotnet_analyzer_diagnostic.severity</c> and
+    /// <c>dotnet_analyzer_diagnostic.category-…severity</c>). Used for the neutral run that observes
+    /// what analyzers report before Roslyn's severity filtering kicks in.
+    /// </summary>
+    public static AnalyzerConfigOptionsProvider CreateNeutralOptionsProvider(
         IReadOnlyDictionary<string, string> globalProperties,
         IReadOnlyDictionary<string, string>? sourceProperties = null,
         IReadOnlyDictionary<string, Dictionary<string, string>>? sourceFileProperties = null)
-        => new([], CreateOptionsProvider(globalProperties, sourceProperties, sourceFileProperties));
+        => new OptionsProvider(globalProperties, sourceProperties ?? new Dictionary<string, string>(), sourceFileProperties ?? new Dictionary<string, Dictionary<string, string>>(), excludeBulkSeverityKeys: true);
 
     sealed class OptionsProvider(
         IReadOnlyDictionary<string, string> globalProperties,
         IReadOnlyDictionary<string, string> sourceProperties,
-        IReadOnlyDictionary<string, Dictionary<string, string>> sourceFileProperties) : AnalyzerConfigOptionsProvider
+        IReadOnlyDictionary<string, Dictionary<string, string>> sourceFileProperties,
+        bool excludeBulkSeverityKeys) : AnalyzerConfigOptionsProvider
     {
         public override AnalyzerConfigOptions GetOptions(SyntaxTree tree)
         {
             var properties = new Dictionary<string, string>(globalProperties);
             AddProperties(properties, sourceProperties);
 
-            if (sourceFileProperties.TryGetValue(tree.FilePath, out var fileProperties))
+            foreach (var (filename, fileProperties) in sourceFileProperties)
             {
-                AddProperties(properties, fileProperties);
+                if (FilenameComparer.Matches(filename, tree.FilePath))
+                {
+                    AddProperties(properties, fileProperties);
+                }
             }
 
             return new DictionaryAnalyzerConfigOptions(properties);
@@ -41,13 +52,22 @@ public override AnalyzerConfigOptions GetOptions(AdditionalText textFile)
 
         public override AnalyzerConfigOptions GlobalOptions { get; } = new DictionaryAnalyzerConfigOptions(globalProperties);
 
-        static void AddProperties(Dictionary<string, string> destination, IReadOnlyDictionary<string, string> source)
+        void AddProperties(Dictionary<string, string> destination, IReadOnlyDictionary<string, string> source)
         {
             foreach (var (key, value) in source)
             {
+                if (excludeBulkSeverityKeys && IsBulkSeverityKey(key))
+                {
+                    continue;
+                }
+
                 destination[key] = value;
             }
         }
+
+        static bool IsBulkSeverityKey(string key)
+            => key.StartsWith("dotnet_analyzer_diagnostic.", StringComparison.Ordinal) &&
+               key.EndsWith(".severity", StringComparison.Ordinal);
     }
 
     sealed class DictionaryAnalyzerConfigOptions(IReadOnlyDictionary<string, string> properties) : AnalyzerConfigOptions