IGNITE-29047 Add ignored failure count metric to FailureProcessor

docs/_docs/monitoring-metrics/new-metrics.adoc

@@ -523,3 +523,15 @@ Register name: `sql.queries.user`
 |resultSetSizeHistogram| histogram | Histogram of fetched result set sizes for SQL queries.
 |maxResultSetSize| max value | Maximum fetched result set size for SQL queries.
 |===
+
+== Failure
+
+Failure processor metrics.
+
+Register name: `failure`
+
+[cols="2,1,3",opts="header"]
+|===
+|Name|    Type|    Description
+|IgnoredFailuresCount| long | The number of failures suppressed in accordance with a configured failure handler.
+|===

modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java

@@ -34,12 +34,15 @@
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.persistence.CorruptedDataStructureException;
 import org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor;
+import org.apache.ignite.internal.processors.metric.MetricRegistryImpl;
+import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_FAILURE_HANDLER_RESERVE_BUFFER_SIZE;
+import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName;
 import static org.apache.ignite.internal.util.IgniteUtils.validateRamUsage;
 
 /**
@@ -67,6 +70,15 @@ public class FailureProcessor extends GridProcessorAdapter {
     static final String FAILURE_LOG_MSG = "Critical system error detected. " +
         "Will be handled accordingly to configured handler ";
 
+    /** Failure metrics group name. */
+    public static final String FAILURE_METRICS = metricName("failure");
+
+    /** Ignored failures count metric name. */
+    public static final String IGNORED_FAILURES_CNT = "IgnoredFailuresCount";
+
+    /** Ignored failures count metric. */
+    private volatile AtomicLongMetric ignoredFailuresCntMetric;
+
     /** Thread dump per failure type timestamps. */
     private final Map<FailureType, Long> threadDumpPerFailureTypeTs;
 
@@ -127,6 +139,14 @@ public FailureProcessor(GridKernalContext ctx) {
         U.quietAndInfo(log, "Configured failure handler: [hnd=" + hnd + ']');
     }
 
+    /** {@inheritDoc} */
+    @Override public void onKernalStart(boolean active) throws IgniteCheckedException {
+        MetricRegistryImpl mreg = ctx.metric().registry(FAILURE_METRICS);
+
+        ignoredFailuresCntMetric = mreg.longMetric(IGNORED_FAILURES_CNT,
+            "The number of failures suppressed in accordance with a configured failure handler.");
+    }
+
     /**
      * @return @{code True} if a node will be stopped by current handler in near time.
      */
@@ -176,6 +196,9 @@ public synchronized boolean process(FailureContext failureCtx, FailureHandler hn
             return false;
 
         if (failureTypeIgnored(failureCtx, hnd)) {
+            if (ignoredFailuresCntMetric != null)
+                ignoredFailuresCntMetric.increment();
+
             U.quietAndWarn(ignite.log(), IGNORED_FAILURE_LOG_MSG +
                 "[hnd=" + hnd + ", failureCtx=" + failureCtx + ']', failureCtx.error());
         }

modules/core/src/test/java/org/apache/ignite/internal/processors/failure/FailureProcessorMetricsTest.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.failure;
+
+import java.util.Set;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.TestFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.spi.metric.LongMetric;
+import org.apache.ignite.testframework.junits.WithSystemProperty;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+import static org.apache.ignite.failure.FailureType.SEGMENTATION;
+import static org.apache.ignite.failure.FailureType.SYSTEM_CRITICAL_OPERATION_TIMEOUT;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_BLOCKED;
+import static org.apache.ignite.internal.processors.failure.FailureProcessor.FAILURE_METRICS;
+import static org.apache.ignite.internal.processors.failure.FailureProcessor.IGNORED_FAILURES_CNT;
+
+/** Tests that the failure processor counts suppressed (ignored) failures via a dedicated metric. */
+@WithSystemProperty(key = IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE, value = "false")
+public class FailureProcessorMetricsTest extends GridCommonAbstractTest {
+    /** */
+    private static final String ERR_MSG = "Failure context error";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        TestFailureHandler hnd = new TestFailureHandler(false);
+
+        hnd.setIgnoredFailureTypes(Set.of(SYSTEM_CRITICAL_OPERATION_TIMEOUT, SYSTEM_WORKER_BLOCKED));
+
+        cfg.setFailureHandler(hnd);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /**
+     * Tests that the ignored failures count metric starts at zero and is incremented per each suppressed failure,
+     * while processed (not ignored) failures do not affect it.
+     */
+    @Test
+    public void testIgnoredFailuresCountMetric() throws Exception {
+        IgniteEx ignite = startGrids(2);
+
+        LongMetric ignoredFailuresCnt = ignoredFailuresMetric(ignite);
+
+        assertEquals(0, ignoredFailuresCnt.value());
+
+        ignite.context().failure().process(new FailureContext(SYSTEM_CRITICAL_OPERATION_TIMEOUT, new Throwable(ERR_MSG)));
+
+        assertEquals(1, ignoredFailuresCnt.value());
+
+        // A processed (not ignored) failure must not affect the ignored failures count.
+        ignite.context().failure().process(new FailureContext(SEGMENTATION, new Throwable(ERR_MSG)));
+
+        assertEquals(1, ignoredFailuresCnt.value());
+
+        ignite.context().failure().process(new FailureContext(SYSTEM_WORKER_BLOCKED, new Throwable(ERR_MSG)));
+
+        assertEquals(2, ignoredFailuresCnt.value());
+        assertEquals(0, ignoredFailuresMetric(grid(1)).value());
+    }
+
+    /** */
+    private LongMetric ignoredFailuresMetric(IgniteEx ignite) {
+        return ignite.context().metric().registry(FAILURE_METRICS).findMetric(IGNORED_FAILURES_CNT);
+    }
+}

modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java

@@ -76,6 +76,7 @@
 import org.apache.ignite.internal.processors.database.IndexStorageSelfTest;
 import org.apache.ignite.internal.processors.database.SwapPathConstructionSelfTest;
 import org.apache.ignite.internal.processors.failure.FailureProcessorLoggingTest;
+import org.apache.ignite.internal.processors.failure.FailureProcessorMetricsTest;
 import org.apache.ignite.internal.processors.failure.FailureProcessorThreadDumpThrottlingTest;
 import org.apache.ignite.internal.processors.metastorage.DistributedMetaStorageClassloadingTest;
 import org.apache.ignite.internal.processors.metastorage.DistributedMetaStorageTest;
@@ -164,6 +165,7 @@
     OomFailureHandlerTest.class,
     TransactionIntegrityWithSystemWorkerDeathTest.class,
     FailureProcessorLoggingTest.class,
+    FailureProcessorMetricsTest.class,
     FailureProcessorThreadDumpThrottlingTest.class,
     ExchangeTaskHandlerWaitingForTasksTest.class,