From 853308301e9bc53b6e50b8e2ce29cff29f169be0 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Thu, 10 Sep 2026 14:32:56 +0500 Subject: [PATCH] IGNITE-29047 Add ignored failure count metric to FailureProcessor --- .../_docs/monitoring-metrics/new-metrics.adoc | 12 +++ .../processors/failure/FailureProcessor.java | 23 +++++ .../failure/FailureProcessorMetricsTest.java | 94 +++++++++++++++++++ .../testsuites/IgniteBasicTestSuite2.java | 2 + 4 files changed, 131 insertions(+) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/failure/FailureProcessorMetricsTest.java diff --git a/docs/_docs/monitoring-metrics/new-metrics.adoc b/docs/_docs/monitoring-metrics/new-metrics.adoc index b07cb629b4c26..1f111e7a0c72b 100644 --- a/docs/_docs/monitoring-metrics/new-metrics.adoc +++ b/docs/_docs/monitoring-metrics/new-metrics.adoc @@ -523,3 +523,15 @@ Register name: `sql.queries.user` |resultSetSizeHistogram| histogram | Histogram of fetched result set sizes for SQL queries. |maxResultSetSize| max value | Maximum fetched result set size for SQL queries. |=== + +== Failure + +Failure processor metrics. + +Register name: `failure` + +[cols="2,1,3",opts="header"] +|=== +|Name| Type| Description +|IgnoredFailuresCount| long | The number of failures suppressed in accordance with a configured failure handler. +|=== diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java index b2fabbe4109ef..6fc7466cb58b3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java @@ -34,12 +34,15 @@ import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.persistence.CorruptedDataStructureException; import org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor; +import org.apache.ignite.internal.processors.metric.MetricRegistryImpl; +import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; import static org.apache.ignite.IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE; import static org.apache.ignite.IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT; import static org.apache.ignite.IgniteSystemProperties.IGNITE_FAILURE_HANDLER_RESERVE_BUFFER_SIZE; +import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName; import static org.apache.ignite.internal.util.IgniteUtils.validateRamUsage; /** @@ -67,6 +70,15 @@ public class FailureProcessor extends GridProcessorAdapter { static final String FAILURE_LOG_MSG = "Critical system error detected. " + "Will be handled accordingly to configured handler "; + /** Failure metrics group name. */ + public static final String FAILURE_METRICS = metricName("failure"); + + /** Ignored failures count metric name. */ + public static final String IGNORED_FAILURES_CNT = "IgnoredFailuresCount"; + + /** Ignored failures count metric. */ + private volatile AtomicLongMetric ignoredFailuresCntMetric; + /** Thread dump per failure type timestamps. */ private final Map threadDumpPerFailureTypeTs; @@ -127,6 +139,14 @@ public FailureProcessor(GridKernalContext ctx) { U.quietAndInfo(log, "Configured failure handler: [hnd=" + hnd + ']'); } + /** {@inheritDoc} */ + @Override public void onKernalStart(boolean active) throws IgniteCheckedException { + MetricRegistryImpl mreg = ctx.metric().registry(FAILURE_METRICS); + + ignoredFailuresCntMetric = mreg.longMetric(IGNORED_FAILURES_CNT, + "The number of failures suppressed in accordance with a configured failure handler."); + } + /** * @return @{code True} if a node will be stopped by current handler in near time. */ @@ -176,6 +196,9 @@ public synchronized boolean process(FailureContext failureCtx, FailureHandler hn return false; if (failureTypeIgnored(failureCtx, hnd)) { + if (ignoredFailuresCntMetric != null) + ignoredFailuresCntMetric.increment(); + U.quietAndWarn(ignite.log(), IGNORED_FAILURE_LOG_MSG + "[hnd=" + hnd + ", failureCtx=" + failureCtx + ']', failureCtx.error()); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/failure/FailureProcessorMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/failure/FailureProcessorMetricsTest.java new file mode 100644 index 0000000000000..c4ac05f59cdf2 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/failure/FailureProcessorMetricsTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.failure; + +import java.util.Set; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.TestFailureHandler; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.spi.metric.LongMetric; +import org.apache.ignite.testframework.junits.WithSystemProperty; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static org.apache.ignite.failure.FailureType.SEGMENTATION; +import static org.apache.ignite.failure.FailureType.SYSTEM_CRITICAL_OPERATION_TIMEOUT; +import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_BLOCKED; +import static org.apache.ignite.internal.processors.failure.FailureProcessor.FAILURE_METRICS; +import static org.apache.ignite.internal.processors.failure.FailureProcessor.IGNORED_FAILURES_CNT; + +/** Tests that the failure processor counts suppressed (ignored) failures via a dedicated metric. */ +@WithSystemProperty(key = IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE, value = "false") +public class FailureProcessorMetricsTest extends GridCommonAbstractTest { + /** */ + private static final String ERR_MSG = "Failure context error"; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + TestFailureHandler hnd = new TestFailureHandler(false); + + hnd.setIgnoredFailureTypes(Set.of(SYSTEM_CRITICAL_OPERATION_TIMEOUT, SYSTEM_WORKER_BLOCKED)); + + cfg.setFailureHandler(hnd); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + super.afterTest(); + } + + /** + * Tests that the ignored failures count metric starts at zero and is incremented per each suppressed failure, + * while processed (not ignored) failures do not affect it. + */ + @Test + public void testIgnoredFailuresCountMetric() throws Exception { + IgniteEx ignite = startGrids(2); + + LongMetric ignoredFailuresCnt = ignoredFailuresMetric(ignite); + + assertEquals(0, ignoredFailuresCnt.value()); + + ignite.context().failure().process(new FailureContext(SYSTEM_CRITICAL_OPERATION_TIMEOUT, new Throwable(ERR_MSG))); + + assertEquals(1, ignoredFailuresCnt.value()); + + // A processed (not ignored) failure must not affect the ignored failures count. + ignite.context().failure().process(new FailureContext(SEGMENTATION, new Throwable(ERR_MSG))); + + assertEquals(1, ignoredFailuresCnt.value()); + + ignite.context().failure().process(new FailureContext(SYSTEM_WORKER_BLOCKED, new Throwable(ERR_MSG))); + + assertEquals(2, ignoredFailuresCnt.value()); + assertEquals(0, ignoredFailuresMetric(grid(1)).value()); + } + + /** */ + private LongMetric ignoredFailuresMetric(IgniteEx ignite) { + return ignite.context().metric().registry(FAILURE_METRICS).findMetric(IGNORED_FAILURES_CNT); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java index 9b5ca62ce8523..4a97a041dc2bd 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java @@ -76,6 +76,7 @@ import org.apache.ignite.internal.processors.database.IndexStorageSelfTest; import org.apache.ignite.internal.processors.database.SwapPathConstructionSelfTest; import org.apache.ignite.internal.processors.failure.FailureProcessorLoggingTest; +import org.apache.ignite.internal.processors.failure.FailureProcessorMetricsTest; import org.apache.ignite.internal.processors.failure.FailureProcessorThreadDumpThrottlingTest; import org.apache.ignite.internal.processors.metastorage.DistributedMetaStorageClassloadingTest; import org.apache.ignite.internal.processors.metastorage.DistributedMetaStorageTest; @@ -164,6 +165,7 @@ OomFailureHandlerTest.class, TransactionIntegrityWithSystemWorkerDeathTest.class, FailureProcessorLoggingTest.class, + FailureProcessorMetricsTest.class, FailureProcessorThreadDumpThrottlingTest.class, ExchangeTaskHandlerWaitingForTasksTest.class,