From f036e9245ba0b6f49d5fbba8262e89a4920a0b5c Mon Sep 17 00:00:00 2001 From: mazhengxuan Date: Sun, 23 Aug 2026 23:54:12 +0800 Subject: [PATCH 1/2] HBASE-29640 Add metric for oldest procedure age --- .../hbase/master/MetricsMasterProcSource.java | 5 ++ .../master/MetricsMasterProcSourceImpl.java | 3 + .../hbase/master/MetricsMasterWrapper.java | 5 ++ .../master/MetricsMasterWrapperImpl.java | 15 ++++ .../master/TestOldestProcedureMetrics.java | 87 +++++++++++++++++++ 5 files changed, 115 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSource.java index 07ceaaf2e241..3e6968a29e78 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSource.java @@ -51,4 +51,9 @@ public interface MetricsMasterProcSource extends BaseSource { String NUM_MASTER_WALS_DESC = "Number of master WAL files"; + String OLDEST_PROCEDURE_AGE_NAME = "oldestProcedureAge"; + + String OLDEST_PROCEDURE_AGE_DESC = + "Age in milliseconds of the oldest active master procedure (Gauge)."; + } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java index 001ab0168145..abfb150400b3 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java @@ -56,6 +56,9 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { if (masterWrapper != null) { metricsRecordBuilder.addGauge(Interns.info(NUM_MASTER_WALS_NAME, NUM_MASTER_WALS_DESC), masterWrapper.getNumWALFiles()); + metricsRecordBuilder.addGauge( + Interns.info(OLDEST_PROCEDURE_AGE_NAME, OLDEST_PROCEDURE_AGE_DESC), + masterWrapper.getOldestProcedureAge()); } metricsRegistry.snapshot(metricsRecordBuilder, all); diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java index 83419e2d5501..e28e36e3f4c2 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java @@ -123,6 +123,11 @@ public interface MetricsMasterWrapper { */ long getNumWALFiles(); + /** + * Get the age in milliseconds of the oldest active master procedure. + */ + long getOldestProcedureAge(); + /** * Get the number of region split plans executed. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java index ff6f5b8e5df8..921c81c3de4e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java @@ -29,8 +29,11 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.quotas.QuotaObserverChore; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.yetus.audience.InterfaceAudience; @@ -180,6 +183,18 @@ public long getNumWALFiles() { return master.getNumWALFiles(); } + @Override + public long getOldestProcedureAge() { + ProcedureExecutor procedureExecutor = master.getMasterProcedureExecutor(); + if (procedureExecutor == null) { + return 0L; + } + long now = EnvironmentEdgeManager.currentTime(); + return procedureExecutor.getActiveProceduresNoCopy().stream() + .filter(procedure -> !procedure.isFinished()) + .mapToLong(procedure -> Math.max(0L, now - procedure.getSubmittedTime())).max().orElse(0L); + } + @Override public Map> getTableSpaceUtilization() { if (master == null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java new file mode 100644 index 000000000000..238947750d9c --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import org.apache.hadoop.hbase.CompatibilityFactory; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) +public class TestOldestProcedureMetrics { + + private static final MetricsAssertHelper METRICS_HELPER = + CompatibilityFactory.getInstance(MetricsAssertHelper.class); + + @AfterEach + public void resetEnvironmentEdge() { + EnvironmentEdgeManager.reset(); + } + + @Test + @SuppressWarnings("unchecked") + public void testOldestProcedureAge() { + ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); + edge.setValue(10_000L); + EnvironmentEdgeManager.injectEdge(edge); + + HMaster master = mock(HMaster.class); + ProcedureExecutor procedureExecutor = mock(ProcedureExecutor.class); + Procedure oldest = mock(Procedure.class); + Procedure newer = mock(Procedure.class); + Procedure finished = mock(Procedure.class); + + when(master.getMasterProcedureExecutor()).thenReturn(procedureExecutor); + when(procedureExecutor.getActiveProceduresNoCopy()) + .thenReturn(List.of(oldest, newer, finished)); + when(oldest.getSubmittedTime()).thenReturn(1_000L); + when(newer.getSubmittedTime()).thenReturn(4_000L); + when(finished.isFinished()).thenReturn(true); + when(finished.getSubmittedTime()).thenReturn(100L); + + MetricsMasterWrapperImpl wrapper = new MetricsMasterWrapperImpl(master); + assertEquals(9_000L, wrapper.getOldestProcedureAge()); + + MetricsMasterProcSource source = new MetricsMasterProcSourceImpl(wrapper); + METRICS_HELPER.assertGauge(MetricsMasterProcSource.OLDEST_PROCEDURE_AGE_NAME, 9_000L, source); + + when(procedureExecutor.getActiveProceduresNoCopy()).thenReturn(List.of()); + assertEquals(0L, wrapper.getOldestProcedureAge()); + + when(procedureExecutor.getActiveProceduresNoCopy()).thenReturn(List.of(newer)); + when(newer.getSubmittedTime()).thenReturn(11_000L); + assertEquals(0L, wrapper.getOldestProcedureAge()); + + when(master.getMasterProcedureExecutor()).thenReturn(null); + assertEquals(0L, wrapper.getOldestProcedureAge()); + } +} From 884636f55b7318b0c8b4872a972635cdc825715c Mon Sep 17 00:00:00 2001 From: mazhengxuan Date: Mon, 24 Aug 2026 11:16:25 +0800 Subject: [PATCH 2/2] HBASE-29640 Expose oldest procedure age by type --- .../master/MetricsMasterProcSourceImpl.java | 7 +++- .../hbase/master/MetricsMasterWrapper.java | 5 +-- .../master/MetricsMasterWrapperImpl.java | 16 ++++++--- .../master/TestOldestProcedureMetrics.java | 35 ++++++++++++------- 4 files changed, 42 insertions(+), 21 deletions(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java index abfb150400b3..5e1bcc0df3d2 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.master; +import java.util.Map; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; import org.apache.hadoop.hbase.metrics.Interns; import org.apache.hadoop.metrics2.MetricsCollector; @@ -56,9 +57,13 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { if (masterWrapper != null) { metricsRecordBuilder.addGauge(Interns.info(NUM_MASTER_WALS_NAME, NUM_MASTER_WALS_DESC), masterWrapper.getNumWALFiles()); + Map oldestProcedureAgeByType = masterWrapper.getOldestProcedureAgeByType(); metricsRecordBuilder.addGauge( Interns.info(OLDEST_PROCEDURE_AGE_NAME, OLDEST_PROCEDURE_AGE_DESC), - masterWrapper.getOldestProcedureAge()); + oldestProcedureAgeByType.values().stream().mapToLong(Long::longValue).max().orElse(0L)); + oldestProcedureAgeByType.forEach((procedureType, age) -> metricsRecordBuilder.addGauge( + Interns.info(OLDEST_PROCEDURE_AGE_NAME + "_" + procedureType, OLDEST_PROCEDURE_AGE_DESC), + age)); } metricsRegistry.snapshot(metricsRecordBuilder, all); diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java index e28e36e3f4c2..ad8728327014 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java @@ -124,9 +124,10 @@ public interface MetricsMasterWrapper { long getNumWALFiles(); /** - * Get the age in milliseconds of the oldest active master procedure. + * Get the age in milliseconds of the oldest active master procedure for each procedure type, + * keyed by the procedure's simple class name. */ - long getOldestProcedureAge(); + Map getOldestProcedureAgeByType(); /** * Get the number of region split plans executed. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java index 921c81c3de4e..7e2e24e1c4e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.quotas.QuotaObserverChore; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; @@ -184,15 +185,20 @@ public long getNumWALFiles() { } @Override - public long getOldestProcedureAge() { + public Map getOldestProcedureAgeByType() { ProcedureExecutor procedureExecutor = master.getMasterProcedureExecutor(); if (procedureExecutor == null) { - return 0L; + return Collections.emptyMap(); } long now = EnvironmentEdgeManager.currentTime(); - return procedureExecutor.getActiveProceduresNoCopy().stream() - .filter(procedure -> !procedure.isFinished()) - .mapToLong(procedure -> Math.max(0L, now - procedure.getSubmittedTime())).max().orElse(0L); + Map oldestProcedureAgeByType = new HashMap<>(); + for (Procedure procedure : procedureExecutor.getActiveProceduresNoCopy()) { + if (!procedure.isFinished()) { + long age = Math.max(0L, now - procedure.getSubmittedTime()); + oldestProcedureAgeByType.merge(procedure.getClass().getSimpleName(), age, Math::max); + } + } + return oldestProcedureAgeByType; } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java index 238947750d9c..b481714c3b44 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java @@ -22,9 +22,11 @@ import static org.mockito.Mockito.when; import java.util.List; +import java.util.Map; import org.apache.hadoop.hbase.CompatibilityFactory; +import org.apache.hadoop.hbase.master.locking.LockProcedure; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; -import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -56,32 +58,39 @@ public void testOldestProcedureAge() { HMaster master = mock(HMaster.class); ProcedureExecutor procedureExecutor = mock(ProcedureExecutor.class); - Procedure oldest = mock(Procedure.class); - Procedure newer = mock(Procedure.class); - Procedure finished = mock(Procedure.class); + LockProcedure oldestLock = mock(LockProcedure.class); + LockProcedure newerLock = mock(LockProcedure.class); + ServerCrashProcedure serverCrash = mock(ServerCrashProcedure.class); + ServerCrashProcedure finished = mock(ServerCrashProcedure.class); when(master.getMasterProcedureExecutor()).thenReturn(procedureExecutor); when(procedureExecutor.getActiveProceduresNoCopy()) - .thenReturn(List.of(oldest, newer, finished)); - when(oldest.getSubmittedTime()).thenReturn(1_000L); - when(newer.getSubmittedTime()).thenReturn(4_000L); + .thenReturn(List.of(oldestLock, newerLock, serverCrash, finished)); + when(oldestLock.getSubmittedTime()).thenReturn(1_000L); + when(newerLock.getSubmittedTime()).thenReturn(4_000L); + when(serverCrash.getSubmittedTime()).thenReturn(2_000L); when(finished.isFinished()).thenReturn(true); when(finished.getSubmittedTime()).thenReturn(100L); MetricsMasterWrapperImpl wrapper = new MetricsMasterWrapperImpl(master); - assertEquals(9_000L, wrapper.getOldestProcedureAge()); + assertEquals(Map.of("LockProcedure", 9_000L, "ServerCrashProcedure", 8_000L), + wrapper.getOldestProcedureAgeByType()); MetricsMasterProcSource source = new MetricsMasterProcSourceImpl(wrapper); METRICS_HELPER.assertGauge(MetricsMasterProcSource.OLDEST_PROCEDURE_AGE_NAME, 9_000L, source); + METRICS_HELPER.assertGauge(MetricsMasterProcSource.OLDEST_PROCEDURE_AGE_NAME + "_LockProcedure", + 9_000L, source); + METRICS_HELPER.assertGauge( + MetricsMasterProcSource.OLDEST_PROCEDURE_AGE_NAME + "_ServerCrashProcedure", 8_000L, source); when(procedureExecutor.getActiveProceduresNoCopy()).thenReturn(List.of()); - assertEquals(0L, wrapper.getOldestProcedureAge()); + assertEquals(Map.of(), wrapper.getOldestProcedureAgeByType()); - when(procedureExecutor.getActiveProceduresNoCopy()).thenReturn(List.of(newer)); - when(newer.getSubmittedTime()).thenReturn(11_000L); - assertEquals(0L, wrapper.getOldestProcedureAge()); + when(procedureExecutor.getActiveProceduresNoCopy()).thenReturn(List.of(newerLock)); + when(newerLock.getSubmittedTime()).thenReturn(11_000L); + assertEquals(Map.of("LockProcedure", 0L), wrapper.getOldestProcedureAgeByType()); when(master.getMasterProcedureExecutor()).thenReturn(null); - assertEquals(0L, wrapper.getOldestProcedureAge()); + assertEquals(Map.of(), wrapper.getOldestProcedureAgeByType()); } }