diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSource.java index 07ceaaf2e241..3e6968a29e78 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSource.java @@ -51,4 +51,9 @@ public interface MetricsMasterProcSource extends BaseSource { String NUM_MASTER_WALS_DESC = "Number of master WAL files"; + String OLDEST_PROCEDURE_AGE_NAME = "oldestProcedureAge"; + + String OLDEST_PROCEDURE_AGE_DESC = + "Age in milliseconds of the oldest active master procedure (Gauge)."; + } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java index 001ab0168145..5e1bcc0df3d2 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.master; +import java.util.Map; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; import org.apache.hadoop.hbase.metrics.Interns; import org.apache.hadoop.metrics2.MetricsCollector; @@ -56,6 +57,13 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { if (masterWrapper != null) { metricsRecordBuilder.addGauge(Interns.info(NUM_MASTER_WALS_NAME, NUM_MASTER_WALS_DESC), masterWrapper.getNumWALFiles()); + Map oldestProcedureAgeByType = masterWrapper.getOldestProcedureAgeByType(); + metricsRecordBuilder.addGauge( + Interns.info(OLDEST_PROCEDURE_AGE_NAME, OLDEST_PROCEDURE_AGE_DESC), + oldestProcedureAgeByType.values().stream().mapToLong(Long::longValue).max().orElse(0L)); + oldestProcedureAgeByType.forEach((procedureType, age) -> metricsRecordBuilder.addGauge( + Interns.info(OLDEST_PROCEDURE_AGE_NAME + "_" + procedureType, OLDEST_PROCEDURE_AGE_DESC), + age)); } metricsRegistry.snapshot(metricsRecordBuilder, all); diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java index 83419e2d5501..ad8728327014 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java @@ -123,6 +123,12 @@ public interface MetricsMasterWrapper { */ long getNumWALFiles(); + /** + * Get the age in milliseconds of the oldest active master procedure for each procedure type, + * keyed by the procedure's simple class name. + */ + Map getOldestProcedureAgeByType(); + /** * Get the number of region split plans executed. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java index ff6f5b8e5df8..7e2e24e1c4e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java @@ -29,8 +29,12 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.quotas.QuotaObserverChore; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.yetus.audience.InterfaceAudience; @@ -180,6 +184,23 @@ public long getNumWALFiles() { return master.getNumWALFiles(); } + @Override + public Map getOldestProcedureAgeByType() { + ProcedureExecutor procedureExecutor = master.getMasterProcedureExecutor(); + if (procedureExecutor == null) { + return Collections.emptyMap(); + } + long now = EnvironmentEdgeManager.currentTime(); + Map oldestProcedureAgeByType = new HashMap<>(); + for (Procedure procedure : procedureExecutor.getActiveProceduresNoCopy()) { + if (!procedure.isFinished()) { + long age = Math.max(0L, now - procedure.getSubmittedTime()); + oldestProcedureAgeByType.merge(procedure.getClass().getSimpleName(), age, Math::max); + } + } + return oldestProcedureAgeByType; + } + @Override public Map> getTableSpaceUtilization() { if (master == null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java new file mode 100644 index 000000000000..b481714c3b44 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hbase.CompatibilityFactory; +import org.apache.hadoop.hbase.master.locking.LockProcedure; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(MasterTests.TAG) +@Tag(SmallTests.TAG) +public class TestOldestProcedureMetrics { + + private static final MetricsAssertHelper METRICS_HELPER = + CompatibilityFactory.getInstance(MetricsAssertHelper.class); + + @AfterEach + public void resetEnvironmentEdge() { + EnvironmentEdgeManager.reset(); + } + + @Test + @SuppressWarnings("unchecked") + public void testOldestProcedureAge() { + ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); + edge.setValue(10_000L); + EnvironmentEdgeManager.injectEdge(edge); + + HMaster master = mock(HMaster.class); + ProcedureExecutor procedureExecutor = mock(ProcedureExecutor.class); + LockProcedure oldestLock = mock(LockProcedure.class); + LockProcedure newerLock = mock(LockProcedure.class); + ServerCrashProcedure serverCrash = mock(ServerCrashProcedure.class); + ServerCrashProcedure finished = mock(ServerCrashProcedure.class); + + when(master.getMasterProcedureExecutor()).thenReturn(procedureExecutor); + when(procedureExecutor.getActiveProceduresNoCopy()) + .thenReturn(List.of(oldestLock, newerLock, serverCrash, finished)); + when(oldestLock.getSubmittedTime()).thenReturn(1_000L); + when(newerLock.getSubmittedTime()).thenReturn(4_000L); + when(serverCrash.getSubmittedTime()).thenReturn(2_000L); + when(finished.isFinished()).thenReturn(true); + when(finished.getSubmittedTime()).thenReturn(100L); + + MetricsMasterWrapperImpl wrapper = new MetricsMasterWrapperImpl(master); + assertEquals(Map.of("LockProcedure", 9_000L, "ServerCrashProcedure", 8_000L), + wrapper.getOldestProcedureAgeByType()); + + MetricsMasterProcSource source = new MetricsMasterProcSourceImpl(wrapper); + METRICS_HELPER.assertGauge(MetricsMasterProcSource.OLDEST_PROCEDURE_AGE_NAME, 9_000L, source); + METRICS_HELPER.assertGauge(MetricsMasterProcSource.OLDEST_PROCEDURE_AGE_NAME + "_LockProcedure", + 9_000L, source); + METRICS_HELPER.assertGauge( + MetricsMasterProcSource.OLDEST_PROCEDURE_AGE_NAME + "_ServerCrashProcedure", 8_000L, source); + + when(procedureExecutor.getActiveProceduresNoCopy()).thenReturn(List.of()); + assertEquals(Map.of(), wrapper.getOldestProcedureAgeByType()); + + when(procedureExecutor.getActiveProceduresNoCopy()).thenReturn(List.of(newerLock)); + when(newerLock.getSubmittedTime()).thenReturn(11_000L); + assertEquals(Map.of("LockProcedure", 0L), wrapper.getOldestProcedureAgeByType()); + + when(master.getMasterProcedureExecutor()).thenReturn(null); + assertEquals(Map.of(), wrapper.getOldestProcedureAgeByType()); + } +}