From b34ec3e4d01b23c724805ecae9998012d8c1c362 Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Tue, 22 Sep 2026 21:07:32 +0530 Subject: [PATCH 1/3] HIVE-30091: Fix K8s operator LLAP/TezAM replica resolution when autoscaling is disabled --- .../autoscaling/HiveClusterAutoscaler.java | 8 +++ .../dependent/HiveDependentResource.java | 9 +++ .../reconciler/HiveClusterReconciler.java | 67 +++++++++---------- 3 files changed, 47 insertions(+), 37 deletions(-) diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java index f6016043fc63..1389ce55ed4c 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java @@ -81,6 +81,14 @@ public static void setManagedReplicas(String namespace, String clusterName, MANAGED_REPLICAS.put(cacheKey(namespace, clusterName, component), replicas); } + /** + * Removes the autoscaler-managed replica count for a component. Used when autoscaling is disabled so + * a pre-existing in-memory scale decision if any is cleared up. + */ + public static void cleanupManagedReplicas(String namespace, String clusterName, String component) { + MANAGED_REPLICAS.remove(cacheKey(namespace, clusterName, component)); + } + private record PendingScaleDown(int targetReplicas, Instant annotatedAt, List podsToDeregister) {} private final BackgroundMetricsScraper bgScraper; diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java index a64440acab8e..94cec69508f8 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java @@ -184,6 +184,15 @@ protected Integer resolveReplicaCount(P primary, Context

context, } } if (autoscaling == null || !autoscaling.isEnabled()) { + if (primary instanceof HiveCluster hc) { + String component = getComponentName(); + if (component != null) { + HiveClusterAutoscaler.cleanupManagedReplicas( + hc.getMetadata().getNamespace(), + hc.getMetadata().getName(), + component); + } + } return staticReplicas; } Optional existing = getSecondaryResource(primary, context); diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java index 557ebe99d545..95f3336b9bda 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java @@ -713,14 +713,15 @@ private int resolveLlapReplicaCount(HiveCluster resource, return 0; } String componentKey = ConfigUtils.llapComponentKey(llapSpec.name()); - Integer managed = HiveClusterAutoscaler.getManagedReplicas(ns, clusterName, componentKey); - if (managed != null) { - return managed; - } - // First reconcile before autoscaler runs: start at minReplicas if autoscaling enabled if (llapSpec.autoscaling().isEnabled()) { + Integer managed = HiveClusterAutoscaler.getManagedReplicas(ns, clusterName, componentKey); + if (managed != null) { + return managed; + } + // First reconcile before autoscaler runs: start at minReplicas if autoscaling enabled return llapSpec.autoscaling().minReplicas(); } + HiveClusterAutoscaler.cleanupManagedReplicas(ns, clusterName, componentKey); return llapSpec.replicas(); } @@ -734,21 +735,19 @@ private int resolveTezAmReplicaCount(HiveCluster resource, return 0; } LlapSpec.LlapTezAmSpec tezAmSpec = llapSpec.tezAm(); - // Check if autoscaler has a managed value for this specific TezAM String tezAmComponentKey = ConfigUtils.tezAmComponentKey(llapSpec.name()); - Integer tezAmManaged = HiveClusterAutoscaler.getManagedReplicas(ns, clusterName, tezAmComponentKey); - if (tezAmManaged != null) { - return tezAmManaged; - } - // TezAM follows LLAP's autoscaling gate: only run if LLAP is running. - String llapComponentKey = ConfigUtils.llapComponentKey(llapSpec.name()); - Integer llapManaged = HiveClusterAutoscaler.getManagedReplicas(ns, clusterName, llapComponentKey); - if (llapManaged != null && llapManaged == 0) { - return 0; + // Check if autoscaler has a managed value for this specific TezAM + if (tezAmSpec.autoscaling().isEnabled()) { + Integer tezAmManaged = HiveClusterAutoscaler.getManagedReplicas(ns, clusterName, tezAmComponentKey); + if (tezAmManaged != null) { + return tezAmManaged; + } + } else { + HiveClusterAutoscaler.cleanupManagedReplicas(ns, clusterName, tezAmComponentKey); } - if (llapSpec.autoscaling().isEnabled() && llapManaged == null - && llapSpec.autoscaling().minReplicas() == 0) { - // First reconcile before autoscaler runs: LLAP starts at 0, so TezAM stays down too. + + int llapDesired = resolveLlapReplicaCount(resource, llapSpec, ns, clusterName); + if (llapDesired == 0) { return 0; } if (tezAmSpec.autoscaling().isEnabled()) { @@ -1012,35 +1011,29 @@ private void wakeCluster(HiveCluster resource) { // the dependent resources (Deployments/StatefulSets) on the next reconcile // and use these values for spec.replicas. We don't call patchReplicas() // because the workloads may have been garbage-collected while suspended. - // With autoscaling disabled the wake value is the spec's static replica - // count — using minReplicas (0 by default) would pin the component to 0. - int hs2Wake = spec.hiveServer2().autoscaling().isEnabled() - ? Math.max(1, spec.hiveServer2().autoscaling().minReplicas()) - : spec.hiveServer2().replicas(); - HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_HIVESERVER2, hs2Wake); + if (spec.hiveServer2().autoscaling().isEnabled()) { + HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_HIVESERVER2, + Math.max(1, spec.hiveServer2().autoscaling().minReplicas())); + } - if (spec.metastore().isEnabled() && spec.autoSuspend().includeMetastore()) { - int hmsWake = spec.metastore().autoscaling().isEnabled() - ? Math.max(1, spec.metastore().autoscaling().minReplicas()) - : spec.metastore().replicas(); - HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_METASTORE, hmsWake); + if (spec.metastore().isEnabled() && spec.autoSuspend().includeMetastore() + && spec.metastore().autoscaling().isEnabled()) { + HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_METASTORE, + Math.max(1, spec.metastore().autoscaling().minReplicas())); } for (var llap : spec.llapClusters()) { - if (llap.isEnabled()) { - int llapWake = llap.autoscaling().isEnabled() - ? llap.autoscaling().minReplicas() : llap.replicas(); - HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.llapComponentKey(llap.name()), llapWake); + if (llap.isEnabled() && llap.autoscaling().isEnabled()) { + HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.llapComponentKey(llap.name()), + llap.autoscaling().minReplicas()); } } if (spec.tezAm().isEnabled()) { for (var llap : spec.llapClusters()) { - if (llap.isEnabled()) { - int tezWake = llap.tezAm().autoscaling().isEnabled() - ? llap.tezAm().autoscaling().minReplicas() : llap.tezAm().replicas(); + if (llap.isEnabled() && llap.tezAm().autoscaling().isEnabled()) { HiveClusterAutoscaler.setManagedReplicas(ns, name, - ConfigUtils.tezAmComponentKey(llap.name()), tezWake); + ConfigUtils.tezAmComponentKey(llap.name()), llap.tezAm().autoscaling().minReplicas()); } } } From 8d08f2a60a53a5ad7182779ae62c7795319569be Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Wed, 23 Sep 2026 14:18:17 +0530 Subject: [PATCH 2/3] Address copilot comments --- .../autoscaling/HiveClusterAutoscaler.java | 11 ++++ .../dependent/HiveDependentResource.java | 9 ---- .../reconciler/HiveClusterReconciler.java | 51 +++++++++++++++++-- 3 files changed, 57 insertions(+), 14 deletions(-) diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java index 1389ce55ed4c..5be325e40853 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java @@ -89,6 +89,17 @@ public static void cleanupManagedReplicas(String namespace, String clusterName, MANAGED_REPLICAS.remove(cacheKey(namespace, clusterName, component)); } + /** + * Clears all in-memory autoscaling state for a component when autoscaling is disabled. + */ + public void resetComponentAutoscalingState(String namespace, String clusterName, String component) { + String key = cacheKey(namespace, clusterName, component); + MANAGED_REPLICAS.remove(key); + pendingScaleDowns.remove(key); + autoscalers.remove(key); + lastScaleTimes.remove(key); + } + private record PendingScaleDown(int targetReplicas, Instant annotatedAt, List podsToDeregister) {} private final BackgroundMetricsScraper bgScraper; diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java index 94cec69508f8..a64440acab8e 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java @@ -184,15 +184,6 @@ protected Integer resolveReplicaCount(P primary, Context

context, } } if (autoscaling == null || !autoscaling.isEnabled()) { - if (primary instanceof HiveCluster hc) { - String component = getComponentName(); - if (component != null) { - HiveClusterAutoscaler.cleanupManagedReplicas( - hc.getMetadata().getNamespace(), - hc.getMetadata().getName(), - component); - } - } return staticReplicas; } Optional existing = getSecondaryResource(primary, context); diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java index 95f3336b9bda..d9a67762de59 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java @@ -162,6 +162,15 @@ public UpdateControl reconcile(HiveCluster resource, Context reconcile(HiveCluster resource, Context entry : eval.patches().entrySet()) { patchReplicas(client, resource, entry.getKey(), entry.getValue()); @@ -553,6 +561,42 @@ private HiveClusterAutoscaler getOrCreateAutoscaler(KubernetesClient client) { return autoscaler; } + /** + * Clears autoscaling state for components with autoscaling disabled in spec. + */ + private static void resetAutoscalingStateForDisabledComponents(HiveCluster resource, HiveClusterAutoscaler scaler) { + HiveClusterSpec spec = resource.getSpec(); + String ns = resource.getMetadata().getNamespace(); + String clusterName = resource.getMetadata().getName(); + + if (!spec.hiveServer2().autoscaling().isEnabled()) { + resetComponentAutoscalingState(scaler, ns, clusterName, ConfigUtils.COMPONENT_HIVESERVER2); + } + if (spec.metastore().isEnabled() && !spec.metastore().autoscaling().isEnabled()) { + resetComponentAutoscalingState(scaler, ns, clusterName, ConfigUtils.COMPONENT_METASTORE); + } + for (var llap : spec.llapClusters()) { + if (!llap.isEnabled()) { + continue; + } + if (!llap.autoscaling().isEnabled()) { + resetComponentAutoscalingState(scaler, ns, clusterName, ConfigUtils.llapComponentKey(llap.name())); + } + if (spec.tezAm().isEnabled() && !llap.tezAm().autoscaling().isEnabled()) { + resetComponentAutoscalingState(scaler, ns, clusterName, ConfigUtils.tezAmComponentKey(llap.name())); + } + } + } + + private static void resetComponentAutoscalingState(HiveClusterAutoscaler scaler, + String namespace, String clusterName, String component) { + if (scaler != null) { + scaler.resetComponentAutoscalingState(namespace, clusterName, component); + } else { + HiveClusterAutoscaler.cleanupManagedReplicas(namespace, clusterName, component); + } + } + private static boolean anyAutoscalingEnabled(HiveClusterSpec spec) { if (spec.hiveServer2().autoscaling().isEnabled()) { return true; @@ -721,7 +765,6 @@ private int resolveLlapReplicaCount(HiveCluster resource, // First reconcile before autoscaler runs: start at minReplicas if autoscaling enabled return llapSpec.autoscaling().minReplicas(); } - HiveClusterAutoscaler.cleanupManagedReplicas(ns, clusterName, componentKey); return llapSpec.replicas(); } @@ -742,8 +785,6 @@ private int resolveTezAmReplicaCount(HiveCluster resource, if (tezAmManaged != null) { return tezAmManaged; } - } else { - HiveClusterAutoscaler.cleanupManagedReplicas(ns, clusterName, tezAmComponentKey); } int llapDesired = resolveLlapReplicaCount(resource, llapSpec, ns, clusterName); From 4a0cafee1c101882de1bb3b9b420867c09c86b1c Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Wed, 23 Sep 2026 16:35:35 +0530 Subject: [PATCH 3/3] Rename method to clearAutoscalingState to avoid shadowing --- .../operator/reconciler/HiveClusterReconciler.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java index d9a67762de59..ccb1390c90bd 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java @@ -570,25 +570,25 @@ private static void resetAutoscalingStateForDisabledComponents(HiveCluster resou String clusterName = resource.getMetadata().getName(); if (!spec.hiveServer2().autoscaling().isEnabled()) { - resetComponentAutoscalingState(scaler, ns, clusterName, ConfigUtils.COMPONENT_HIVESERVER2); + clearAutoscalingState(scaler, ns, clusterName, ConfigUtils.COMPONENT_HIVESERVER2); } if (spec.metastore().isEnabled() && !spec.metastore().autoscaling().isEnabled()) { - resetComponentAutoscalingState(scaler, ns, clusterName, ConfigUtils.COMPONENT_METASTORE); + clearAutoscalingState(scaler, ns, clusterName, ConfigUtils.COMPONENT_METASTORE); } for (var llap : spec.llapClusters()) { if (!llap.isEnabled()) { continue; } if (!llap.autoscaling().isEnabled()) { - resetComponentAutoscalingState(scaler, ns, clusterName, ConfigUtils.llapComponentKey(llap.name())); + clearAutoscalingState(scaler, ns, clusterName, ConfigUtils.llapComponentKey(llap.name())); } if (spec.tezAm().isEnabled() && !llap.tezAm().autoscaling().isEnabled()) { - resetComponentAutoscalingState(scaler, ns, clusterName, ConfigUtils.tezAmComponentKey(llap.name())); + clearAutoscalingState(scaler, ns, clusterName, ConfigUtils.tezAmComponentKey(llap.name())); } } } - private static void resetComponentAutoscalingState(HiveClusterAutoscaler scaler, + private static void clearAutoscalingState(HiveClusterAutoscaler scaler, String namespace, String clusterName, String component) { if (scaler != null) { scaler.resetComponentAutoscalingState(namespace, clusterName, component);