Skip to content

HIVE-30091: Fix K8s operator LLAP/TezAM replica resolution when autos… #6814

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ayushtkn merged 3 commits into from
Sep 24, 2026
+92 −39
Merged

HIVE-30091: Fix K8s operator LLAP/TezAM replica resolution when autos… #6814

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b34ec3e
HIVE-30091: Fix K8s operator LLAP/TezAM replica resolution when autos…
tanishq-chugh Sep 22, 2026
8d08f2a
Address copilot comments
tanishq-chugh Sep 23, 2026
4a0cafe
Rename method to clearAutoscalingState to avoid shadowing
tanishq-chugh Sep 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions ...netes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,25 @@ public static void setManagedReplicas(String namespace, String clusterName,
MANAGED_REPLICAS.put(cacheKey(namespace, clusterName, component), replicas);
}

/**
* Removes the autoscaler-managed replica count for a component. Used when autoscaling is disabled so
* a pre-existing in-memory scale decision if any is cleared up.
*/
public static void cleanupManagedReplicas(String namespace, String clusterName, String component) {
MANAGED_REPLICAS.remove(cacheKey(namespace, clusterName, component));
Comment on lines +88 to +89

@tanishq-chugh tanishq-chugh Sep 23, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed in commit: 8d08f2a

}

/**
* Clears all in-memory autoscaling state for a component when autoscaling is disabled.
*/
public void resetComponentAutoscalingState(String namespace, String clusterName, String component) {
String key = cacheKey(namespace, clusterName, component);
MANAGED_REPLICAS.remove(key);
pendingScaleDowns.remove(key);
autoscalers.remove(key);
lastScaleTimes.remove(key);
}

private record PendingScaleDown(int targetReplicas, Instant annotatedAt, List<String> podsToDeregister) {}

private final BackgroundMetricsScraper bgScraper;
Expand Down
112 changes: 73 additions & 39 deletions ...rnetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,15 +162,23 @@ public UpdateControl<HiveCluster> reconcile(HiveCluster resource, Context<HiveCl
break;
}

boolean canEvaluateAutoscaling = (rescheduleSeconds == 0 && anyAutoscalingEnabled(resource.getSpec()));

HiveClusterAutoscaler scaler = autoscaler;
if (scaler == null && canEvaluateAutoscaling) {
scaler = getOrCreateAutoscaler(client);
}

resetAutoscalingStateForDisabledComponents(resource, scaler);

// --- Imperative LLAP cluster management ---
// Must also run while suspended: resolve*ReplicaCount() return 0 when
// spec.suspend() is set, and skipping the call would leave LLAP/TezAM
// at their pre-suspend scale.
reconcileLlapClusters(resource, client);

// --- Autoscaling evaluation (only when enabled and not suspended) ---
if (rescheduleSeconds == 0 && anyAutoscalingEnabled(resource.getSpec())) {
HiveClusterAutoscaler scaler = getOrCreateAutoscaler(client);
if (canEvaluateAutoscaling) {
HiveClusterAutoscaler.AutoscalingEvaluation eval = scaler.evaluate(resource, client);
for (Map.Entry<String, Integer> entry : eval.patches().entrySet()) {
patchReplicas(client, resource, entry.getKey(), entry.getValue());
Expand Down Expand Up @@ -553,6 +561,42 @@ private HiveClusterAutoscaler getOrCreateAutoscaler(KubernetesClient client) {
return autoscaler;
}

/**
* Clears autoscaling state for components with autoscaling disabled in spec.
*/
private static void resetAutoscalingStateForDisabledComponents(HiveCluster resource, HiveClusterAutoscaler scaler) {
HiveClusterSpec spec = resource.getSpec();
String ns = resource.getMetadata().getNamespace();
String clusterName = resource.getMetadata().getName();

if (!spec.hiveServer2().autoscaling().isEnabled()) {
clearAutoscalingState(scaler, ns, clusterName, ConfigUtils.COMPONENT_HIVESERVER2);
}
if (spec.metastore().isEnabled() && !spec.metastore().autoscaling().isEnabled()) {
clearAutoscalingState(scaler, ns, clusterName, ConfigUtils.COMPONENT_METASTORE);
}
for (var llap : spec.llapClusters()) {
if (!llap.isEnabled()) {
continue;
}
if (!llap.autoscaling().isEnabled()) {
clearAutoscalingState(scaler, ns, clusterName, ConfigUtils.llapComponentKey(llap.name()));
}
if (spec.tezAm().isEnabled() && !llap.tezAm().autoscaling().isEnabled()) {
clearAutoscalingState(scaler, ns, clusterName, ConfigUtils.tezAmComponentKey(llap.name()));
}
}
}

private static void clearAutoscalingState(HiveClusterAutoscaler scaler,
String namespace, String clusterName, String component) {
if (scaler != null) {
scaler.resetComponentAutoscalingState(namespace, clusterName, component);
} else {
HiveClusterAutoscaler.cleanupManagedReplicas(namespace, clusterName, component);
}
}

private static boolean anyAutoscalingEnabled(HiveClusterSpec spec) {
if (spec.hiveServer2().autoscaling().isEnabled()) {
return true;
Expand Down Expand Up @@ -713,12 +757,12 @@ private int resolveLlapReplicaCount(HiveCluster resource,
return 0;
}
String componentKey = ConfigUtils.llapComponentKey(llapSpec.name());
Integer managed = HiveClusterAutoscaler.getManagedReplicas(ns, clusterName, componentKey);
if (managed != null) {
return managed;
}
// First reconcile before autoscaler runs: start at minReplicas if autoscaling enabled
if (llapSpec.autoscaling().isEnabled()) {
Integer managed = HiveClusterAutoscaler.getManagedReplicas(ns, clusterName, componentKey);
if (managed != null) {
return managed;
}
// First reconcile before autoscaler runs: start at minReplicas if autoscaling enabled
return llapSpec.autoscaling().minReplicas();
}
return llapSpec.replicas();
Expand All @@ -734,21 +778,17 @@ private int resolveTezAmReplicaCount(HiveCluster resource,
return 0;
}
LlapSpec.LlapTezAmSpec tezAmSpec = llapSpec.tezAm();
// Check if autoscaler has a managed value for this specific TezAM
String tezAmComponentKey = ConfigUtils.tezAmComponentKey(llapSpec.name());
Integer tezAmManaged = HiveClusterAutoscaler.getManagedReplicas(ns, clusterName, tezAmComponentKey);
if (tezAmManaged != null) {
return tezAmManaged;
}
// TezAM follows LLAP's autoscaling gate: only run if LLAP is running.
String llapComponentKey = ConfigUtils.llapComponentKey(llapSpec.name());
Integer llapManaged = HiveClusterAutoscaler.getManagedReplicas(ns, clusterName, llapComponentKey);
if (llapManaged != null && llapManaged == 0) {
return 0;
// Check if autoscaler has a managed value for this specific TezAM
if (tezAmSpec.autoscaling().isEnabled()) {
Integer tezAmManaged = HiveClusterAutoscaler.getManagedReplicas(ns, clusterName, tezAmComponentKey);
if (tezAmManaged != null) {
return tezAmManaged;
}
}
if (llapSpec.autoscaling().isEnabled() && llapManaged == null
&& llapSpec.autoscaling().minReplicas() == 0) {
// First reconcile before autoscaler runs: LLAP starts at 0, so TezAM stays down too.

int llapDesired = resolveLlapReplicaCount(resource, llapSpec, ns, clusterName);
if (llapDesired == 0) {
return 0;
}
if (tezAmSpec.autoscaling().isEnabled()) {
Expand Down Expand Up @@ -1012,35 +1052,29 @@ private void wakeCluster(HiveCluster resource) {
// the dependent resources (Deployments/StatefulSets) on the next reconcile
// and use these values for spec.replicas. We don't call patchReplicas()
// because the workloads may have been garbage-collected while suspended.
// With autoscaling disabled the wake value is the spec's static replica
// count — using minReplicas (0 by default) would pin the component to 0.
int hs2Wake = spec.hiveServer2().autoscaling().isEnabled()
? Math.max(1, spec.hiveServer2().autoscaling().minReplicas())
: spec.hiveServer2().replicas();
HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_HIVESERVER2, hs2Wake);
if (spec.hiveServer2().autoscaling().isEnabled()) {
HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_HIVESERVER2,
Math.max(1, spec.hiveServer2().autoscaling().minReplicas()));
}

if (spec.metastore().isEnabled() && spec.autoSuspend().includeMetastore()) {
int hmsWake = spec.metastore().autoscaling().isEnabled()
? Math.max(1, spec.metastore().autoscaling().minReplicas())
: spec.metastore().replicas();
HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_METASTORE, hmsWake);
if (spec.metastore().isEnabled() && spec.autoSuspend().includeMetastore()
&& spec.metastore().autoscaling().isEnabled()) {
HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_METASTORE,
Math.max(1, spec.metastore().autoscaling().minReplicas()));
}

for (var llap : spec.llapClusters()) {
if (llap.isEnabled()) {
int llapWake = llap.autoscaling().isEnabled()
? llap.autoscaling().minReplicas() : llap.replicas();
HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.llapComponentKey(llap.name()), llapWake);
if (llap.isEnabled() && llap.autoscaling().isEnabled()) {
HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.llapComponentKey(llap.name()),
llap.autoscaling().minReplicas());
}
}

if (spec.tezAm().isEnabled()) {
for (var llap : spec.llapClusters()) {
if (llap.isEnabled()) {
int tezWake = llap.tezAm().autoscaling().isEnabled()
? llap.tezAm().autoscaling().minReplicas() : llap.tezAm().replicas();
if (llap.isEnabled() && llap.tezAm().autoscaling().isEnabled()) {
HiveClusterAutoscaler.setManagedReplicas(ns, name,
ConfigUtils.tezAmComponentKey(llap.name()), tezWake);
ConfigUtils.tezAmComponentKey(llap.name()), llap.tezAm().autoscaling().minReplicas());
}
}
}
Expand Down
Loading